From ca27f60bc4d5c0d10a110e16b43ae930ea085af2 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Wed, 19 Aug 2026 16:59:28 +0700 Subject: [PATCH 01/29] fix(darwin): probe interception after the host wakes Rule text survives a suspend unchanged, so the watchdog reports the pf anchor intact while nothing translates, and the probe monitor is only armed by interface changes with a schedule frozen through the sleep. Detect the resume from the wall-clock gap and probe from there, with a bounded repair: at most two forced reloads across the retry schedule. Refresh the OS resolver before every probe - the probe aims at the first OS nameserver and passes when it has none, so a pre-sleep list makes it meaningless. Stand down for stabilization, which verifies on completion. --- cmd/cli/dns_intercept_darwin.go | 253 ++++++++++++++-- cmd/cli/dns_intercept_darwin_test.go | 422 +++++++++++++++++++++++++++ cmd/cli/prog.go | 6 + cmd/cli/suspend_detector.go | 109 +++++++ cmd/cli/suspend_detector_test.go | 208 +++++++++++++ 5 files changed, 971 insertions(+), 27 deletions(-) create mode 100644 cmd/cli/suspend_detector.go create mode 100644 cmd/cli/suspend_detector_test.go diff --git a/cmd/cli/dns_intercept_darwin.go b/cmd/cli/dns_intercept_darwin.go index 9a6e5b8b..eb9eafc9 100644 --- a/cmd/cli/dns_intercept_darwin.go +++ b/cmd/cli/dns_intercept_darwin.go @@ -209,6 +209,44 @@ func setCtrldGroupID(gid int) error { func (p *prog) startDNSIntercept() error { mainLog.Load().Info().Msg("DNS intercept: initializing macOS packet filter (pf) redirect") + if err := installPFInterceptFn(p); err != nil { + return err + } + + p.dnsInterceptState = &pfState{ + anchorFile: pfAnchorFile, + anchorName: pfAnchorName, + } + + // Store the initial set of tunnel interfaces so we can detect changes later. + p.mu.Lock() + p.lastTunnelIfaces = discoverTunnelInterfacesForReconcile() + p.pendingTunnelIfaces = nil + p.hasPendingTunnelIfaces = false + p.mu.Unlock() + + lc := p.cfg.FirstListener() + if lc != nil { + mainLog.Load().Info().Msgf("DNS intercept: pf redirect active - all outbound DNS (port 53) redirected to %s:%d via anchor %q", lc.IP, lc.Port, pfAnchorName) + } else { + mainLog.Load().Info().Msgf("DNS intercept: pf redirect active - all outbound DNS (port 53) redirected via anchor %q", pfAnchorName) + } + + p.startInterceptBackgroundWork() + + return nil +} + +// installPFInterceptFn is the privileged-install seam. Everything behind it needs root, +// a live pf and a writable /etc/pf.anchors - which is what kept startDNSIntercept, and +// with it the handoff to the background watchers, out of reach of every test. +var installPFInterceptFn = (*prog).installPFIntercept + +// installPFIntercept performs the privileged part of starting interception: validation, +// _ctrld group setup, anchor file write, pfctl load, enable, and post-load verification. +// It returns nil only once pf is authoritatively active, so the caller may publish +// intercept mode; on failure it has already rolled back whatever it applied. +func (p *prog) installPFIntercept() error { if err := p.validateDNSIntercept(); err != nil { return err } @@ -316,30 +354,31 @@ func (p *prog) startDNSIntercept() error { return fmt.Errorf("dns intercept: post-load PF verification failed") } - p.dnsInterceptState = &pfState{ - anchorFile: pfAnchorFile, - anchorName: pfAnchorName, - } - - // Store the initial set of tunnel interfaces so we can detect changes later. - p.mu.Lock() - p.lastTunnelIfaces = discoverTunnelInterfacesForReconcile() - p.pendingTunnelIfaces = nil - p.hasPendingTunnelIfaces = false - p.mu.Unlock() - - lc := p.cfg.FirstListener() - if lc != nil { - mainLog.Load().Info().Msgf("DNS intercept: pf redirect active — all outbound DNS (port 53) redirected to %s:%d via anchor %q", lc.IP, lc.Port, pfAnchorName) - } else { - mainLog.Load().Info().Msgf("DNS intercept: pf redirect active — all outbound DNS (port 53) redirected via anchor %q", pfAnchorName) - } + return nil +} - // Start the pf watchdog to detect and restore rules if another program - // (e.g., Windscribe desktop, macOS configd) replaces the pf ruleset. - go p.pfWatchdog() +// pfWatchdogFn and runSuspendWatcherFn are the background-work seams. Tests use them to +// observe what startInterceptBackgroundWork registers without running either loop, which +// would otherwise reach pfctl. +var ( + pfWatchdogFn = (*prog).pfWatchdog + runSuspendWatcherFn = runSuspendWatcher +) - return nil +// startInterceptBackgroundWork launches the goroutines that guard an active intercept. +// +// Split out of startDNSIntercept so the registration is reachable from a test: +// startDNSIntercept needs root, a live pf and a writable /etc/pf.anchors, so nothing +// covers this wiring if it lives inline there. +func (p *prog) startInterceptBackgroundWork() { + // Detect and restore rules if another program (e.g., Windscribe desktop, macOS + // configd) replaces the pf ruleset. + go pfWatchdogFn(p) + + // A host sleep freezes every timer above it, and rule text survives the suspend + // unchanged, so no existing path notices that pf stopped translating while the host + // was asleep. Watch for the resume itself and probe from there. + go runSuspendWatcherFn(p.stopCh, p.verifyInterceptAfterWake) } func (p *prog) rollbackDNSInterceptStart() { @@ -518,6 +557,10 @@ func (p *prog) stopDNSIntercept() error { } } p.pfDelayedRecheckTimers = nil + if p.pfSettleFollowupTimer != nil { + p.pfSettleFollowupTimer.Stop() + p.pfSettleFollowupTimer = nil + } p.pfDelayedRecheckMu.Unlock() for !p.pfEnsureRunning.CompareAndSwap(false, true) { @@ -1525,6 +1568,146 @@ func (p *prog) verifyInterceptAfterStabilization() { mainLog.Load().Error().Msg("DNS intercept: interception still not translating after the post-stabilization reload — the watchdog will retry") } +// pfWakeProbeDelays is the bounded retry schedule for the post-resume functional probe. +// Each value is the wait before its own attempt, not an offset from the resume, so the +// attempts land at +0s, +2s, +6s and +14s after the suspend is noticed. A resumed host +// brings its link, default route, resolver list and pf translation state back over +// several seconds, so a single probe at +0 decides too early. A var so tests can shorten +// it. +var pfWakeProbeDelays = []time.Duration{0, 2 * time.Second, 4 * time.Second, 8 * time.Second} + +// pfWakeMaxRepairs bounds forced reloads across the whole schedule. QA on the post-wake +// outage saw one forced reload fail to restore translation and a later one succeed, so a +// single attempt is not enough - but this stays a bounded repair rather than a reload +// loop, because every reload flushes pf state and kills in-flight DoH connections. +const pfWakeMaxRepairs = 2 + +// pfWakeProbeOwnerWait bounds how long one attempt waits for another prober to release +// functional-probe ownership. Short on purpose: a holder that is genuinely probing is +// doing this path's job, and there are further attempts left in the schedule. +var pfWakeProbeOwnerWait = 250 * time.Millisecond + +// verifyInterceptAfterWake proves pf is actually translating after the host resumes from +// sleep, and repairs it a bounded number of times when it is not. +// +// Nothing else covers this window. The periodic watchdog compares rule text, which +// survives a suspend unchanged, so it reports the anchor intact while every query through +// the system resolver times out. The interception probe monitor is only armed by +// interface changes and its backoff schedule is frozen through the suspend, so whether it +// probes after a resume depends on where its timers happened to be when the host slept. +// QA measured 18s of no public DNS on a host whose link and default route were back +// within the first second, recovered only because an unrelated VPN reconnect happened to +// drive the post-stabilization verifier; without that reconnect nothing would have probed. +// +// The OS resolver list is refreshed before every probe. probePFIntercept aims at the +// first OS nameserver and reports success when it has no target, so a list still holding +// a pre-sleep VPN resolver turns this into a meaningless pass or a false failure. +func (p *prog) verifyInterceptAfterWake(gap time.Duration) { + if p.dnsInterceptState == nil { + return + } + mainLog.Load().Info().Msgf("DNS intercept: host resumed after a %s gap - verifying interception is still translating", gap.Round(time.Second)) + + repairs := 0 + probed := 0 + for attempt, delay := range pfWakeProbeDelays { + if !p.waitBeforeWakeProbe(delay) { + return + } + if p.dnsInterceptState == nil { + return + } + // Stabilization owns pf while a VPN's ruleset settles, and runs its own + // functional verification when it finishes. Probing or reloading underneath it + // is the mutual overwriting stabilization exists to prevent. + if p.pfStabilizing.Load() { + mainLog.Load().Debug().Msg("DNS intercept: post-wake probe standing down - stabilization owns pf and verifies on completion") + return + } + if p.pfExecBackoffActive() { + mainLog.Load().Debug().Msg("DNS intercept: post-wake probe deferred this attempt - pf exec backoff active") + continue + } + // Eligibility first, ownership second, and release before the next delay: a + // prober that holds the flag while sleeping starves the post-stabilization + // verifier, which reads a held flag as "somebody is probing". + if !p.claimFunctionalProbeOwner(pfWakeProbeOwnerWait) { + mainLog.Load().Debug().Msg("DNS intercept: post-wake probe skipped this attempt - another prober holds ownership") + continue + } + restored, repaired := p.runWakeProbeAttempt(attempt, repairs) + probed++ + p.pfMonitorRunning.Store(false) + if repaired { + repairs++ + } + if restored { + return + } + } + // Report what actually ran: an attempt deferred by pf exec backoff or skipped for + // want of probe ownership never probes, so the schedule length would claim + // verification that did not happen - and with every attempt deferred, nothing was + // measured at all. + if probed == 0 { + mainLog.Load().Warn().Msg("DNS intercept: post-wake verification never probed - every attempt was deferred or skipped - the watchdog will retry") + return + } + mainLog.Load().Error().Msgf("DNS intercept: interception still not translating after resume (%d probe attempts, %d forced reloads) - the watchdog will retry", + probed, repairs) +} + +// waitBeforeWakeProbe waits out one attempt's delay, reporting false when the service +// stopped while waiting. +func (p *prog) waitBeforeWakeProbe(delay time.Duration) bool { + if delay <= 0 { + return true + } + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-p.stopCh: + return false + case <-timer.C: + return true + } +} + +// runWakeProbeAttempt runs one probe of the post-resume schedule, forcing a reload when +// the probe fails and repairs so far leave budget for one. It reports whether +// interception is translating and whether this attempt spent a repair. A reload that +// could not run does not count: nothing was changed, so the next attempt may try again. +// The caller holds functional-probe ownership for the call. +func (p *prog) runWakeProbeAttempt(attempt, repairs int) (restored, repaired bool) { + initializeOsResolver(true) + if probePFInterceptFn(p) { + if attempt > 0 || repairs > 0 { + mainLog.Load().Info().Msgf("DNS intercept: post-wake probe %d/%d passed - interception is translating", attempt+1, len(pfWakeProbeDelays)) + } else { + mainLog.Load().Debug().Msg("DNS intercept: post-wake probe passed - interception is translating") + } + return true, false + } + + if repairs >= pfWakeMaxRepairs { + mainLog.Load().Warn().Msgf("DNS intercept: post-wake probe %d/%d FAILED - repair budget of %d forced reloads is spent, not reloading again", + attempt+1, len(pfWakeProbeDelays), pfWakeMaxRepairs) + return false, false + } + + mainLog.Load().Warn().Msgf("DNS intercept: post-wake probe %d/%d FAILED - rules survived the suspend but pf is not translating, forcing a reload", + attempt+1, len(pfWakeProbeDelays)) + if !forceReloadPFInterceptFn(p) { + mainLog.Load().Error().Msg("DNS intercept: post-wake forced reload did not run") + return false, false + } + if probePFInterceptFn(p) { + mainLog.Load().Info().Msg("DNS intercept: interception restored by the post-wake reload") + return true, true + } + return false, true +} + var runPFAnchorCheckCommand = func(args ...string) ([]byte, error) { return exec.Command("pfctl", args...).CombinedOutput() } @@ -1687,18 +1870,34 @@ func (p *prog) dnsInterceptIgnoredChangeReconcileDue(now time.Time) bool { } } +// scheduleDNSAfterVPNSettleRefresh queues one follow-up VPN DNS refresh. +// +// The timer is tracked rather than fired and forgotten. Teardown has to be able to +// cancel it, and VPN churn can finish stabilization several times in a row, where +// untracked timers stack up into repeats of the same scutil/VPN-DNS work. func (p *prog) scheduleDNSAfterVPNSettleRefresh(reason string, delay time.Duration) { - timer := time.AfterFunc(delay, func() { + p.pfDelayedRecheckMu.Lock() + defer p.pfDelayedRecheckMu.Unlock() + if p.pfSettleFollowupTimer != nil { + p.pfSettleFollowupTimer.Stop() + } + p.pfSettleFollowupTimer = time.AfterFunc(delay, func() { if p.dnsInterceptState == nil { return } p.refreshDNSAfterVPNSettle(reason) }) - // Track the timer like the other delayed rechecks, so intercept teardown - // (and test cleanup) can stop it instead of letting it fire afterwards. +} + +// stopPFSettleFollowup cancels a pending post-settle refresh. A callback that already +// fired still checks dnsInterceptState, so either teardown order is safe. +func (p *prog) stopPFSettleFollowup() { p.pfDelayedRecheckMu.Lock() - p.pfDelayedRecheckTimers = append(p.pfDelayedRecheckTimers, timer) - p.pfDelayedRecheckMu.Unlock() + defer p.pfDelayedRecheckMu.Unlock() + if p.pfSettleFollowupTimer != nil { + p.pfSettleFollowupTimer.Stop() + p.pfSettleFollowupTimer = nil + } } func (p *prog) pfExecBackoffActive() bool { diff --git a/cmd/cli/dns_intercept_darwin_test.go b/cmd/cli/dns_intercept_darwin_test.go index f8515c5c..aad97a9d 100644 --- a/cmd/cli/dns_intercept_darwin_test.go +++ b/cmd/cli/dns_intercept_darwin_test.go @@ -1000,6 +1000,16 @@ func TestFinishPFStabilizationRunsFunctionalVerification(t *testing.T) { t.Errorf("forced reloads = %d, want 1", *reloads) } + stopPFTestTimers(p) +} + +// stopPFTestTimers cancels every timer a stabilization pass can leave pending. +// +// The whole test binary shares one unsynchronised log sink, so a timer that outlives its +// test logs concurrently with whichever test is running next and trips the race detector +// there - reported against the innocent test. The post-settle follow-up is a 4s timer, so +// the collision lands far from its origin. +func stopPFTestTimers(p *prog) { p.pfDelayedRecheckMu.Lock() timers := append([]*time.Timer(nil), p.pfDelayedRecheckTimers...) p.pfDelayedRecheckTimers = nil @@ -1007,4 +1017,416 @@ func TestFinishPFStabilizationRunsFunctionalVerification(t *testing.T) { for _, timer := range timers { timer.Stop() } + p.stopPFSettleFollowup() +} + +// TestScheduleDNSAfterVPNSettleRefreshIsCancellable pins the tracking itself. An +// untracked follow-up cannot be cancelled by teardown or by this helper, which is how a +// 4s timer escaped its test and raced the log sink of a later one. +func TestScheduleDNSAfterVPNSettleRefreshIsCancellable(t *testing.T) { + p := &prog{dnsInterceptState: &pfState{}} + p.scheduleDNSAfterVPNSettleRefresh("test", time.Hour) + + p.pfDelayedRecheckMu.Lock() + tracked := p.pfSettleFollowupTimer != nil + p.pfDelayedRecheckMu.Unlock() + if !tracked { + t.Fatal("the follow-up refresh was scheduled without being tracked; nothing can cancel it") + } + + p.stopPFSettleFollowup() + p.pfDelayedRecheckMu.Lock() + cleared := p.pfSettleFollowupTimer == nil + p.pfDelayedRecheckMu.Unlock() + if !cleared { + t.Error("stopPFSettleFollowup left the timer in place") + } +} + +// TestScheduleDNSAfterVPNSettleRefreshReplacesPending keeps VPN churn from stacking +// repeats of the same scutil/VPN-DNS refresh. +func TestScheduleDNSAfterVPNSettleRefreshReplacesPending(t *testing.T) { + p := &prog{dnsInterceptState: &pfState{}} + p.scheduleDNSAfterVPNSettleRefresh("first", time.Hour) + p.pfDelayedRecheckMu.Lock() + first := p.pfSettleFollowupTimer + p.pfDelayedRecheckMu.Unlock() + + p.scheduleDNSAfterVPNSettleRefresh("second", time.Hour) + p.pfDelayedRecheckMu.Lock() + second := p.pfSettleFollowupTimer + p.pfDelayedRecheckMu.Unlock() + defer p.stopPFSettleFollowup() + + if first == second { + t.Fatal("the second schedule reused the first timer") + } + if first.Stop() { + t.Error("the superseded timer was still armed; stabilization churn would stack refreshes") + } +} + +// ============================================================================= +// post-wake functional verification tests +// ============================================================================= + +// stubWakeProbeSchedule shortens the post-resume retry schedule and neutralises the OS +// resolver refresh, so the bounded-repair contract can be tested without real delays or +// a real scutil call. It returns the number of resolver refreshes performed. +func stubWakeProbeSchedule(t *testing.T, attempts int) (refreshes *int) { + t.Helper() + originalDelays, originalWait := pfWakeProbeDelays, pfWakeProbeOwnerWait + originalResolver := initializeOsResolver + t.Cleanup(func() { + pfWakeProbeDelays, pfWakeProbeOwnerWait = originalDelays, originalWait + initializeOsResolver = originalResolver + }) + + pfWakeProbeDelays = make([]time.Duration, attempts) + for i := range pfWakeProbeDelays { + pfWakeProbeDelays[i] = time.Millisecond + } + pfWakeProbeOwnerWait = 10 * time.Millisecond + + calls := 0 + initializeOsResolver = func(bool) []string { + calls++ + return []string{"10.0.0.1:53"} + } + return &calls +} + +// TestWakeProbeDetectsInterceptionBrokenBySuspend is the hole this closes. Rule text +// survives a suspend unchanged, so the watchdog reports the anchor intact, and the probe +// monitor is only armed by interface changes with a schedule frozen through the sleep. +// Nothing probed after the resume, and QA measured 18s of no public DNS on a host whose +// link and default route were already back. +func TestWakeProbeDetectsInterceptionBrokenBySuspend(t *testing.T) { + stubWakeProbeSchedule(t, 4) + probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.verifyInterceptAfterWake(18 * time.Second) + + if *probes != 2 { + t.Errorf("probe calls = %d, want 2: one to detect and one to confirm the repair", *probes) + } + if *reloads != 1 { + t.Errorf("forced reloads = %d, want 1", *reloads) + } +} + +// TestWakeProbePassStopsImmediately keeps a healthy resume free of pf work: a reload +// flushes pf state and kills in-flight DoH connections for nothing. +func TestWakeProbePassStopsImmediately(t *testing.T) { + stubWakeProbeSchedule(t, 4) + probes, reloads := stubStabilizationProbe(t, []bool{true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.verifyInterceptAfterWake(18 * time.Second) + + if *probes != 1 { + t.Errorf("probe calls = %d, want 1: a passing probe must end the schedule", *probes) + } + if *reloads != 0 { + t.Errorf("forced reloads = %d, want 0 on a healthy resume", *reloads) + } +} + +// TestWakeProbeRetriesAcrossTheSchedule covers the observed shape directly: the first +// forced reload did not restore translation and a later one did, so a single attempt is +// not enough. The second repair must still happen, and the confirming pass must end it. +func TestWakeProbeRetriesAcrossTheSchedule(t *testing.T) { + stubWakeProbeSchedule(t, 4) + // attempt 1: probe fails, reload, confirm fails. + // attempt 2: probe fails, reload, confirm passes. + probes, reloads := stubStabilizationProbe(t, []bool{false, false, false, true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.verifyInterceptAfterWake(18 * time.Second) + + if *reloads != 2 { + t.Errorf("forced reloads = %d, want 2: one failed repair must not end the schedule", *reloads) + } + if *probes != 4 { + t.Errorf("probe calls = %d, want 4", *probes) + } +} + +// TestWakeProbeRepairsAreBounded pins the bound. A probe that never passes must exhaust +// the repair budget and then keep probing without reloading, so this can never become the +// unbounded reload loop the issue rules out. +func TestWakeProbeRepairsAreBounded(t *testing.T) { + stubWakeProbeSchedule(t, 6) + probes, reloads := stubStabilizationProbe(t, nil, true) // every probe fails + + p := &prog{dnsInterceptState: &pfState{}} + p.verifyInterceptAfterWake(18 * time.Second) + + if *reloads != pfWakeMaxRepairs { + t.Errorf("forced reloads = %d, want the budget of %d", *reloads, pfWakeMaxRepairs) + } + // Two repaired attempts probe twice each; the remaining four probe once each. + if want := 2*pfWakeMaxRepairs + (6 - pfWakeMaxRepairs); *probes != want { + t.Errorf("probe calls = %d, want %d", *probes, want) + } + if p.pfMonitorRunning.Load() { + t.Error("functional-probe ownership was left held after the schedule finished") + } +} + +// TestWakeProbeRefreshesResolverBeforeEveryProbe is why a stale list matters: the probe +// aims at the first OS nameserver and reports success when it has none, so a list still +// holding a pre-sleep VPN resolver turns the probe into a meaningless pass or a false +// failure. The captured run showed the resolver set changing after the resume. +func TestWakeProbeRefreshesResolverBeforeEveryProbe(t *testing.T) { + refreshes := stubWakeProbeSchedule(t, 3) + stubStabilizationProbe(t, nil, false) // probes fail, reload refuses to run + + p := &prog{dnsInterceptState: &pfState{}} + p.verifyInterceptAfterWake(18 * time.Second) + + if *refreshes != 3 { + t.Errorf("OS resolver refreshes = %d, want one per attempt (3)", *refreshes) + } +} + +// TestWakeProbeStandsDownForStabilization keeps the two repair paths from fighting over +// pf. Stabilization owns the ruleset while a VPN settles and runs its own functional +// verification when it finishes; reloading underneath it is the mutual overwriting +// stabilization exists to prevent. +func TestWakeProbeStandsDownForStabilization(t *testing.T) { + stubWakeProbeSchedule(t, 4) + probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.pfStabilizing.Store(true) + p.verifyInterceptAfterWake(18 * time.Second) + + if *probes != 0 || *reloads != 0 { + t.Errorf("probe calls = %d, forced reloads = %d, want 0/0 while stabilization owns pf", *probes, *reloads) + } + if !p.claimFunctionalProbeOwner(0) { + t.Error("standing down took functional-probe ownership; the post-stabilization verifier would skip") + } + p.pfMonitorRunning.Store(false) +} + +// TestWakeProbeYieldsToAnotherProberThenRetries is the ownership handoff. A probe monitor +// that is genuinely working owns this attempt, but the schedule must not be spent waiting +// on it: the next attempt still has to probe once ownership is free. +func TestWakeProbeYieldsToAnotherProberThenRetries(t *testing.T) { + stubWakeProbeSchedule(t, 3) + probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.pfMonitorRunning.Store(true) + go func() { + time.Sleep(30 * time.Millisecond) + p.pfMonitorRunning.Store(false) + }() + + p.verifyInterceptAfterWake(18 * time.Second) + + if *probes == 0 { + t.Fatal("the schedule gave up entirely because another prober held ownership at first") + } + if *reloads != 1 { + t.Errorf("forced reloads = %d, want 1", *reloads) + } +} + +// TestWakeProbeSkippedDuringExecBackoff protects the resource-exhaustion guard: each +// probe forks a helper, which is exactly what backoff exists to stop. +func TestWakeProbeSkippedDuringExecBackoff(t *testing.T) { + stubWakeProbeSchedule(t, 3) + probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true) + + p := &prog{dnsInterceptState: &pfState{}} + p.pfExecBackoffUntil.Store(time.Now().Add(time.Minute).UnixMilli()) + p.verifyInterceptAfterWake(18 * time.Second) + + if *probes != 0 || *reloads != 0 { + t.Errorf("probe calls = %d, forced reloads = %d, want 0/0 during pf exec backoff", *probes, *reloads) + } +} + +// TestWakeProbeSkippedWhenInterceptDisabled keeps the watcher harmless when intercept +// mode is off: the detector runs for the process, the repair is intercept-only. +func TestWakeProbeSkippedWhenInterceptDisabled(t *testing.T) { + stubWakeProbeSchedule(t, 3) + probes, reloads := stubStabilizationProbe(t, []bool{false}, true) + + p := &prog{} + p.verifyInterceptAfterWake(18 * time.Second) + + if *probes != 0 || *reloads != 0 { + t.Errorf("probe calls = %d, forced reloads = %d, want 0/0 with intercept disabled", *probes, *reloads) + } +} + +// TestWakeProbeStopsWhenServiceStops keeps a resume during shutdown from holding the +// schedule open past the stop request. +func TestWakeProbeStopsWhenServiceStops(t *testing.T) { + originalDelays := pfWakeProbeDelays + pfWakeProbeDelays = []time.Duration{0, time.Hour} + t.Cleanup(func() { pfWakeProbeDelays = originalDelays }) + originalResolver := initializeOsResolver + initializeOsResolver = func(bool) []string { return []string{"10.0.0.1:53"} } + t.Cleanup(func() { initializeOsResolver = originalResolver }) + + probes, _ := stubStabilizationProbe(t, nil, false) // first probe fails, reload refuses + + stopCh := make(chan struct{}) + close(stopCh) + p := &prog{dnsInterceptState: &pfState{}, stopCh: stopCh} + + done := make(chan struct{}) + go func() { + p.verifyInterceptAfterWake(18 * time.Second) + close(done) + }() + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("verifyInterceptAfterWake ignored the stop request and waited out its schedule") + } + if *probes != 1 { + t.Errorf("probe calls = %d, want 1 before the stop was observed", *probes) + } +} + +// ============================================================================= +// post-wake watcher registration tests +// ============================================================================= + +// TestStartInterceptBackgroundWorkRegistersTheWakeVerifier covers the caller wiring. The +// detector and the wake-repair path can both be correct while nothing connects them, and +// then a host sleep is never noticed in production. +func TestStartInterceptBackgroundWorkRegistersTheWakeVerifier(t *testing.T) { + type registration struct { + stopCh <-chan struct{} + onResume func(time.Duration) + } + registered := make(chan registration, 1) + watchdogStarted := make(chan struct{}, 1) + originalWatcher, originalWatchdog := runSuspendWatcherFn, pfWatchdogFn + runSuspendWatcherFn = func(stopCh <-chan struct{}, onResume func(time.Duration)) { + registered <- registration{stopCh: stopCh, onResume: onResume} + } + // Both loops are stubbed rather than run: each would reach pfctl, and stopping the + // real watchdog means closing stopCh, which the wake verifier reads as "service + // stopping" and would abandon its probe schedule before the first probe. + pfWatchdogFn = func(*prog) { watchdogStarted <- struct{}{} } + t.Cleanup(func() { runSuspendWatcherFn, pfWatchdogFn = originalWatcher, originalWatchdog }) + + stubWakeProbeSchedule(t, 4) + probes, _ := stubStabilizationProbe(t, []bool{false, true}, true) + + // Open for the whole test: an active intercept is not stopping. + stopCh := make(chan struct{}) + p := &prog{dnsInterceptState: &pfState{}, stopCh: stopCh} + p.startInterceptBackgroundWork() + + select { + case <-watchdogStarted: + case <-time.After(2 * time.Second): + t.Error("no pf watchdog was started; a replaced pf ruleset would never be restored") + } + + var reg registration + select { + case reg = <-registered: + case <-time.After(2 * time.Second): + t.Fatal("no suspend watcher was started; a host sleep would go unnoticed") + } + if reg.stopCh != stopCh { + t.Error("the watcher was not wired to the service stop channel; it would outlive the intercept") + } + + // Prove the callback is the post-wake verifier and not some other function: a + // reported resume has to probe interception. How many probes and repairs that + // schedule is worth belongs to TestWakeProbeDetectsInterceptionBrokenBySuspend; + // pinning it again here would only couple this test to that tuning. + reg.onResume(18 * time.Second) + if *probes == 0 { + t.Error("the resume callback probed nothing; it is not wired to verifyInterceptAfterWake") + } +} + +// TestStartDNSInterceptStartsBackgroundWork closes the caller-wiring gap. Everything the +// privileged install does sits behind installPFInterceptFn, so this drives the real +// startDNSIntercept and checks what a successful start hands off: without the handoff, an +// active intercept runs with no pf watchdog and no post-wake verification, and every +// detector, recovery and registration test still passes. +func TestStartDNSInterceptStartsBackgroundWork(t *testing.T) { + installed := 0 + originalInstall := installPFInterceptFn + installPFInterceptFn = func(*prog) error { installed++; return nil } + originalDiscover := discoverTunnelInterfacesForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return nil } + + watchdogStarted := make(chan struct{}, 1) + watcherStarted := make(chan struct{}, 1) + originalWatchdog, originalWatcher := pfWatchdogFn, runSuspendWatcherFn + pfWatchdogFn = func(*prog) { watchdogStarted <- struct{}{} } + runSuspendWatcherFn = func(<-chan struct{}, func(time.Duration)) { watcherStarted <- struct{}{} } + t.Cleanup(func() { + installPFInterceptFn = originalInstall + discoverTunnelInterfacesForReconcile = originalDiscover + pfWatchdogFn, runSuspendWatcherFn = originalWatchdog, originalWatcher + }) + + p := &prog{ + cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}}}, + stopCh: make(chan struct{}), + } + if err := p.startDNSIntercept(); err != nil { + t.Fatalf("startDNSIntercept() = %v, want nil", err) + } + if installed != 1 { + t.Errorf("privileged install ran %d times, want 1", installed) + } + if p.dnsInterceptState == nil { + t.Fatal("a successful start did not publish intercept state") + } + + select { + case <-watchdogStarted: + case <-time.After(2 * time.Second): + t.Error("startDNSIntercept did not start the pf watchdog; a replaced pf ruleset would never be restored") + } + select { + case <-watcherStarted: + case <-time.After(2 * time.Second): + t.Error("startDNSIntercept did not start the suspend watcher; a host sleep would go unnoticed") + } +} + +// TestStartDNSInterceptSkipsBackgroundWorkWhenInstallFails keeps the handoff conditional: +// watchers over an intercept that was never installed would repair state nothing owns. +func TestStartDNSInterceptSkipsBackgroundWorkWhenInstallFails(t *testing.T) { + originalInstall := installPFInterceptFn + installPFInterceptFn = func(*prog) error { return errors.New("pf unavailable") } + originalWatchdog, originalWatcher := pfWatchdogFn, runSuspendWatcherFn + pfWatchdogFn = func(*prog) { t.Error("the pf watchdog started after a failed install") } + runSuspendWatcherFn = func(<-chan struct{}, func(time.Duration)) { + t.Error("the suspend watcher started after a failed install") + } + t.Cleanup(func() { + installPFInterceptFn = originalInstall + pfWatchdogFn, runSuspendWatcherFn = originalWatchdog, originalWatcher + }) + + p := &prog{stopCh: make(chan struct{})} + if err := p.startDNSIntercept(); err == nil { + t.Fatal("startDNSIntercept() = nil after the privileged install failed") + } + if p.dnsInterceptState != nil { + t.Error("a failed start published intercept state; the caller would skip its DNS fallback") + } + // The goroutines above are started before this returns if at all, but give a failing + // wiring a moment to report rather than racing the test's end. + time.Sleep(50 * time.Millisecond) } diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 1c0fb29e..375bdc20 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -250,6 +250,12 @@ type prog struct { pfDelayedRecheckMu sync.Mutex //lint:ignore U1000 used on darwin pfDelayedRecheckTimers []*time.Timer //lint:ignore U1000 used on darwin + // pfSettleFollowupTimer is the pending post-settle VPN DNS refresh, if any. + // Tracked so teardown can cancel it and a later stabilization replaces it + // instead of stacking another copy of the same refresh. Protected by + // pfDelayedRecheckMu. + pfSettleFollowupTimer *time.Timer //lint:ignore U1000 used on darwin + // pfIgnoredChangeLastReconcile bounds immediate pf/VPN-DNS work for noisy // ignored macOS network deltas. Tunnel changes bypass this limit, and the // existing delayed checks provide a trailing reconciliation after churn. diff --git a/cmd/cli/suspend_detector.go b/cmd/cli/suspend_detector.go new file mode 100644 index 00000000..b3e3f815 --- /dev/null +++ b/cmd/cli/suspend_detector.go @@ -0,0 +1,109 @@ +package cli + +import "time" + +// Suspend/resume detection without an OS notification API. +// +// A host sleep suspends the whole process: timers stop advancing and resume where they +// left off, so a tick scheduled two seconds out can arrive a minute later. That +// overshoot is the only in-process evidence that the host slept, and nothing else tells +// the DNS-intercept repair paths that the state they proved before sleep - rule text, +// resolver list, upstream connections - may no longer hold. + +const ( + // suspendProbeInterval is how often the detector measures elapsed wall-clock time. + // Short enough that a resume is noticed within a couple of seconds, which is what + // the post-wake continuity contract measures from. + suspendProbeInterval = 2 * time.Second + + // suspendGapThreshold is the elapsed wall clock over one interval that counts as a + // suspend. A pfctl storm or a loaded host can stretch a tick, so this sits at + // several times the interval; a real sleep overshoots by far more. Erring low is + // cheap on purpose: a false positive costs one probe, and a stall this long is + // worth a probe either way. + suspendGapThreshold = 8 * time.Second +) + +// wallNanoNow reads the wall clock in nanoseconds. +// +// The detector works in wall-clock nanoseconds rather than time.Time on purpose. +// time.Time.Sub prefers the monotonic reading whenever both operands carry one, and +// Darwin's monotonic clock stops while the host is asleep - so a monotonic difference +// across a suspend is just the tick interval and hides the very gap this looks for. +// Taking int64 wall nanos makes that mistake unrepresentable rather than a comment. +func wallNanoNow() int64 { return time.Now().UnixNano() } + +// suspendDetector turns observations of a fixed-interval tick into suspend/resume +// events. It measures elapsed time rather than counting ticks, because a suspended +// process gets one late tick, not the ticks it missed. +type suspendDetector struct { + threshold time.Duration + wallNano func() int64 + lastNano int64 +} + +// newSuspendDetector returns a detector that reports gaps of at least threshold. +// wallNano is a seam for tests; nil means the real wall clock. +func newSuspendDetector(threshold time.Duration, wallNano func() int64) *suspendDetector { + if wallNano == nil { + wallNano = wallNanoNow + } + return &suspendDetector{threshold: threshold, wallNano: wallNano, lastNano: wallNano()} +} + +// observe records a tick and reports the elapsed time when it reaches the threshold, +// which means the process was not running for most of that span. +func (d *suspendDetector) observe() (time.Duration, bool) { + current := d.wallNano() + gap := time.Duration(current - d.lastNano) + d.lastNano = current + if gap < d.threshold { + return 0, false + } + return gap, true +} + +// reset re-baselines the detector to the current wall clock. Time that has already +// passed is deliberately discarded rather than measured: use it after doing work +// outside the sampling loop, which is not a suspend even though it looks like one. +func (d *suspendDetector) reset() { d.lastNano = d.wallNano() } + +// watchForResume calls onResume once per detected suspend/resume gap, until stopCh +// closes or tick is closed. tick is a parameter so tests can drive it directly. +// +// onResume runs on this goroutine, so recovery is serialized against detection - one +// resume cannot start a second recovery on top of the first. The cost is that no +// sampling happens while it runs, which the re-baseline below accounts for. +func watchForResume(stopCh <-chan struct{}, tick <-chan time.Time, d *suspendDetector, onResume func(gap time.Duration)) { + for { + select { + case <-stopCh: + return + case _, ok := <-tick: + if !ok { + return + } + if gap, resumed := d.observe(); resumed { + onResume(gap) + // Re-baseline: the recovery just ran, and its own runtime exceeds the + // threshold on its own (the post-wake probe schedule waits for seconds + // before a single probe timeout or reload). The ticker keeps firing and + // buffers a tick while it works, so without this the next observe() + // measures the callback and reports it as a fresh suspend - which starts + // another recovery, which is again long enough to look like a suspend. + // Under a persistent pf failure that is an unbounded reload loop, and + // each pass flushes pf state and kills in-flight DoH connections. + d.reset() + } + } + } +} + +// runSuspendWatcher watches for suspend/resume against a real ticker. +// +//lint:ignore U1000 used on darwin (post-wake interception verification) +func runSuspendWatcher(stopCh <-chan struct{}, onResume func(gap time.Duration)) { + ticker := time.NewTicker(suspendProbeInterval) + defer ticker.Stop() + watchForResume(stopCh, ticker.C, newSuspendDetector(suspendGapThreshold, nil), onResume) +} diff --git a/cmd/cli/suspend_detector_test.go b/cmd/cli/suspend_detector_test.go new file mode 100644 index 00000000..4f31a31c --- /dev/null +++ b/cmd/cli/suspend_detector_test.go @@ -0,0 +1,208 @@ +package cli + +import ( + "testing" + "time" +) + +// fakeWallClock returns the queued offsets from a fixed base, then repeats the last one. +type fakeWallClock struct { + nanos []int64 + calls int +} + +func (c *fakeWallClock) now() int64 { + if c.calls < len(c.nanos) { + n := c.nanos[c.calls] + c.calls++ + return n + } + return c.nanos[len(c.nanos)-1] +} + +// wallClockAt builds a clock whose readings are the given offsets from an arbitrary +// epoch. The first reading is consumed by newSuspendDetector; the rest are ticks. +func wallClockAt(offsets ...time.Duration) *fakeWallClock { + const base = int64(1787081656_000000000) // 2026-08-18T14:34:16-05:00 + nanos := make([]int64, 0, len(offsets)) + for _, off := range offsets { + nanos = append(nanos, base+int64(off)) + } + return &fakeWallClock{nanos: nanos} +} + +func TestSuspendDetectorIgnoresOrdinaryTicks(t *testing.T) { + // Construction, then three ticks two seconds apart, one stretched to 3s by a + // loaded host. None of these is a suspend. + clock := wallClockAt(0, 2*time.Second, 4*time.Second, 7*time.Second) + d := newSuspendDetector(suspendGapThreshold, clock.now) + + for i := 0; i < 3; i++ { + if gap, resumed := d.observe(); resumed { + t.Errorf("tick %d reported a suspend of %s; ordinary jitter must not count", i, gap) + } + } +} + +// TestSuspendDetectorReportsTheGapOnce replays the captured incident: the last pre-sleep +// sample at 14:34:47 and the first post-resume one at 14:35:05. +func TestSuspendDetectorReportsTheGapOnce(t *testing.T) { + clock := wallClockAt( + 31*time.Second, // construction, 14:34:47 + 33*time.Second, // ordinary tick + 49*time.Second, // 14:35:05 - a 16s gap over a 2s interval + 51*time.Second, // ordinary again + ) + d := newSuspendDetector(suspendGapThreshold, clock.now) + + if _, resumed := d.observe(); resumed { + t.Fatal("the tick before the suspend reported a resume") + } + gap, resumed := d.observe() + if !resumed { + t.Fatal("a 16s gap over a 2s interval was not reported as a suspend") + } + if gap != 16*time.Second { + t.Errorf("gap = %s, want 16s: the reported gap must be the measured wall clock", gap) + } + if _, resumed := d.observe(); resumed { + t.Error("the resume was reported twice; the baseline must advance to the late tick") + } +} + +// TestSuspendDetectorThresholdCoversShortNaps guards the lower bound. A nap only a few +// seconds longer than the interval still breaks pf translation, so the threshold must not +// drift up to a value that only catches long sleeps. +func TestSuspendDetectorThresholdCoversShortNaps(t *testing.T) { + if suspendGapThreshold > 10*time.Second { + t.Errorf("suspendGapThreshold = %s; a short nap would go unnoticed", suspendGapThreshold) + } + if suspendGapThreshold <= 2*suspendProbeInterval { + t.Errorf("suspendGapThreshold = %s is too close to the %s interval; ordinary jitter would fire it", + suspendGapThreshold, suspendProbeInterval) + } + + clock := wallClockAt(0, suspendGapThreshold) + d := newSuspendDetector(suspendGapThreshold, clock.now) + if _, resumed := d.observe(); !resumed { + t.Error("a gap exactly at the threshold was not reported") + } +} + +func TestSuspendDetectorIgnoresBackwardClockSteps(t *testing.T) { + // An NTP step backwards must not read as a resume, and must not leave the baseline + // in the future either: the next ordinary tick still has to look ordinary. + clock := wallClockAt(0, -30*time.Second, -28*time.Second) + d := newSuspendDetector(suspendGapThreshold, clock.now) + + if gap, resumed := d.observe(); resumed { + t.Errorf("a backward clock step reported a suspend of %s", gap) + } + if gap, resumed := d.observe(); resumed { + t.Errorf("the tick after a backward step reported a suspend of %s", gap) + } +} + +func TestWatchForResumeCallsBackPerGap(t *testing.T) { + clock := wallClockAt( + 0, + 2*time.Second, // ordinary + 40*time.Second, // resume: 38s + 42*time.Second, // ordinary + 99*time.Second, // second resume: 57s + ) + d := newSuspendDetector(suspendGapThreshold, clock.now) + + tick := make(chan time.Time, 4) + for i := 0; i < 4; i++ { + tick <- time.Time{} + } + close(tick) + + var gaps []time.Duration + watchForResume(nil, tick, d, func(gap time.Duration) { gaps = append(gaps, gap) }) + + if len(gaps) != 2 { + t.Fatalf("onResume called %d times (%v), want 2", len(gaps), gaps) + } + if gaps[0] != 38*time.Second || gaps[1] != 57*time.Second { + t.Errorf("gaps = %v, want [38s 57s]", gaps) + } +} + +// advancingClock is a wall clock the test moves by hand, so a callback's runtime can be +// simulated without sleeping for it. +type advancingClock struct{ nano int64 } + +func (c *advancingClock) now() int64 { return c.nano } +func (c *advancingClock) advance(d time.Duration) { c.nano += int64(d) } + +// TestWatchForResumeDoesNotRetriggerOnItsOwnCallback pins the loop this closes. onResume +// runs on the watcher goroutine, and the real callback - verifyInterceptAfterWake - takes +// longer than the gap threshold by design: its configured waits alone total more than the +// threshold, before any probe timeout or forced reload. The ticker buffers a tick while it +// works, so a detector that keeps its pre-callback baseline measures the callback itself as +// the next gap and starts another recovery, with another repair budget, indefinitely under +// a persistent pf failure. +func TestWatchForResumeDoesNotRetriggerOnItsOwnCallback(t *testing.T) { + clock := &advancingClock{} + d := newSuspendDetector(suspendGapThreshold, clock.now) + + // Capacity 1, like time.Ticker's channel: exactly one tick can queue up behind a + // callback in production. + tick := make(chan time.Time, 1) + calls := 0 + done := make(chan struct{}) + go func() { + defer close(done) + watchForResume(nil, tick, d, func(time.Duration) { + calls++ + if calls > 1 { + // A retrigger - the failure this test reports. There is nothing left to + // simulate, and tick is already closed. + return + } + // The recovery takes longer than the threshold, and a tick lands while it + // runs. Closing here ends the watcher after it drains that queued tick, + // which is the observation under test. + clock.advance(2 * suspendGapThreshold) + tick <- time.Time{} + close(tick) + }) + }() + + clock.advance(2 * suspendGapThreshold) // the genuine suspend + tick <- time.Time{} + + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("watchForResume did not finish; the queued tick was never consumed") + } + + if calls != 1 { + t.Errorf("onResume called %d times, want 1: the callback's own runtime was measured as a second suspend", calls) + } +} + +func TestWatchForResumeStopsOnStopCh(t *testing.T) { + stopCh := make(chan struct{}) + tick := make(chan time.Time) // never fires + clock := wallClockAt(0, time.Hour) + d := newSuspendDetector(suspendGapThreshold, clock.now) + + done := make(chan struct{}) + go func() { + watchForResume(stopCh, tick, d, func(time.Duration) { + t.Error("onResume ran without a tick") + }) + close(done) + }() + + close(stopCh) + select { + case <-done: + case <-time.After(2 * time.Second): + t.Fatal("watchForResume did not return once stopCh was closed") + } +} From 9d724869e9eadd8d5af4f9c37ee81121df74b8a4 Mon Sep 17 00:00:00 2001 From: Dev Scribe Date: Wed, 2 Sep 2026 09:51:27 +0000 Subject: [PATCH 02/29] dns proxy: DNS64 synthesis for IPv6-only networks without CLAT --- cmd/cli/dns64.go | 445 +++++++++++++++++++++++++++++++++++++ cmd/cli/dns64_test.go | 366 ++++++++++++++++++++++++++++++ cmd/cli/dns_proxy.go | 116 ++++++++-- cmd/cli/dns_proxy_test.go | 46 ++++ cmd/cli/prog.go | 4 + internal/dnscache/cache.go | 9 + 6 files changed, 965 insertions(+), 21 deletions(-) create mode 100644 cmd/cli/dns64.go create mode 100644 cmd/cli/dns64_test.go diff --git a/cmd/cli/dns64.go b/cmd/cli/dns64.go new file mode 100644 index 00000000..352a1b1d --- /dev/null +++ b/cmd/cli/dns64.go @@ -0,0 +1,445 @@ +package cli + +import ( + "context" + "net" + "net/netip" + "sync" + "time" + + "github.com/miekg/dns" + "tailscale.com/net/netmon" + + ctrld "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/dnscache" +) + +// DNS64 synthesis for IPv6-only networks WITHOUT client-side 464XLAT (no +// CLAT). On such networks the carrier's DNS64 resolver is load-bearing: it +// synthesizes AAAA records mapping IPv4-only destinations into the NAT64 +// prefix, and there is no CLAT interface to carry real IPv4 traffic. When +// ctrld answers with genuine A records there, IPv4-only destinations become +// unreachable — DNS resolves but connectivity fails (issue companion to +// #533; tethering is unaffected because Apple/Android always provide CLAT). +// +// ctrld therefore performs its own RFC 6147-style synthesis after filtering: +// when the network is IPv6-only with no CLAT and a NAT64 prefix is known, +// an AAAA query whose (policy-approved) answer contains no AAAA records is +// re-resolved as an A query through the same upstream, and the A records are +// mapped into the NAT64 prefix. Blocked answers are never synthesized — +// synthesis runs on the answer the policy engine already approved. +// +// NAT64 prefix discovery uses RFC 7050: resolve AAAA for ipv4only.arpa +// through the network's own resolvers and derive the prefix from the +// embedded well-known IPv4 addresses (192.0.0.170/171). PREF64 router +// advertisements (RFC 8781) are not parsed; RFC 7050 covers the same +// networks without OS-specific RA plumbing. + +const ( + // dns64RecheckInterval bounds how often network state (CLAT presence, + // IPv4 availability, NAT64 prefix) is re-evaluated. + dns64RecheckInterval = 5 * time.Minute + // dns64WellKnownName is the RFC 7050 discovery name. + dns64WellKnownName = "ipv4only.arpa." + // dns64DiscoverTimeout bounds one background discovery attempt. + dns64DiscoverTimeout = 5 * time.Second +) + +// rfc7050WellKnown are the IPv4 addresses embedded in ipv4only.arpa AAAA +// answers on DNS64 networks (RFC 7050). +var rfc7050WellKnown = []netip.Addr{ + netip.AddrFrom4([4]byte{192, 0, 0, 170}), + netip.AddrFrom4([4]byte{192, 0, 0, 171}), +} + +var dns64WellKnownPrefix = netip.MustParsePrefix("64:ff9b::/96") + +// clatPrefix is the RFC 7335 IPv4 service-continuity prefix used by +// client-side translators (CLAT). +var clatPrefix = netip.PrefixFrom(netip.AddrFrom4([4]byte{192, 0, 0, 0}), 29) + +type dns64State struct { + mu sync.Mutex + checkedAt time.Time + active bool // network is v6-only, no CLAT, prefix known + prefix netip.Prefix // discovered NAT64 prefix (/96) + discovering bool + generation uint64 +} + +// dns64NetworkClassFn is a seam for tests. +var dns64NetworkClassFn = currentDNS64NetworkClass + +func addrFromNetAddr(a net.Addr) (netip.Addr, bool) { + var ip net.IP + switch v := a.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + default: + return netip.Addr{}, false + } + nip, ok := netip.AddrFromSlice(ip) + if !ok { + return netip.Addr{}, false + } + return nip.Unmap(), true +} + +// dns64NetworkClass classifies the host addressing state from interface +// addresses. Only IPv4 on the default-route interface counts as usable, so +// RFC1918 addresses owned by Docker, Parallels, VMware, and similar virtual +// interfaces do not disable DNS64. CLAT is detected across all interfaces. +func dns64NetworkClass(defaultRouteAddrs, allAddrs []net.Addr) (hasUsableIPv4, hasCLAT bool) { + for _, a := range allAddrs { + nip, ok := addrFromNetAddr(a) + if !ok || !nip.Is4() { + continue + } + if clatPrefix.Contains(nip) { + hasCLAT = true + } + } + for _, a := range defaultRouteAddrs { + nip, ok := addrFromNetAddr(a) + if !ok || !nip.Is4() || clatPrefix.Contains(nip) { + continue + } + if nip.IsLoopback() || nip.IsLinkLocalUnicast() || nip.IsUnspecified() { + continue + } + hasUsableIPv4 = true + } + return hasUsableIPv4, hasCLAT +} + +func currentDNS64NetworkClass() (hasUsableIPv4, hasCLAT bool, err error) { + defaultRouteInterface, err := netmon.DefaultRouteInterface() + if err != nil { + return false, false, err + } + iface, err := net.InterfaceByName(defaultRouteInterface) + if err != nil { + return false, false, err + } + defaultRouteAddrs, err := iface.Addrs() + if err != nil { + return false, false, err + } + allAddrs, err := net.InterfaceAddrs() + if err != nil { + return false, false, err + } + hasUsableIPv4, hasCLAT = dns64NetworkClass(defaultRouteAddrs, allAddrs) + return hasUsableIPv4, hasCLAT, nil +} + +// nat64PrefixFromAnswer derives the NAT64 prefix from an ipv4only.arpa AAAA +// answer per RFC 7050: find an AAAA embedding a well-known IPv4 address in +// its last 4 bytes and take the leading /96. +func nat64PrefixFromAnswer(answer *dns.Msg) (netip.Prefix, bool) { + if answer == nil { + return netip.Prefix{}, false + } + for _, rr := range answer.Answer { + aaaa, ok := rr.(*dns.AAAA) + if !ok { + continue + } + v6, ok := netip.AddrFromSlice(aaaa.AAAA.To16()) + if !ok || v6.Is4() { + continue + } + b := v6.As16() + embedded := netip.AddrFrom4([4]byte{b[12], b[13], b[14], b[15]}) + for _, wk := range rfc7050WellKnown { + if embedded == wk { + var p [16]byte + copy(p[:12], b[:12]) + return netip.PrefixFrom(netip.AddrFrom16(p), 96), true + } + } + } + return netip.Prefix{}, false +} + +// synthesizeAAAAFromA returns a copy of aAnswer converted into an AAAA +// answer for the original AAAA request: every A record is mapped into the +// NAT64 prefix; other records (CNAMEs etc.) are preserved. +func synthesizeAAAAFromA(req *dns.Msg, aAnswer *dns.Msg, prefix netip.Prefix) *dns.Msg { + if aAnswer == nil { + return nil + } + out := aAnswer.Copy() + out.SetReply(req) + out.Rcode = aAnswer.Rcode + out.Compress = true + answers := make([]dns.RR, 0, len(aAnswer.Answer)) + pb := prefix.Addr().As16() + for _, rr := range aAnswer.Answer { + a, ok := rr.(*dns.A) + if !ok { + // Preserve CNAME chain records unchanged. + answers = append(answers, dns.Copy(rr)) + continue + } + v4 := a.A.To4() + if v4 == nil { + continue + } + v4Addr, ok := netip.AddrFromSlice(v4) + if !ok || !v4Addr.IsGlobalUnicast() || (prefix == dns64WellKnownPrefix && v4Addr.IsPrivate()) { + continue + } + var b [16]byte + copy(b[:12], pb[:12]) + copy(b[12:], v4) + aaaa := &dns.AAAA{ + Hdr: dns.RR_Header{ + Name: a.Hdr.Name, + Rrtype: dns.TypeAAAA, + Class: a.Hdr.Class, + Ttl: a.Hdr.Ttl, + }, + AAAA: net.IP(b[:]), + } + answers = append(answers, aaaa) + } + out.Answer = answers + return out +} + +// answerHasAAAA reports whether the answer section contains any AAAA record. +func answerHasAAAA(answer *dns.Msg) bool { + if answer == nil { + return false + } + for _, rr := range answer.Answer { + if _, ok := rr.(*dns.AAAA); ok { + return true + } + } + return false +} + +// dns64Eligible reports whether an answer qualifies for DNS64 synthesis: +// an AAAA query answered NOERROR with no AAAA records. NXDOMAIN and error +// rcodes are never synthesized (RFC 6147 §5.1.2: the name genuinely does +// not exist or the query failed). +func dns64Eligible(req, answer *dns.Msg) bool { + if req == nil || answer == nil || len(req.Question) == 0 || req.CheckingDisabled { + return false + } + if req.Question[0].Qtype != dns.TypeAAAA { + return false + } + if answer.Rcode != dns.RcodeSuccess { + return false + } + return !answerHasAAAA(answer) +} + +// dns64Active reports whether synthesis should currently run, re-evaluating +// network class and (if needed) kicking off background prefix discovery at +// most every dns64RecheckInterval. +func (p *prog) dns64Active() bool { + s := &p.dns64 + s.mu.Lock() + if time.Since(s.checkedAt) < dns64RecheckInterval { + active := s.active + s.mu.Unlock() + return active + } + s.checkedAt = time.Now() + generation := s.generation + s.mu.Unlock() + + hasV4, hasCLAT, err := dns64NetworkClassFn() + s.mu.Lock() + defer s.mu.Unlock() + if generation != s.generation { + return s.active + } + if err != nil { + s.active = false + return false + } + if hasV4 || hasCLAT { + // Dual-stack or 464XLAT: the OS/CLAT handles IPv4 reachability; + // synthesis would be unnecessary (and on CLAT networks, harmful — + // real A records are preferable so traffic uses the CLAT). + s.generation++ + s.active = false + s.prefix = netip.Prefix{} + s.discovering = false + return false + } + s.active = s.prefix.IsValid() + if !s.discovering { + s.discovering = true + generation := s.generation + go p.discoverNAT64Prefix(generation) + } + return s.active +} + +func (p *prog) activeDNS64Prefix() (netip.Prefix, bool) { + if !p.dns64Active() { + return netip.Prefix{}, false + } + p.dns64.mu.Lock() + defer p.dns64.mu.Unlock() + return p.dns64.prefix, p.dns64.prefix.IsValid() +} + +func dns64CacheVariant(prefix netip.Prefix) string { + return "dns64:" + prefix.String() +} + +func dns64CacheKey(msg *dns.Msg, upstream string, prefix netip.Prefix) dnscache.Key { + return dnscache.NewVariantKey(msg, upstream, dns64CacheVariant(prefix)) +} + +func (p *prog) resetDNS64State() { + p.dns64.mu.Lock() + p.dns64.generation++ + p.dns64.checkedAt = time.Time{} + p.dns64.active = false + p.dns64.prefix = netip.Prefix{} + p.dns64.discovering = false + p.dns64.mu.Unlock() +} + +func dns64RouteStateChanged(delta *netmon.ChangeDelta) bool { + if delta == nil || delta.Old == nil || delta.New == nil { + return true + } + if delta.Old.DefaultRouteInterface != delta.New.DefaultRouteInterface || + delta.Old.HaveV4 != delta.New.HaveV4 || delta.Old.HaveV6 != delta.New.HaveV6 { + return true + } + if dns64StateHasCLAT(delta.Old) != dns64StateHasCLAT(delta.New) { + return true + } + iface := delta.New.DefaultRouteInterface + oldPrefixes := delta.Old.InterfaceIPs[iface] + newPrefixes := delta.New.InterfaceIPs[iface] + if len(oldPrefixes) != len(newPrefixes) { + return true + } + newPrefixSet := make(map[netip.Prefix]struct{}, len(newPrefixes)) + for _, prefix := range newPrefixes { + newPrefixSet[prefix] = struct{}{} + } + for _, prefix := range oldPrefixes { + if _, ok := newPrefixSet[prefix]; !ok { + return true + } + } + return false +} + +func dns64StateHasCLAT(state *netmon.State) bool { + for _, prefixes := range state.InterfaceIPs { + for _, prefix := range prefixes { + if clatPrefix.Contains(prefix.Addr().Unmap()) { + return true + } + } + } + return false +} + +func (p *prog) handleDNS64NetworkChange(delta *netmon.ChangeDelta, major bool) { + if major || dns64RouteStateChanged(delta) { + p.resetDNS64State() + } +} + +func (p *prog) storeDiscoveredNAT64Prefix(generation uint64, prefix netip.Prefix) bool { + p.dns64.mu.Lock() + defer p.dns64.mu.Unlock() + if p.dns64.generation != generation { + return false + } + p.dns64.prefix = prefix + p.dns64.active = true + p.dns64.checkedAt = time.Now() + return true +} + +// discoverNAT64Prefix resolves ipv4only.arpa AAAA through the OS-discovered +// resolvers (the network's own DNS64 resolver) and stores the derived +// prefix. Runs in the background; failures leave synthesis inactive until +// the next recheck window. +func (p *prog) discoverNAT64Prefix(generation uint64) { + defer func() { + p.dns64.mu.Lock() + if p.dns64.generation == generation { + p.dns64.discovering = false + } + p.dns64.mu.Unlock() + }() + ctx, cancel := context.WithTimeout(context.Background(), dns64DiscoverTimeout) + defer cancel() + + msg := new(dns.Msg) + msg.SetQuestion(dns64WellKnownName, dns.TypeAAAA) + resolver, err := ctrld.NewResolver(osUpstreamConfig) + if err != nil { + mainLog.Load().Debug().Err(err).Msg("dns64: could not create OS resolver for NAT64 discovery") + return + } + answer, err := resolver.Resolve(ctx, msg) + if err != nil { + mainLog.Load().Debug().Err(err).Msg("dns64: NAT64 prefix discovery query failed") + return + } + prefix, ok := nat64PrefixFromAnswer(answer) + if !ok { + mainLog.Load().Debug().Msg("dns64: no NAT64 prefix present (not a DNS64 network)") + return + } + if !p.storeDiscoveredNAT64Prefix(generation, prefix) { + return + } + mainLog.Load().Info().Msgf("dns64: discovered NAT64 prefix %s; enabling AAAA synthesis for IPv6-only network without CLAT", prefix) +} + +// maybeDNS64 applies DNS64 synthesis to an already-filtered answer when the +// network requires it. resolveA re-resolves the question as an A query +// through the same upstream that produced the answer. +func (p *prog) maybeDNS64(ctx context.Context, req *dns.Msg, answer *dns.Msg, resolveA func(*dns.Msg) *dns.Msg) (*dns.Msg, netip.Prefix) { + if !dns64Eligible(req, answer) || !p.dns64Active() { + return answer, netip.Prefix{} + } + p.dns64.mu.Lock() + prefix := p.dns64.prefix + generation := p.dns64.generation + p.dns64.mu.Unlock() + if !prefix.IsValid() { + return answer, netip.Prefix{} + } + aReq := req.Copy() + aReq.Question[0].Qtype = dns.TypeA + aAnswer := resolveA(aReq) + if aAnswer == nil || aAnswer.Rcode != dns.RcodeSuccess || !sameQuestion(aReq, aAnswer) { + return answer, netip.Prefix{} + } + synth := synthesizeAAAAFromA(req, aAnswer, prefix) + p.dns64.mu.Lock() + current := p.dns64.active && p.dns64.generation == generation && p.dns64.prefix == prefix + p.dns64.mu.Unlock() + if !current { + return answer, netip.Prefix{} + } + if synth == nil || !answerHasAAAA(synth) { + // The companion A lookup completed successfully, so this passthrough + // answer is definitive for the current prefix and may be cached in the + // DNS64 variant to avoid repeating both upstream lookups. + return answer, prefix + } + ctrld.Log(ctx, mainLog.Load().Debug(), "dns64: synthesized AAAA from A records via NAT64 prefix %s", prefix) + return synth, prefix +} diff --git a/cmd/cli/dns64_test.go b/cmd/cli/dns64_test.go new file mode 100644 index 00000000..292450e8 --- /dev/null +++ b/cmd/cli/dns64_test.go @@ -0,0 +1,366 @@ +package cli + +import ( + "net" + "net/netip" + "testing" + "time" + + "github.com/miekg/dns" + "tailscale.com/net/netmon" + + "github.com/Control-D-Inc/ctrld/internal/dnscache" +) + +func mkAAAAReq(name string) *dns.Msg { + m := new(dns.Msg) + m.SetQuestion(dns.Fqdn(name), dns.TypeAAAA) + return m +} + +func mkNetAddr(cidr string) net.Addr { + ip, n, err := net.ParseCIDR(cidr) + if err != nil { + panic(err) + } + n.IP = ip + return n +} + +func TestDNS64NetworkClass(t *testing.T) { + tests := []struct { + name string + defaultRouteAddrs []net.Addr + allAddrs []net.Addr + wantV4 bool + wantCLAT bool + }{ + {"dual stack", []net.Addr{mkNetAddr("10.0.11.61/23"), mkNetAddr("2605:8d80::1/64")}, []net.Addr{mkNetAddr("10.0.11.61/23"), mkNetAddr("2605:8d80::1/64")}, true, false}, + {"virtual rfc1918 does not imply ipv4 connectivity", []net.Addr{mkNetAddr("2605:8d80::1/64")}, []net.Addr{mkNetAddr("2605:8d80::1/64"), mkNetAddr("192.168.65.1/24")}, false, false}, + {"464xlat tether (customer case)", []net.Addr{mkNetAddr("2605:8d80:6b41:122::1/64")}, []net.Addr{mkNetAddr("192.0.0.2/32"), mkNetAddr("2605:8d80:6b41:122::1/64")}, false, true}, + {"v6 only no clat (dns64 network)", []net.Addr{mkNetAddr("2001:db8::1/64")}, []net.Addr{mkNetAddr("2001:db8::1/64")}, false, false}, + {"loopback only", []net.Addr{mkNetAddr("127.0.0.1/8"), mkNetAddr("::1/128")}, []net.Addr{mkNetAddr("127.0.0.1/8"), mkNetAddr("::1/128")}, false, false}, + {"link local v4 ignored", []net.Addr{mkNetAddr("169.254.10.1/16"), mkNetAddr("2001:db8::1/64")}, []net.Addr{mkNetAddr("169.254.10.1/16"), mkNetAddr("2001:db8::1/64")}, false, false}, + {"clat plus real v4", []net.Addr{mkNetAddr("10.0.0.5/24")}, []net.Addr{mkNetAddr("192.0.0.2/32"), mkNetAddr("10.0.0.5/24")}, true, true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + v4, clat := dns64NetworkClass(tc.defaultRouteAddrs, tc.allAddrs) + if v4 != tc.wantV4 || clat != tc.wantCLAT { + t.Errorf("dns64NetworkClass() = (v4=%v, clat=%v), want (v4=%v, clat=%v)", v4, clat, tc.wantV4, tc.wantCLAT) + } + }) + } +} + +func TestNAT64PrefixFromAnswer(t *testing.T) { + mkAnswer := func(v6 string) *dns.Msg { + m := new(dns.Msg) + m.SetQuestion(dns64WellKnownName, dns.TypeAAAA) + r := new(dns.Msg) + r.SetReply(m) + if v6 != "" { + r.Answer = append(r.Answer, &dns.AAAA{ + Hdr: dns.RR_Header{Name: dns64WellKnownName, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 300}, + AAAA: net.ParseIP(v6), + }) + } + return r + } + tests := []struct { + name string + answer *dns.Msg + wantPrefix string + wantOK bool + }{ + {"well-known prefix + 192.0.0.170", mkAnswer("64:ff9b::c000:aa"), "64:ff9b::/96", true}, + {"well-known prefix + 192.0.0.171", mkAnswer("64:ff9b::c000:ab"), "64:ff9b::/96", true}, + {"carrier-specific prefix", mkAnswer("2001:db8:64::c000:aa"), "2001:db8:64::/96", true}, + {"non-dns64 answer (real aaaa)", mkAnswer("2001:db8::1"), "", false}, + {"empty answer", mkAnswer(""), "", false}, + {"nil answer", nil, "", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + p, ok := nat64PrefixFromAnswer(tc.answer) + if ok != tc.wantOK { + t.Fatalf("nat64PrefixFromAnswer() ok = %v, want %v", ok, tc.wantOK) + } + if ok && p != netip.MustParsePrefix(tc.wantPrefix) { + t.Errorf("nat64PrefixFromAnswer() = %s, want %s", p, tc.wantPrefix) + } + }) + } +} + +func TestSynthesizeAAAAFromA(t *testing.T) { + req := mkAAAAReq("legacy.example.com") + aReq := req.Copy() + aReq.Question[0].Qtype = dns.TypeA + aAns := new(dns.Msg) + aAns.SetReply(aReq) + aAns.Answer = []dns.RR{ + &dns.CNAME{Hdr: dns.RR_Header{Name: "legacy.example.com.", Rrtype: dns.TypeCNAME, Class: dns.ClassINET, Ttl: 60}, Target: "cdn.example.net."}, + &dns.A{Hdr: dns.RR_Header{Name: "cdn.example.net.", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, A: net.ParseIP("198.51.100.7")}, + } + prefix := netip.MustParsePrefix("64:ff9b::/96") + + out := synthesizeAAAAFromA(req, aAns, prefix) + if out == nil { + t.Fatal("synthesizeAAAAFromA returned nil") + } + var gotAAAA *dns.AAAA + var gotCNAME *dns.CNAME + for _, rr := range out.Answer { + switch v := rr.(type) { + case *dns.AAAA: + gotAAAA = v + case *dns.CNAME: + gotCNAME = v + case *dns.A: + t.Error("synthesized answer still contains an A record") + } + } + if gotCNAME == nil { + t.Error("CNAME chain record not preserved") + } + if gotAAAA == nil { + t.Fatal("no synthesized AAAA record") + } + want := net.ParseIP("64:ff9b::c633:6407") // 198.51.100.7 embedded + if !gotAAAA.AAAA.Equal(want) { + t.Errorf("synthesized AAAA = %s, want %s", gotAAAA.AAAA, want) + } + if gotAAAA.Hdr.Ttl != 60 { + t.Errorf("TTL not preserved: got %d", gotAAAA.Hdr.Ttl) + } + if out.Question[0].Qtype != dns.TypeAAAA { + t.Errorf("reply question type = %d, want AAAA", out.Question[0].Qtype) + } +} + +func TestSynthesizeAAAAFromAIPv4Eligibility(t *testing.T) { + req := mkAAAAReq("blocked.example") + tests := []struct { + name string + prefix netip.Prefix + ip string + want bool + }{ + {"unspecified with well-known prefix", dns64WellKnownPrefix, "0.0.0.0", false}, + {"loopback with well-known prefix", dns64WellKnownPrefix, "127.0.0.1", false}, + {"link-local with well-known prefix", dns64WellKnownPrefix, "169.254.1.1", false}, + {"private with well-known prefix", dns64WellKnownPrefix, "10.0.0.1", false}, + {"private with network-specific prefix", netip.MustParsePrefix("2001:db8:64::/96"), "10.0.0.1", true}, + {"unspecified with network-specific prefix", netip.MustParsePrefix("2001:db8:64::/96"), "0.0.0.0", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + aReq := req.Copy() + aReq.Question[0].Qtype = dns.TypeA + aAns := new(dns.Msg) + aAns.SetReply(aReq) + aAns.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: req.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, A: net.ParseIP(tc.ip)}} + if got := answerHasAAAA(synthesizeAAAAFromA(req, aAns, tc.prefix)); got != tc.want { + t.Fatalf("answerHasAAAA() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestDNS64Eligible(t *testing.T) { + emptyReply := func(req *dns.Msg, rcode int) *dns.Msg { + r := new(dns.Msg) + r.SetReply(req) + r.Rcode = rcode + return r + } + aaaaReq := mkAAAAReq("x.example.") + cdReq := aaaaReq.Copy() + cdReq.CheckingDisabled = true + withAAAA := emptyReply(aaaaReq, dns.RcodeSuccess) + withAAAA.Answer = []dns.RR{&dns.AAAA{Hdr: dns.RR_Header{Name: "x.example.", Rrtype: dns.TypeAAAA, Class: dns.ClassINET}, AAAA: net.ParseIP("2001:db8::1")}} + aReq := new(dns.Msg) + aReq.SetQuestion("x.example.", dns.TypeA) + + tests := []struct { + name string + req *dns.Msg + answer *dns.Msg + want bool + }{ + {"AAAA empty NOERROR -> eligible", aaaaReq, emptyReply(aaaaReq, dns.RcodeSuccess), true}, + {"AAAA with records -> not eligible", aaaaReq, withAAAA, false}, + {"CD query is not synthesized", cdReq, emptyReply(cdReq, dns.RcodeSuccess), false}, + {"NXDOMAIN never synthesized", aaaaReq, emptyReply(aaaaReq, dns.RcodeNameError), false}, + {"SERVFAIL never synthesized", aaaaReq, emptyReply(aaaaReq, dns.RcodeServerFailure), false}, + {"A query not eligible", aReq, emptyReply(aReq, dns.RcodeSuccess), false}, + {"nil answer", aaaaReq, nil, false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := dns64Eligible(tc.req, tc.answer); got != tc.want { + t.Errorf("dns64Eligible() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestDNS64ActiveGating(t *testing.T) { + cases := []struct { + name string + hasV4 bool + hasCLAT bool + prefix string + want bool + }{ + {"dual stack, prefix known", true, false, "64:ff9b::/96", false}, + {"clat network, prefix known", false, true, "64:ff9b::/96", false}, + {"v6-only no clat, prefix known", false, false, "64:ff9b::/96", true}, + {"v6-only no clat, no prefix yet", false, false, "", false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + old := dns64NetworkClassFn + dns64NetworkClassFn = func() (bool, bool, error) { return tc.hasV4, tc.hasCLAT, nil } + t.Cleanup(func() { dns64NetworkClassFn = old }) + + p := &prog{} + p.dns64.discovering = true // block background discovery in tests + if tc.prefix != "" { + p.dns64.prefix = netip.MustParsePrefix(tc.prefix) + } + if got := p.dns64Active(); got != tc.want { + t.Errorf("dns64Active() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestStoreDiscoveredNAT64PrefixActivatesImmediately(t *testing.T) { + p := &prog{} + p.dns64.generation = 4 + p.dns64.checkedAt = time.Now() + prefix := netip.MustParsePrefix("64:ff9b::/96") + if !p.storeDiscoveredNAT64Prefix(4, prefix) { + t.Fatal("current discovery result was rejected") + } + if !p.dns64.active || p.dns64.prefix != prefix || p.dns64.checkedAt.IsZero() { + t.Fatalf("discovery did not immediately activate DNS64: active=%v prefix=%s checkedAt=%s", p.dns64.active, p.dns64.prefix, p.dns64.checkedAt) + } + if p.storeDiscoveredNAT64Prefix(3, netip.MustParsePrefix("2001:db8:64::/96")) { + t.Fatal("stale discovery result was accepted") + } + if p.dns64.prefix != prefix { + t.Fatalf("stale discovery replaced prefix: %s", p.dns64.prefix) + } +} + +func TestDNS64NetworkChangeInvalidatesPrefix(t *testing.T) { + p := &prog{} + p.dns64.prefix = netip.MustParsePrefix("64:ff9b::/96") + p.dns64.active = true + p.dns64.checkedAt = time.Now() + p.dns64.discovering = true + + delta := &netmon.ChangeDelta{ + Old: &netmon.State{DefaultRouteInterface: "en0", HaveV6: true, InterfaceIPs: map[string][]netip.Prefix{"en0": {netip.MustParsePrefix("2001:db8:1::1/64")}}}, + New: &netmon.State{DefaultRouteInterface: "en0", HaveV6: true, InterfaceIPs: map[string][]netip.Prefix{"en0": {netip.MustParsePrefix("2001:db8:2::1/64")}}}, + } + p.handleDNS64NetworkChange(delta, false) + if p.dns64.active || p.dns64.prefix.IsValid() || !p.dns64.checkedAt.IsZero() || p.dns64.discovering { + t.Fatalf("network change did not invalidate DNS64 state: active=%v prefix=%s checkedAt=%s discovering=%v", p.dns64.active, p.dns64.prefix, p.dns64.checkedAt, p.dns64.discovering) + } + if p.dns64.generation != 1 { + t.Fatalf("generation = %d, want 1", p.dns64.generation) + } +} + +func TestDNS64CacheKeyPartitionsByPrefix(t *testing.T) { + req := mkAAAAReq("legacy.example") + normal := dnscache.NewKey(req, "upstream.0") + wellKnown := dns64CacheKey(req, "upstream.0", netip.MustParsePrefix("64:ff9b::/96")) + carrier := dns64CacheKey(req, "upstream.0", netip.MustParsePrefix("2001:db8:64::/96")) + if normal == wellKnown || wellKnown == carrier { + t.Fatalf("normal and per-prefix synthesized cache keys must be distinct: normal=%+v well-known=%+v carrier=%+v", normal, wellKnown, carrier) + } + cache, err := dnscache.NewLRUCache(4) + if err != nil { + t.Fatal(err) + } + answer := new(dns.Msg) + answer.SetReply(req) + cache.Add(wellKnown, dnscache.NewValue(answer, time.Now().Add(time.Minute))) + if cache.Get(wellKnown) == nil || cache.Get(normal) != nil || cache.Get(carrier) != nil { + t.Fatal("synthesized cache entry crossed the normal or carrier-prefix partition") + } +} + +func TestMaybeDNS64EndToEnd(t *testing.T) { + old := dns64NetworkClassFn + dns64NetworkClassFn = func() (bool, bool, error) { return false, false, nil } + t.Cleanup(func() { dns64NetworkClassFn = old }) + + p := &prog{} + p.dns64.discovering = true + p.dns64.prefix = netip.MustParsePrefix("64:ff9b::/96") + + req := mkAAAAReq("legacy.example.com") + empty := new(dns.Msg) + empty.SetReply(req) + + resolveA := func(aReq *dns.Msg) *dns.Msg { + if aReq.Question[0].Qtype != dns.TypeA { + t.Fatalf("resolveA called with qtype %d", aReq.Question[0].Qtype) + } + r := new(dns.Msg) + r.SetReply(aReq) + r.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: aReq.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 30}, A: net.ParseIP("203.0.113.9")}} + return r + } + out, usedPrefix := p.maybeDNS64(t.Context(), req, empty, resolveA) + if !answerHasAAAA(out) { + t.Fatal("expected synthesized AAAA answer") + } + if usedPrefix != p.dns64.prefix { + t.Fatalf("used prefix = %s, want %s", usedPrefix, p.dns64.prefix) + } + + blocked := new(dns.Msg) + blocked.SetReply(req) + blocked.Rcode = dns.RcodeNameError + if got, _ := p.maybeDNS64(t.Context(), req, blocked, resolveA); got != blocked { + t.Error("NXDOMAIN answer must pass through unsynthesized") + } + + blockedA := func(aReq *dns.Msg) *dns.Msg { + r := new(dns.Msg) + r.SetReply(aReq) + r.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: aReq.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 30}, A: net.IPv4zero}} + return r + } + if got, prefix := p.maybeDNS64(t.Context(), req, empty, blockedA); got != empty || prefix != p.dns64.prefix { + t.Error("NODATA plus 0.0.0.0 block answer must pass through unsynthesized and be cacheable for the current prefix") + } +} + +func TestMaybeDNS64DropsStaleInFlightPrefix(t *testing.T) { + p := &prog{} + p.dns64.active = true + p.dns64.checkedAt = time.Now() + p.dns64.prefix = netip.MustParsePrefix("64:ff9b::/96") + req := mkAAAAReq("legacy.example") + empty := new(dns.Msg) + empty.SetReply(req) + + got, prefix := p.maybeDNS64(t.Context(), req, empty, func(aReq *dns.Msg) *dns.Msg { + p.resetDNS64State() + r := new(dns.Msg) + r.SetReply(aReq) + r.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: aReq.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 30}, A: net.ParseIP("203.0.113.9")}} + return r + }) + if got != empty || prefix.IsValid() { + t.Fatal("in-flight synthesis used a prefix invalidated by a network change") + } +} diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index a3f6b57e..35a2c717 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -84,6 +84,52 @@ type upstreamForResult struct { srcAddr string } +func (p *prog) addCachedResponse(key dnscache.Key, answer *dns.Msg) { + ttl := ttlFromMsg(answer) + now := time.Now() + expired := now.Add(time.Duration(ttl) * time.Second) + if cachedTTL := p.cfg.Service.CacheTTLOverride; cachedTTL > 0 { + expired = now.Add(time.Duration(cachedTTL) * time.Second) + } + setCachedAnswerTTL(answer, now, expired) + p.cache.Add(key, dnscache.NewValue(answer, expired)) +} + +func (p *prog) cachedResponse(req *dns.Msg, upstream string, dns64Prefix netip.Prefix, dns64Active bool, now time.Time) (answer, stale *dns.Msg, hit, dns64Hit, dns64Bypass bool) { + if dns64Active { + if cachedValue := p.cache.Get(dns64CacheKey(req, upstream, dns64Prefix)); cachedValue != nil { + answer = cachedValue.Msg.Copy() + ctrld.SetCacheReply(answer, req, answer.Rcode) + if cachedValue.Expire.After(now) { + setCachedAnswerTTL(answer, now, cachedValue.Expire) + return answer, nil, true, true, false + } + stale = answer + } + } + + cachedValue := p.cache.Get(dnscache.NewKey(req, upstream)) + if cachedValue == nil { + return nil, stale, false, false, false + } + answer = cachedValue.Msg.Copy() + ctrld.SetCacheReply(answer, req, answer.Rcode) + if cachedValue.Expire.After(now) { + if dns64Eligible(req, answer) && dns64Active { + if stale == nil { + stale = answer + } + return nil, stale, false, false, true + } + setCachedAnswerTTL(answer, now, cachedValue.Expire) + return answer, stale, true, false, false + } + if stale == nil { + stale = answer + } + return nil, stale, false, false, false +} + func (p *prog) serveDNS(listenerNum string) error { listenerConfig := p.cfg.Listener[listenerNum] // make sure ip is allocated @@ -517,22 +563,29 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { // Inverse query should not be cached: https://www.rfc-editor.org/rfc/rfc1035#section-7.4 if p.cache != nil && req.msg.Question[0].Qtype != dns.TypePTR { + dns64Prefix, dns64Active := netip.Prefix{}, false + if req.msg.Question[0].Qtype == dns.TypeAAAA { + dns64Prefix, dns64Active = p.activeDNS64Prefix() + } for _, upstream := range upstreams { - cachedValue := p.cache.Get(dnscache.NewKey(req.msg, upstream)) - if cachedValue == nil { + answer, stale, hit, dns64Hit, dns64Bypass := p.cachedResponse(req.msg, upstream, dns64Prefix, dns64Active, time.Now()) + if stale != nil { + staleAnswer = stale + } + if dns64Bypass { + ctrld.Log(ctx, mainLog.Load().Debug(), "dns64: bypassing cached empty-AAAA answer for synthesis") + } + if !hit { continue } - answer := cachedValue.Msg.Copy() - ctrld.SetCacheReply(answer, req.msg, answer.Rcode) - now := time.Now() - if cachedValue.Expire.After(now) { + if dns64Hit { + ctrld.Log(ctx, mainLog.Load().Debug(), "dns64: hit cached response variant") + } else { ctrld.Log(ctx, mainLog.Load().Debug(), "hit cached response") - setCachedAnswerTTL(answer, now, cachedValue.Expire) - res.answer = answer - res.cached = true - return res } - staleAnswer = answer + res.answer = answer + res.cached = true + return res } } @@ -756,20 +809,40 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { answer.Compress = true if p.cache != nil && req.msg.Question[0].Qtype != dns.TypePTR { - ttl := ttlFromMsg(answer) - now := time.Now() - expired := now.Add(time.Duration(ttl) * time.Second) - if cachedTTL := p.cfg.Service.CacheTTLOverride; cachedTTL > 0 { - expired = now.Add(time.Duration(cachedTTL) * time.Second) - } - setCachedAnswerTTL(answer, now, expired) - p.cache.Add(dnscache.NewKey(req.msg, upstreams[n]), dnscache.NewValue(answer, expired)) + p.addCachedResponse(dnscache.NewKey(req.msg, upstreams[n]), answer) ctrld.Log(ctx, mainLog.Load().Debug(), "add cached response") } hostname := "" if req.ci != nil { hostname = req.ci.Hostname } + // DNS64 synthesis for IPv6-only networks without CLAT: applied to the + // policy-approved answer only, using the same upstream for the companion + // A resolution. No-op unless the network state requires it. + var synthesizedPrefix netip.Prefix + answer, synthesizedPrefix = p.maybeDNS64(ctx, req.msg, answer, func(aReq *dns.Msg) *dns.Msg { + key := dnscache.NewKey(aReq, upstreams[n]) + if p.cache != nil { + if cachedValue := p.cache.Get(key); cachedValue != nil { + now := time.Now() + if cachedValue.Expire.After(now) { + cached := cachedValue.Msg.Copy() + ctrld.SetCacheReply(cached, aReq, cached.Rcode) + setCachedAnswerTTL(cached, now, cachedValue.Expire) + return cached + } + } + } + resolved := resolve(upstreams[n], upstreamConfig, aReq) + if p.cache != nil && resolved != nil && sameQuestion(aReq, resolved) { + p.addCachedResponse(key, resolved) + } + return resolved + }) + if p.cache != nil && synthesizedPrefix.IsValid() { + p.addCachedResponse(dns64CacheKey(req.msg, upstreams[n], synthesizedPrefix), answer) + ctrld.Log(ctx, mainLog.Load().Debug(), "dns64: add cached response variant") + } ctrld.Log(ctx, mainLog.Load().Info(), "REPLY: %s -> %s (%s): %s", upstreams[n], req.ufr.srcAddr, hostname, dns.RcodeToString[answer.Rcode]) res.answer = answer res.upstream = upstreamConfig.Endpoint @@ -1460,11 +1533,12 @@ func (p *prog) monitorNetworkChanges() error { } mon.RegisterChangeCallback(func(delta *netmon.ChangeDelta) { + isMajorChange := mon.IsMajorChangeFrom(delta.Old, delta.New) + p.handleDNS64NetworkChange(delta, isMajorChange) + // Get map of valid interfaces validIfaces := validInterfacesMap() - isMajorChange := mon.IsMajorChangeFrom(delta.Old, delta.New) - mainLog.Load().Debug(). Interface("old_state", delta.Old). Interface("new_state", delta.New). diff --git a/cmd/cli/dns_proxy_test.go b/cmd/cli/dns_proxy_test.go index 5a82a1a9..5462ff04 100644 --- a/cmd/cli/dns_proxy_test.go +++ b/cmd/cli/dns_proxy_test.go @@ -200,6 +200,52 @@ func TestCache(t *testing.T) { assert.Equal(t, answer2.Rcode, got2.answer.Rcode) } +func TestDNS64CacheLookup(t *testing.T) { + cfg := testhelper.SampleConfig(t) + p := &prog{cfg: cfg} + cache, err := dnscache.NewLRUCache(16) + require.NoError(t, err) + p.cache = cache + + now := time.Now() + prefix := dns64WellKnownPrefix + req := mkAAAAReq("legacy.example") + upstream := "upstream.0" + empty := new(dns.Msg) + empty.SetReply(req) + synthesized := new(dns.Msg) + synthesized.SetReply(req) + synthesized.Answer = []dns.RR{&dns.AAAA{Hdr: dns.RR_Header{Name: req.Question[0].Name, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}, AAAA: net.ParseIP("64:ff9b::c000:201")}} + + t.Run("fresh variant hit", func(t *testing.T) { + p.cache.Purge() + p.cache.Add(dns64CacheKey(req, upstream, prefix), dnscache.NewValue(synthesized, now.Add(time.Minute))) + answer, stale, hit, dns64Hit, bypass := p.cachedResponse(req, upstream, prefix, true, now) + if answer == nil || !answerHasAAAA(answer) || stale != nil || !hit || !dns64Hit || bypass { + t.Fatalf("unexpected lookup result: answer=%v stale=%v hit=%v dns64Hit=%v bypass=%v", answer, stale, hit, dns64Hit, bypass) + } + }) + + t.Run("fresh empty normal answer is retained as stale while bypassed", func(t *testing.T) { + p.cache.Purge() + p.cache.Add(dnscache.NewKey(req, upstream), dnscache.NewValue(empty, now.Add(time.Minute))) + answer, stale, hit, dns64Hit, bypass := p.cachedResponse(req, upstream, prefix, true, now) + if answer != nil || stale == nil || hit || dns64Hit || !bypass { + t.Fatalf("unexpected lookup result: answer=%v stale=%v hit=%v dns64Hit=%v bypass=%v", answer, stale, hit, dns64Hit, bypass) + } + }) + + t.Run("expired variant is preferred as stale", func(t *testing.T) { + p.cache.Purge() + p.cache.Add(dns64CacheKey(req, upstream, prefix), dnscache.NewValue(synthesized, now.Add(-time.Minute))) + p.cache.Add(dnscache.NewKey(req, upstream), dnscache.NewValue(empty, now.Add(-time.Minute))) + answer, stale, hit, dns64Hit, bypass := p.cachedResponse(req, upstream, prefix, true, now) + if answer != nil || stale == nil || !answerHasAAAA(stale) || hit || dns64Hit || bypass { + t.Fatalf("unexpected lookup result: answer=%v stale=%v hit=%v dns64Hit=%v bypass=%v", answer, stale, hit, dns64Hit, bypass) + } + }) +} + func Test_ipAndMacFromMsg(t *testing.T) { tests := []struct { name string diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 375bdc20..082269e4 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -172,6 +172,10 @@ type prog struct { // instead of using the normal upstream flow. recoveryBypass atomic.Bool + // dns64 tracks DNS64/NAT64 synthesis state for IPv6-only networks + // without client-side 464XLAT. See cmd/cli/dns64.go. + dns64 dns64State + // interceptDNSTargetService names the macOS network service on which // ctrld set a loopback DNS value because the service provided no usable // IPv4 DNS while DNS intercept mode was active (issue #533); diff --git a/internal/dnscache/cache.go b/internal/dnscache/cache.go index 1437370c..90f6bb9f 100644 --- a/internal/dnscache/cache.go +++ b/internal/dnscache/cache.go @@ -29,6 +29,7 @@ type Key struct { Name string Upstream string ECS string + Variant string } type Value struct { @@ -71,6 +72,14 @@ func NewKey(msg *dns.Msg, upstream string) Key { return Key{Qtype: q.Qtype, Qclass: q.Qclass, Name: normalizeQname(q.Name), Upstream: upstream, ECS: CanonicalECS(msg)} } +// NewVariantKey creates a cache key in a named result variant. Variants keep +// derived answers separate from the upstream's answer to the same question. +func NewVariantKey(msg *dns.Msg, upstream, variant string) Key { + key := NewKey(msg, upstream) + key.Variant = variant + return key +} + // CanonicalECS returns a canonical string form of the EDNS Client Subnet (ECS, // EDNS option 8) carried by msg, suitable for partitioning cache and // singleflight keys. A request with no ECS option returns "", so all ECS-less From e109c9be3353d16624f64c6ff3371019bdafffff Mon Sep 17 00:00:00 2001 From: Dev Scribe Date: Fri, 4 Sep 2026 13:14:57 -0400 Subject: [PATCH 03/29] clientinfo: skip unavailable Ubios Mongo discovery --- internal/clientinfo/ubios.go | 28 ++++++++++++++++++++++++-- internal/clientinfo/ubios_test.go | 22 ++++++++++++++++++++ internal/clientinfo/ubios_unix_test.go | 27 +++++++++++++++++++++++++ 3 files changed, 75 insertions(+), 2 deletions(-) create mode 100644 internal/clientinfo/ubios_unix_test.go diff --git a/internal/clientinfo/ubios.go b/internal/clientinfo/ubios.go index 0ffd6e59..cffc6895 100644 --- a/internal/clientinfo/ubios.go +++ b/internal/clientinfo/ubios.go @@ -3,8 +3,10 @@ package clientinfo import ( "bytes" "encoding/json" + "errors" "fmt" "io" + "os" "os/exec" "strings" "sync" @@ -13,9 +15,12 @@ import ( "github.com/Control-D-Inc/ctrld/internal/router/ubios" ) +const defaultUbiosMongoPath = "/usr/bin/mongo" + // ubiosDiscover provides client discovery functionality on Ubios routers. type ubiosDiscover struct { - hostname sync.Map // mac => hostname + hostname sync.Map // mac => hostname + mongoPath string } // refresh reloads unifi devices from database. @@ -42,7 +47,19 @@ func (u *ubiosDiscover) LookupHostnameByMac(mac string) string { // refreshDevices updates unifi devices name from local mongodb. func (u *ubiosDiscover) refreshDevices() error { - cmd := exec.Command("/usr/bin/mongo", "localhost:27117/ace", "--quiet", "--eval", ` + mongoPath := u.mongoExecutable() + fi, err := os.Stat(mongoPath) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return fmt.Errorf("stat mongo executable: %w", err) + } + if !fi.Mode().IsRegular() || fi.Mode().Perm()&0111 == 0 { + return nil + } + + cmd := exec.Command(mongoPath, "localhost:27117/ace", "--quiet", "--eval", ` DBQuery.shellBatchSize = 256; db.user.find({name: {$exists: true, $ne: ""}}, {_id:0, mac:1, name:1});`) b, err := cmd.CombinedOutput() @@ -52,6 +69,13 @@ func (u *ubiosDiscover) refreshDevices() error { return u.storeDevices(bytes.NewReader(b)) } +func (u *ubiosDiscover) mongoExecutable() string { + if u.mongoPath != "" { + return u.mongoPath + } + return defaultUbiosMongoPath +} + // storeDevices saves unifi devices name for caching. func (u *ubiosDiscover) storeDevices(r io.Reader) error { decoder := json.NewDecoder(r) diff --git a/internal/clientinfo/ubios_test.go b/internal/clientinfo/ubios_test.go index 657cf180..4b788ba9 100644 --- a/internal/clientinfo/ubios_test.go +++ b/internal/clientinfo/ubios_test.go @@ -1,10 +1,32 @@ package clientinfo import ( + "os" + "path/filepath" "strings" "testing" ) +func Test_ubiosDiscover_refreshDevices_skipsUnavailableMongo(t *testing.T) { + t.Run("missing", func(t *testing.T) { + ud := &ubiosDiscover{mongoPath: filepath.Join(t.TempDir(), "mongo")} + if err := ud.refreshDevices(); err != nil { + t.Fatalf("refreshDevices() error = %v", err) + } + }) + + t.Run("not executable", func(t *testing.T) { + path := filepath.Join(t.TempDir(), "mongo") + if err := os.WriteFile(path, nil, 0600); err != nil { + t.Fatal(err) + } + ud := &ubiosDiscover{mongoPath: path} + if err := ud.refreshDevices(); err != nil { + t.Fatalf("refreshDevices() error = %v", err) + } + }) +} + func Test_ubiosDiscover_storeDevices(t *testing.T) { ud := &ubiosDiscover{} r := strings.NewReader(`{ "mac": "00:00:00:00:00:01", "name": "device 1" } diff --git a/internal/clientinfo/ubios_unix_test.go b/internal/clientinfo/ubios_unix_test.go new file mode 100644 index 00000000..edb89ecb --- /dev/null +++ b/internal/clientinfo/ubios_unix_test.go @@ -0,0 +1,27 @@ +//go:build unix + +package clientinfo + +import ( + "os" + "path/filepath" + "testing" +) + +func Test_ubiosDiscover_refreshDevices_usesExecutableMongo(t *testing.T) { + path := filepath.Join(t.TempDir(), "mongo") + const script = `#!/bin/sh +printf '%s\n' '{"mac":"00:00:00:00:00:01","name":"device 1"}' +` + if err := os.WriteFile(path, []byte(script), 0700); err != nil { + t.Fatal(err) + } + + ud := &ubiosDiscover{mongoPath: path} + if err := ud.refreshDevices(); err != nil { + t.Fatalf("refreshDevices() error = %v", err) + } + if got := ud.LookupHostnameByMac("00:00:00:00:00:01"); got != "device 1" { + t.Fatalf("LookupHostnameByMac() = %q, want %q", got, "device 1") + } +} From 3cd20a3eef98634a735eb145248b6793947a820c Mon Sep 17 00:00:00 2001 From: Anthony Wong Date: Wed, 9 Sep 2026 14:35:41 +0000 Subject: [PATCH 04/29] feat(cli): port provisioning hardening to v1.0 --- .gitignore | 4 + SPEC.md | 206 ------ cmd/cli/cli.go | 324 ++++++-- cmd/cli/cli_provision_test.go | 239 ++++++ cmd/cli/commands.go | 65 +- cmd/cli/commands_diag.go | 473 ++++++++++++ cmd/cli/commands_diag_test.go | 429 +++++++++++ cmd/cli/commands_start_test.go | 152 ++++ cmd/cli/hostname.go | 70 +- cmd/cli/hostname_test.go | 62 ++ cmd/cli/prog.go | 30 +- cmd/cli/prog_intercept_fallback_test.go | 2 +- cmd/cli/provision_boundary_guard_test.go | 243 ++++++ cmd/cli/provision_input_test.go | 417 +++++++++++ cmd/cli/provision_result.go | 125 +++- cmd/cli/provision_result_test.go | 24 +- cmd/cli/provision_terminal_paths_test.go | 244 ++++++ docs/provisioning-failure-codes.md | 137 +++- internal/controld/config.go | 45 +- internal/controld/config_test.go | 110 +++ internal/controld/reachability.go | 33 + internal/controld/reachability_test.go | 14 + tasks/plan.md | 213 ------ tasks/todo.md | 21 - .../darwin/test-pkg-intercept-mode.sh | 20 +- .../test-postinstall-provision-failure.sh | 692 +++++++++++++++--- 26 files changed, 3714 insertions(+), 680 deletions(-) delete mode 100644 SPEC.md create mode 100644 cmd/cli/commands_diag.go create mode 100644 cmd/cli/commands_diag_test.go create mode 100644 cmd/cli/provision_boundary_guard_test.go create mode 100644 cmd/cli/provision_input_test.go create mode 100644 cmd/cli/provision_terminal_paths_test.go create mode 100644 internal/controld/reachability.go create mode 100644 internal/controld/reachability_test.go delete mode 100644 tasks/plan.md delete mode 100644 tasks/todo.md diff --git a/.gitignore b/.gitignore index 799011f6..e9121bf7 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,7 @@ ctrld-* cmd/cli/rsrc_*.syso ctrld ctrld.exe + +# Local planning artifacts - never commit +/SPEC.md +/tasks/ diff --git a/SPEC.md b/SPEC.md deleted file mode 100644 index 706baee3..00000000 --- a/SPEC.md +++ /dev/null @@ -1,206 +0,0 @@ -# SPEC: Stable customer-visible provisioning failure codes - -Issue: [#586](https://gitlab.int.windscribe.com/controld/clients/ctrld/-/issues/586) -Requested by: Catt Garrod (@catt). Scope expanded by: Anthony Wong (@anthony). - -## 1. Objective - -Terminal provisioning failures in ctrld — bootstrap/API setup, listener -binding, and service installation/startup — must produce a stable, -support-facing failure identifier that survives process exit and reaches -both manual CLI users and MDM-driven installs. A customer or admin reports -one code; Support maps it to a scenario and a next action without asking -for reruns or verbose logs. - -Motivating incident (v1.5.5, macOS): provisioning reached the Control D -API, then died with only `FTL listener.0 could not find available listen -ip and port`. The per-address UDP/TCP bind errors existed only at Info -level in an in-memory logger and vanished on exit. The macOS pkg -`postinstall` discards ctrld's stdout/stderr entirely and judges success -by plist existence, so nothing useful reached the MDM log. - -**Users:** end customers and IT admins reporting failures; Support agents -triaging them; MDM/RMM operators reading installer logs. - -### Failure contract (agreed design) - -Three surfaces, all carrying the same identifier: - -1. **Result file** — on terminal provisioning failure, ctrld writes a - small redacted JSON file (atomic write: temp + rename) in the ctrld - home directory (same base dir as the internal `ctrld.log`, - via `absHomeDir`). Removed/overwritten on later successful - provisioning so stale failures don't mislead. Schema: - - ```json - { - "version": 1, - "timestamp": "2026-08-18T12:00:00Z", - "stage": "listener", - "code": "LISTENER_BIND_FAILED", - "exit_code": 41, - "message": "could not find available listen ip and port", - "detail": { - "attempts": [ - {"addr": "127.0.0.1:53", "proto": "udp", "os_error": "address already in use"} - ] - } - } - ``` - - `detail` is bounded (cap recorded bind attempts; cap string lengths) - and redacted by construction: no provisioning tokens, resolver IDs, - config contents, or unrelated host data. - -2. **Exit code + final stderr line** — the installer-facing command - (`ctrld start`, and `ctrld run` when run manually in the foreground) - exits with a stage-scoped code and prints one final line containing - the string code and stage, e.g. - `provisioning failed: stage=listener code=LISTENER_BIND_FAILED (exit 41)`. - -3. **Installer log (MDM path)** — `scripts/pkg/postinstall` stops - discarding the signal: it captures `ctrld start`'s output to a - private temp file, extracts only the fixed-charset identifier line - (`stage=[a-z]* code=[A-Z_]* (exit [0-9]*)` — structurally unable to - carry the token), and echoes it with the exit code into the - installer log. The result file's `message`/`detail` fields are - deliberately never surfaced there. The plist-existence check remains - the final success gate. - -### Identifier format - -- **Primary identifier: stable string codes.** Initial set — - bootstrap: `API_UNREACHABLE`, `API_REJECTED`, `API_DEVICE_INVALID`; - listener: `LISTENER_BIND_FAILED`, `LISTENER_CONFIGURED_ADDR_UNAVAILABLE`; - service: `SERVICE_INSTALL_FAILED`, `SERVICE_START_FAILED`, - `SERVICE_SELFCHECK_FAILED`. Codes are append-only; renames are new - codes plus a deprecation note in the mapping doc. -- **Secondary: stage-scoped process exit codes** as a coarse machine - signal: bootstrap 30–39, listener 40–49, service install/start 50–59. - Each string code owns one exit code. Existing contracts are untouched: - `ctrld status` 0–3, deactivation-pin 126, success 0. -- One underlying failure maps to one code on every path (manual CLI and - MDM), on both branches. - -### Propagation (daemon → installer) - -The listener/bootstrap fatals fire inside the daemon process -(`ctrld run` under launchd/systemd/SCM), not in `ctrld start`. The -daemon writes the result file before exiting; the existing log-socket -exit notification (`notifyExitToLogServer`) already unblocks `ctrld -start`'s self-check. `ctrld start` then reads the result file, prints -the identifier, and exits with the mapped stage exit code. The daemon's -own exit-status semantics toward service managers are preserved — -in particular the deliberate exit-0 on permanent API rejection that -protects the restart-policy budget; the result file carries the failure -identity in that case. - -### Support mapping - -`docs/provisioning-failure-codes.md` in this repo: one row per code — -code, stage, exit code, failure scenario, next safe troubleshooting -action or evidence request. Updated in the same MR whenever a code is -added or changed. - -### Branch scope - -Full implementation on **both** `v1.0` (release line for v1.5.5) and -`master`. The branches diverge heavily (`v1.0`: zerolog fork, -`commands.go`, `service_status.go`, macOS pkg scripts; `master`: zap, -inline commands, no pkg scripts), so this is one shared contract -(codes, exit-code ranges, file schema, doc) implemented twice, as two -MRs referencing #586. - -## 2. Commands - -- Build: `go build ./...` -- Test: `go test ./cmd/cli/...` (full: `go test ./...`) -- Vet: `go vet ./...` -- Branch workflow: feature branch off `v1.0` for the v1.0 MR; separate - feature branch off `master` for the port MR. Rebase, never merge the - base branch in. - -## 3. Project structure - -New and touched files on `v1.0` (master port mirrors the same contract -at its equivalent emission points in its `cli.go`): - -- `cmd/cli/provision_result.go` (new) — stage + code enums, exit-code - mapping, result-file schema, atomic write/read/clear helpers, - bounded/redacted detail builders. Pattern follows `service_status.go` - (small file: named constants + classifier + dedicated tests). -- `cmd/cli/provision_result_test.go` (new). -- `cmd/cli/cli.go` — emission points: `run()` bootstrap failure branches - (permanent rejection, invalid-device, fatal fetch), and - `tryUpdateListenerConfig` / `tryUpdateListenerConfigIntercept` fatals, - which now record per-attempt `{addr, proto, os_error}` bind detail. -- `cmd/cli/commands.go` — `initStartCmd`: doTasks install/start failures - and the self-check failure branch read the result file, print the - identifier, and exit with the stage code (replacing bare `os.Exit(1)` - on those paths). -- `scripts/pkg/postinstall` — propagate exit code + result-file contents - into the installer log (v1.0 only; master has no pkg scripts). -- `docs/provisioning-failure-codes.md` (new) — support mapping. - -## 4. Code style - -- Per repo conventions and global rules: guard clauses, small functions, - descriptive names, explicit error handling — never weaken existing - handling (e.g. keep the permanent-rejection exit-0 rationale intact). -- Comments only for non-obvious constraints (e.g. why the daemon must - still exit 0 on permanent rejection), simple-english, self-contained — - no issue/MR references in code. -- Match each branch's logging idiom: zerolog fork on `v1.0`, zap on - `master`. No new dependencies. -- Conventional Commits; MR titles in simple-english; both MRs reference - #586 (release-line MR carries `Closes #586`). - -## 5. Testing strategy - -Test-first where the harness allows. Coverage required by the issue: - -- **Code/mapping unit tests** — every string code maps to exactly one - stage and one in-range exit code; ranges don't collide with existing - contracts (0–3 status, 126 pin). -- **Result file round-trip** — write/read/clear; atomic write; stale - file removed on success. -- **Redaction** — serialize a result built from inputs containing a - provision token, resolver ID, and config content; assert none appear. -- **Listener bind failure (regression test for the incident)** — occupy - a port, drive the listener-config path to exhaustion, assert the - result records `LISTENER_BIND_FAILED` with attempted address, UDP/TCP - operation, and OS error (`address already in use`-class). -- **Bootstrap failures** — mock API: permanent 4xx → `API_REJECTED`; - invalid-device 40402 → `API_DEVICE_INVALID`; unreachable → - `API_UNREACHABLE`. -- **Service install/start/self-check failures** — injected task - failures assert code selection and `ctrld start` exit code. -- **MDM surface** — shell-level check of `postinstall` failure branch - (result file present → correct log line and exit), aligned with the - existing `test-scripts/` approach; manual pkg verification steps - documented in the MR. -- Both branches: the shared contract tests exist on both; branch-specific - emission tests match each branch's structure. - -## 6. Boundaries - -**Always:** -- Redact tokens, resolver IDs, config contents, host data from every - customer-visible surface (result file, stderr line, installer log). -- Preserve existing exit-code contracts (`ctrld status` 0–3, pin 126) - and the daemon's service-manager-facing exit semantics. -- Bound all recorded detail (attempt counts, string lengths). -- Keep codes append-only once merged. - -**Ask first:** -- Changing the daemon's (`ctrld run` under a service manager) exit codes - or restart-relevant behavior beyond writing the result file. -- Adding any persisted file outside the ctrld home directory. -- Expanding scope to runtime (post-provisioning) failures — this ticket - owns terminal provisioning failures only. - -**Never:** -- Print or persist the provisioning token (the reason postinstall - discards output today — the replacement surface must stay token-free). -- Auto-detect or kill conflicting processes (explicitly out of scope). -- Break `ctrld status`'s documented exit-code contract. diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 98dc7637..2c4a6a36 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -24,6 +24,7 @@ import ( "strings" "sync/atomic" "time" + "unicode" "github.com/Masterminds/semver/v3" "github.com/cuonglm/osinfo" @@ -140,6 +141,7 @@ func initCLI() { initClientsCmd() initUpgradeCmd() initLogCmd() + initDiagCmd() } // isMobile reports whether the current OS is a mobile platform. @@ -183,8 +185,12 @@ func isStableVersion(vs string) bool { // RunCobraCommand runs ctrld cli. func RunCobraCommand(cmd *cobra.Command) { noConfigStart = isNoConfigStart(cmd) - checkStrFlagEmpty(cmd, cdUidFlagName) - checkStrFlagEmpty(cmd, cdOrgFlagName) + if !checkStrFlagEmpty(cmd, cdUidFlagName) { + return + } + if !checkStrFlagEmpty(cmd, cdOrgFlagName) { + return + } run(nil, make(chan struct{})) } @@ -223,7 +229,8 @@ func CheckDeactivationPin(pin int64, stopCh chan struct{}) int { // run runs ctrld cli with given app callback and stop channel. func run(appCallback *AppCallback, stopCh chan struct{}) { if stopCh == nil { - mainLog.Load().Fatal().Msg("stopCh is nil") + failRunUnclassified(mainLog.Load().Error(), "run() called with a nil stop channel", nil) + return } waitCh := make(chan struct{}) p := &prog{ @@ -261,14 +268,10 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { } else { mainLog.Load().Warn().Err(err).Msgf("unable to resolve socket address: %s", sockPath) } - notifyExitToLogServer := func() { - if p.logConn != nil { - _, _ = p.logConn.Write([]byte(msgExit)) - } - } if daemon && runtime.GOOS == "windows" { - mainLog.Load().Fatal().Msg("Cannot run in daemon mode. Please install a Windows service.") + failRunUnclassified(mainLog.Load().Error(), "cannot run in daemon mode; please install a Windows service", p.notifyExitToLogServer) + return } if !daemon { @@ -277,7 +280,8 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { go func() { s, err := newService(p, svcConfig) if err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed create new service") + failRunUnclassified(mainLog.Load().Error().Err(err), fmt.Sprintf("failed to create new service: %v", err), p.notifyExitToLogServer) + return } if err := s.Run(); err != nil { mainLog.Load().Error().Err(err).Msg("failed to start service") @@ -288,18 +292,20 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { tryReadingConfig(writeDefaultConfig) if err := readBase64Config(configBase64); err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to read base64 config") + failRunUnclassified(mainLog.Load().Error().Err(err), fmt.Sprintf("failed to read base64 config: %v", err), p.notifyExitToLogServer) + return } processNoConfigFlags(noConfigStart) // After s.Run() was called, if ctrld is going to be terminated for any reason, // write msgExit to p.logConn so others (like "ctrld start") won't have to wait for timeout. p.mu.Lock() - if err := v.Unmarshal(&cfg); err != nil { - notifyExitToLogServer() - mainLog.Load().Fatal().Msgf("failed to unmarshal config: %v", err) - } + unmarshalErr := v.Unmarshal(&cfg) p.mu.Unlock() + if unmarshalErr != nil { + failRunUnclassified(mainLog.Load().Error(), fmt.Sprintf("failed to unmarshal config: %v", unmarshalErr), p.notifyExitToLogServer) + return + } processLogAndCacheFlags(v, &cfg) @@ -312,8 +318,8 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { // Wait for network up. if !ctrldnet.Up() { - notifyExitToLogServer() - mainLog.Load().Fatal().Msg("network is not up yet") + failRunUnclassified(mainLog.Load().Error(), "network is not up yet", p.notifyExitToLogServer) + return } p.router = router.New(&cfg, cdUID != "") @@ -327,8 +333,8 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { // time for validating server certificate. Some routers need NTP synchronization // to set the current time, so this check must happen before processCDFlags. if err := p.router.PreRun(); err != nil { - notifyExitToLogServer() - mainLog.Load().Fatal().Err(err).Msg("failed to perform router pre-run check") + failRunUnclassified(mainLog.Load().Error().Err(err), fmt.Sprintf("failed to perform router pre-run check: %v", err), p.notifyExitToLogServer) + return } oldLogPath := cfg.Service.LogPath @@ -336,7 +342,9 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { cdUID = uid } if cdUID != "" { - validateCdUpstreamProtocol() + if !validateCdUpstreamProtocol(p.notifyExitToLogServer) { + return + } // Bound API preflight by the service lifetime. Without this, a stop request // arriving while the API is unreachable leaves this retry/backoff loop running // after "service stopped" was logged, so the process keeps working on behalf of @@ -353,7 +361,7 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { // not see a failed start and apply its restart policy to a service the // operator just asked to stop. mainLog.Load().Notice().Msg("stop requested while fetching resolver config, shutting down") - notifyExitToLogServer() + p.notifyExitToLogServer() return case pf.err != nil: if isMobile() { @@ -361,7 +369,7 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { return } - handleAPIPreflightFailure(p, pf.err, notifyExitToLogServer) + handleAPIPreflightFailure(p, pf.err, p.notifyExitToLogServer) return default: p.mu.Lock() @@ -370,7 +378,7 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { } } - updated := updateListenerConfig(&cfg, notifyExitToLogServer) + updated := updateListenerConfig(&cfg, p.notifyExitToLogServer) // Bootstrap and listener binding both succeeded, so an earlier run's // recorded failure no longer describes this install. @@ -390,11 +398,10 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { if updated { if err := writeConfigFile(&cfg); err != nil { - notifyExitToLogServer() - mainLog.Load().Fatal().Err(err).Msg("failed to write config file") - } else { - mainLog.Load().Info().Msg("writing config file to: " + defaultConfigFile) + failRunUnclassified(mainLog.Load().Error().Err(err), fmt.Sprintf("failed to write config file: %v", err), p.notifyExitToLogServer) + return } + mainLog.Load().Info().Msg("writing config file to: " + defaultConfigFile) } if newLogPath := cfg.Service.LogPath; newLogPath != "" && oldLogPath != newLogPath { @@ -412,31 +419,28 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { } if err := validateConfig(&cfg); err != nil { - notifyExitToLogServer() - os.Exit(1) + failRunUnclassified(mainLog.Load().Error().Err(err), fmt.Sprintf("invalid config: %v", err), p.notifyExitToLogServer) + return } initCache() if daemon { exe, err := os.Executable() if err != nil { - mainLog.Load().Error().Err(err).Msg("failed to find the binary") - notifyExitToLogServer() - os.Exit(1) + failRunUnclassified(mainLog.Load().Error().Err(err), "failed to find the binary", p.notifyExitToLogServer) + return } curDir, err := os.Getwd() if err != nil { - mainLog.Load().Error().Err(err).Msg("failed to get current working directory") - notifyExitToLogServer() - os.Exit(1) + failRunUnclassified(mainLog.Load().Error().Err(err), "failed to get current working directory", p.notifyExitToLogServer) + return } // If running as daemon, re-run the command in background, with daemon off. cmd := exec.Command(exe, append(os.Args[1:], "-d=false")...) cmd.Dir = curDir if err := cmd.Start(); err != nil { - mainLog.Load().Error().Err(err).Msg("failed to start process as daemon") - notifyExitToLogServer() - os.Exit(1) + failRunUnclassified(mainLog.Load().Error().Err(err), "failed to start process as daemon", p.notifyExitToLogServer) + return } mainLog.Load().Info().Int("pid", cmd.Process.Pid).Msg("DNS proxy started") os.Exit(0) @@ -741,9 +745,10 @@ func permanentAPIRejection(err error) (*controld.ErrorResponse, bool) { } // apiFailureCode maps a bootstrap preflight error to its provisioning code. -// A deleted device gets its own code because it triggers self-uninstall; -// other permanent rejections are generic; anything else counts as -// reachability trouble worth retrying. +// A deleted device gets its own code because it triggers self-uninstall; a +// permanent rejection with a known token reason gets its own code; other +// permanent rejections are generic; anything else counts as reachability +// trouble worth retrying. func apiFailureCode(err error) (provisionFailureCode, bool) { if err == nil { return "", false @@ -752,12 +757,62 @@ func apiFailureCode(err error) (provisionFailureCode, bool) { if errors.As(err, &uer) && uer.ErrorField.Code == controld.InvalidConfigCode { return provisionCodeAPIDeviceInvalid, true } - if _, ok := permanentAPIRejection(err); ok { + if rejection, ok := permanentAPIRejection(err); ok { + if code, ok := tokenFailureCodeForReason(rejection.ErrorField.Metadata.Reason); ok { + return code, true + } return provisionCodeAPIRejected, true } return provisionCodeAPIUnreachable, true } +// tokenFailureCodeForReason maps error.metadata.reason on a provisioning-token +// rejection to its failure code. An empty or unrecognized reason is not +// classified here; the caller falls back to the generic rejection code. +func tokenFailureCodeForReason(reason string) (provisionFailureCode, bool) { + switch reason { + case controld.ReasonTokenInvalid: + return provisionCodeTokenInvalid, true + case controld.ReasonTokenExpired: + return provisionCodeTokenExpired, true + case controld.ReasonTokenLimitReached: + return provisionCodeTokenLimitReached, true + case controld.ReasonTokenDisabled: + return provisionCodeTokenDisabled, true + default: + return "", false + } +} + +// tokenRejectionMessage returns the human message for a TOKEN_* code: what is +// wrong with the provisioning code and one concrete next action. Empty for +// any other code, so callers know to fall back to a generic message. Never +// includes the token itself. +func tokenRejectionMessage(code provisionFailureCode) string { + switch code { + case provisionCodeTokenInvalid: + return "the provisioning code is not valid; check the code and re-enter it exactly as given" + case provisionCodeTokenExpired: + return "the provisioning code has expired; get a new provisioning code from your administrator" + case provisionCodeTokenLimitReached: + return "the provisioning code reached its device limit; free up a device slot or use a different code" + case provisionCodeTokenDisabled: + return "the provisioning code was invalidated; download a profile from an active provisioning code" + default: + return "" + } +} + +// provisionTokenFailureMessage returns the result-file message for a failed +// provision-token exchange: a token-specific message when the API gave a +// known reason, the generic exchange-failure summary otherwise. +func provisionTokenFailureMessage(code provisionFailureCode, err error) string { + if msg := tokenRejectionMessage(code); msg != "" { + return msg + } + return fmt.Sprintf("provision token exchange failed: %v", err) +} + // apiRejectionSummary reports the HTTP status only. The API's raw error body // can echo back the value the caller sent, so it stays out of the artifact. func apiRejectionSummary(statusCode int) string { @@ -1215,7 +1270,15 @@ func selfCheckResolveDomain(ctx context.Context, addr, scope string, domain stri return errSelfCheckNoAnswer } -func userHomeDir() (string, error) { +// unixServiceHomeDir is where a ctrld service with root rights keeps its +// files on macOS, Linux, and the BSDs. +const unixServiceHomeDir = "/etc/controld" + +// serviceHomeDir returns the directory where a ctrld service with root or +// administrator rights keeps its files. It has no fallback to the home +// directory of the current user. A caller without root that only reads can +// thus look where the service wrote. +func serviceHomeDir() (string, error) { dir, err := router.HomeDir() if err != nil { return "", err @@ -1237,7 +1300,17 @@ func userHomeDir() (string, error) { if isMobile() { return homedir, nil } - dir = "/etc/controld" + return unixServiceHomeDir, nil +} + +// userHomeDir is serviceHomeDir with a fallback: when the current user +// cannot write to the unix service home, it returns the home directory of +// that user instead. +func userHomeDir() (string, error) { + dir, err := serviceHomeDir() + if err != nil || dir != unixServiceHomeDir { + return dir, err + } if err := os.MkdirAll(dir, 0750); err != nil { return os.UserHomeDir() // fallback to user home directory } @@ -1898,6 +1971,84 @@ func osVersion() string { return oi.String() } +// provisionTokenMinLen, provisionTokenMaxLen, and provisionTokenExpectedPrefix +// describe the shape of a --cd-org value the input stage checks before any +// network call. +const ( + provisionTokenMinLen = 6 + provisionTokenMaxLen = 64 + provisionTokenExpectedPrefix = "org-v1-" +) + +// provisionTokenShapeValid reports whether token looks like something the API +// could parse: a length between provisionTokenMinLen and provisionTokenMaxLen, +// with no whitespace or control characters. It does not require the +// "org-v1-" prefix - legacy provisioning codes predate that convention and +// are handled with a warning, not a failure. +func provisionTokenShapeValid(token string) bool { + if len(token) < provisionTokenMinLen || len(token) > provisionTokenMaxLen { + return false + } + for _, r := range token { + if unicode.IsSpace(r) || unicode.IsControl(r) { + return false + } + } + return true +} + +// validateProvisionTokenShape classifies an obviously malformed --cd-org +// value (PROVISION_TOKEN_MALFORMED) before any network call, so a typo or a +// pasted URL fails fast instead of going through a doomed API round trip. A +// token missing the "org-v1-" prefix only gets a logged warning: legacy +// provisioning codes may lack it and still work. +func validateProvisionTokenShape() bool { + if !provisionTokenShapeValid(cdOrg) { + msg := fmt.Sprintf("--cd-org provisioning code is malformed: must be %d-%d characters with no whitespace or control characters", provisionTokenMinLen, provisionTokenMaxLen) + failProvision(newProvisionResult(provisionCodeProvisionTokenMalformed, msg, nil, provisionSecrets()...), nil) + return false + } + if !strings.HasPrefix(cdOrg, provisionTokenExpectedPrefix) { + mainLog.Load().Warn().Msg("--cd-org provisioning code does not have the expected org-v1- prefix; continuing, since legacy codes may lack it") + } + return true +} + +// validateCustomHostnameFlag classifies an invalid --custom-hostname value +// (CUSTOM_HOSTNAME_INVALID) before any network call. ctrld's accept/reject +// rule (validHostname) is unchanged - this only names the field, the +// offending character(s), and the allowed format instead of a bare fatal +// exit. ControlD's device-name formatting folds or strips some characters +// ctrld still accepts (dot, space, plus), so a hostname using one of those +// gets a one-line notice instead: the registered device name may differ from +// what was requested. +func validateCustomHostnameFlag() bool { + if customHostname == "" { + return true + } + if !validHostname(customHostname) { + failProvision(newProvisionResult(provisionCodeCustomHostnameInvalid, customHostnameFailureMessage(customHostname), nil, provisionSecrets()...), nil) + return false + } + if hostnameMayBeFoldedByServer(customHostname) { + mainLog.Load().Notice().Msgf("device name %q contains characters ControlD may fold or strip when it registers the device; the registered name may differ", customHostname) + } + return true +} + +// validateInterceptModeFlag classifies an invalid --intercept-mode value +// (INTERCEPT_MODE_INVALID) on the provisioning boundary, so a typo fails with +// a stable code instead of a bare fatal exit an installer wrapper cannot tell +// apart from any other crash. An empty mode means the flag was not set. +func validateInterceptModeFlag(mode string) bool { + if mode == "" || validInterceptMode(mode) { + return true + } + msg := fmt.Sprintf("--intercept-mode %q is not valid: must be 'off', 'dns', or 'hard'", mode) + failProvision(newProvisionResult(provisionCodeInterceptModeInvalid, msg, nil, provisionSecrets()...), nil) + return false +} + // cdUIDFromProvToken fetch UID from ControlD API using provision token. func cdUIDFromProvToken() string { // --cd flag supersedes --cd-org, ignore it if both are supplied. @@ -1908,9 +2059,11 @@ func cdUIDFromProvToken() string { if cdOrg == "" { return "" } - // Validate custom hostname if provided. - if customHostname != "" && !validHostname(customHostname) { - mainLog.Load().Fatal().Msgf("invalid custom hostname: %q", customHostname) + if !validateProvisionTokenShape() { + return "" + } + if !validateCustomHostnameFlag() { + return "" } req := &controld.UtilityOrgRequest{ @@ -1926,7 +2079,7 @@ func cdUIDFromProvToken() string { code, _ := apiFailureCode(err) mainLog.Load().Error().Msgf("failed to fetch resolver uid with provision token: %s: %s", redactToken(cdOrg), redactSecrets(err.Error(), provisionSecrets()...)) - failProvision(newProvisionResult(code, fmt.Sprintf("provision token exchange failed: %v", err), nil, provisionSecrets()...), nil) + failProvision(newProvisionResult(code, provisionTokenFailureMessage(code, err), nil, provisionSecrets()...), nil) return "" } return resolverConfig.UID @@ -2032,32 +2185,55 @@ func newSocketControlClientMobile(dir string, stopCh chan struct{}) *controlClie } // checkStrFlagEmpty validates if a string flag was set to an empty string. -// If yes, emitting a fatal error message. -func checkStrFlagEmpty(cmd *cobra.Command, flagName string) { +// An explicit empty --cd-org is a malformed provisioning token, classified on +// the provisioning boundary; every other flag keeps the bare fatal, since +// this helper is shared with flags outside that boundary. Returns false when +// the caller must stop: provisionExit is stubbed out under test, so nothing +// else stops execution from falling through. +func checkStrFlagEmpty(cmd *cobra.Command, flagName string) bool { fl := cmd.Flags().Lookup(flagName) - if !fl.Changed || fl.Value.Type() != "string" { - return + if !fl.Changed || fl.Value.Type() != "string" || fl.Value.String() != "" { + return true } - if fl.Value.String() == "" { - mainLog.Load().Fatal().Msgf(`flag "--%s" value must be non-empty`, fl.Name) + if flagName == cdOrgFlagName { + msg := fmt.Sprintf("--%s provisioning code is malformed: value must be non-empty", cdOrgFlagName) + failProvision(newProvisionResult(provisionCodeProvisionTokenMalformed, msg, nil, provisionSecrets()...), nil) + return false } + mainLog.Load().Fatal().Msgf(`flag "--%s" value must be non-empty`, fl.Name) + return false } -func validateCdUpstreamProtocol() { +// validateCdUpstreamProtocol validates the Control D upstream protocol, +// classified on the provisioning boundary since it only runs once --cd is +// set. notify unblocks a waiting "ctrld start" on the daemon side; the parent +// process, which has no one waiting, passes nil. Returns false when the +// caller must stop. +func validateCdUpstreamProtocol(notify func()) bool { if cdUID == "" { - return + return true } switch cdUpstreamProto { case ctrld.ResolverTypeDOH, ctrld.ResolverTypeDOH3: + return true default: - mainLog.Load().Fatal().Msg(`flag "--protocol" must be "doh" or "doh3"`) + msg := fmt.Sprintf("--proto %q is not valid: must be 'doh' or 'doh3'", cdUpstreamProto) + failProvision(newProvisionResult(provisionCodeInvalidFlagCombination, msg, nil, provisionSecrets()...), notify) + return false } } -func validateCdAndNextDNSFlags() { +// validateCdAndNextDNSFlags validates that Control D and NextDNS flags are +// not used together, classified on the provisioning boundary since it only +// fires once --cd or --cd-org is set. Returns false when the caller must +// stop. +func validateCdAndNextDNSFlags() bool { if (cdUID != "" || cdOrg != "") && nextdns != "" { - mainLog.Load().Fatal().Msgf("--%s/--%s could not be used with --%s", cdUidFlagName, cdOrgFlagName, nextdnsFlagName) + msg := fmt.Sprintf("--%s/--%s cannot be used together with --%s", cdUidFlagName, cdOrgFlagName, nextdnsFlagName) + failProvision(newProvisionResult(provisionCodeInvalidFlagCombination, msg, nil, provisionSecrets()...), nil) + return false } + return true } // removeNextDNSFromArgs removes the --nextdns from command line arguments. @@ -2274,7 +2450,25 @@ func runningIface(s service.Service) *ifaceResponse { return nil } +// apiRejectionMessage returns the result-file message for a permanent API +// rejection: the HTTP status for a device-invalid or generic rejection, or +// the raw error for anything else (network trouble, an unreachable API). +func apiRejectionMessage(err error) string { + var uer *controld.ErrorResponse + if errors.As(err, &uer) && uer.ErrorField.Code == controld.InvalidConfigCode { + return apiRejectionSummary(uer.StatusCode) + } + if rejection, ok := permanentAPIRejection(err); ok { + return apiRejectionSummary(rejection.StatusCode) + } + return fmt.Sprintf("failed to fetch resolver config: %v", err) +} + // doValidateCdRemoteConfig fetches and validates custom config for cdUID. +// fatal distinguishes the two callers: a direct install-time caller passes +// true and classifies a fetch failure on the provisioning boundary; the +// restart path passes false and gets the error back to decide for itself, +// with no process exit. func doValidateCdRemoteConfig(cdUID string, fatal bool) error { // Username is only sent during initial provisioning (cdUIDFromProvToken). // All subsequent calls use lightweight metadata to avoid EDR triggers. @@ -2283,16 +2477,16 @@ func doValidateCdRemoteConfig(cdUID string, fatal bool) error { Version: rootCmd.Version, Metadata: ctrld.SystemMetadataRuntime(context.Background()), } - rc, err := controld.FetchResolverConfig(context.Background(), req, cdDev) + rc, err := fetchResolverConfig(context.Background(), req, cdDev) if err != nil { - logger := mainLog.Load().Fatal() - if !fatal { - logger = mainLog.Load().Warn() - } - logger.Err(err).Err(err).Msgf("failed to fetch resolver uid: %s", cdUID) if !fatal { + mainLog.Load().Warn().Err(err).Msgf("failed to fetch resolver config for %s", redactToken(cdUID)) return err } + code, _ := apiFailureCode(err) + mainLog.Load().Error().Err(err).Msgf("failed to fetch resolver config for %s", redactToken(cdUID)) + failProvision(newProvisionResult(code, apiRejectionMessage(err), nil, provisionSecrets()...), nil) + return err } // return earlier if there's no custom config. diff --git a/cmd/cli/cli_provision_test.go b/cmd/cli/cli_provision_test.go index 34628da5..0eeae639 100644 --- a/cmd/cli/cli_provision_test.go +++ b/cmd/cli/cli_provision_test.go @@ -2,6 +2,8 @@ package cli import ( "context" + "encoding/json" + "errors" "fmt" "net" "net/http" @@ -73,6 +75,140 @@ func TestApiFailureCode(t *testing.T) { } } +// TestApiFailureCodeMapsRejectionReason covers the token_* reasons the API sends in +// error.metadata.reason on a provisioning-token rejection. An absent or unknown +// reason must fall back to the generic API_REJECTED exactly as before this reason +// field existed. +func TestApiFailureCodeMapsRejectionReason(t *testing.T) { + rejectionWithReason := func(reason string) error { + e := &controld.ErrorResponse{StatusCode: http.StatusBadRequest} + e.ErrorField.Code = 40003 + e.ErrorField.Message = "invalid token" + e.ErrorField.Metadata.Reason = reason + return e + } + + tests := []struct { + name string + reason string + wantCode provisionFailureCode + }{ + {name: "token_invalid", reason: "token_invalid", wantCode: provisionCodeTokenInvalid}, + {name: "token_expired", reason: "token_expired", wantCode: provisionCodeTokenExpired}, + {name: "token_limit_reached", reason: "token_limit_reached", wantCode: provisionCodeTokenLimitReached}, + {name: "token_disabled", reason: "token_disabled", wantCode: provisionCodeTokenDisabled}, + {name: "reason absent falls back", reason: "", wantCode: provisionCodeAPIRejected}, + {name: "unknown reason falls back", reason: "some_future_reason", wantCode: provisionCodeAPIRejected}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + code, ok := apiFailureCode(rejectionWithReason(tc.reason)) + if !ok { + t.Fatal("apiFailureCode() ok = false, want true") + } + if code != tc.wantCode { + t.Errorf("apiFailureCode() code = %s, want %s", code, tc.wantCode) + } + }) + } +} + +// TestApiFailureCodeSurvivesMalformedReasonType covers a rejection body whose +// metadata.reason is the wrong JSON type end to end: decode it exactly as +// internal/controld does (json.Unmarshal into the same exported type), then +// classify it. Before the metadata decode fix, this body failed the whole +// decode and apiFailureCode never saw an *ErrorResponse at all, so it fell +// back to API_UNREACHABLE - the retryable bootstrap code - instead of the +// permanent rejection this HTTP 400 with a known error code actually is. +func TestApiFailureCodeSurvivesMalformedReasonType(t *testing.T) { + tests := []struct { + name string + body string + }{ + {name: "reason as a number", body: `{"error":{"message":"invalid token","code":40003,"metadata":{"reason":12345}}}`}, + {name: "reason as an object", body: `{"error":{"message":"invalid token","code":40003,"metadata":{"reason":{"inner":"value"}}}}`}, + {name: "reason as null", body: `{"error":{"message":"invalid token","code":40003,"metadata":{"reason":null}}}`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + e := &controld.ErrorResponse{StatusCode: http.StatusBadRequest} + if err := json.Unmarshal([]byte(tc.body), e); err != nil { + t.Fatalf("a malformed reason must not fail the whole decode: %v", err) + } + code, ok := apiFailureCode(e) + if !ok { + t.Fatal("apiFailureCode() ok = false, want true") + } + if code != provisionCodeAPIRejected { + t.Errorf("apiFailureCode() code = %s, want %s (not %s)", code, provisionCodeAPIRejected, provisionCodeAPIUnreachable) + } + }) + } +} + +// TestCdUIDFromProvTokenReasonCodes covers the full path from an API rejection +// reason to a persisted result file: each known reason gets its own code, exit +// code, and stage, with a message naming the field and a next action but never +// echoing the token. Absent and unknown reasons keep the generic rejection. +func TestCdUIDFromProvTokenReasonCodes(t *testing.T) { + const secretToken = "org-secret-token-999" + + tests := []struct { + name string + reason string + wantCode provisionFailureCode + wantContains string + }{ + {name: "token_invalid", reason: "token_invalid", wantCode: provisionCodeTokenInvalid, wantContains: "provisioning code"}, + {name: "token_expired", reason: "token_expired", wantCode: provisionCodeTokenExpired, wantContains: "expired"}, + {name: "token_limit_reached", reason: "token_limit_reached", wantCode: provisionCodeTokenLimitReached, wantContains: "limit"}, + {name: "token_disabled", reason: "token_disabled", wantCode: provisionCodeTokenDisabled, wantContains: "invalidated"}, + {name: "reason absent", reason: "", wantCode: provisionCodeAPIRejected, wantContains: ""}, + {name: "unknown reason", reason: "brand_new_reason", wantCode: provisionCodeAPIRejected, wantContains: ""}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldFetch, oldHostname := fetchResolverUIDFn, customHostname + t.Cleanup(func() { fetchResolverUIDFn, customHostname = oldFetch, oldHostname }) + cdUID = "" + cdOrg = secretToken + customHostname = "" + + rejected := &controld.ErrorResponse{StatusCode: http.StatusBadRequest} + rejected.ErrorField.Code = 40003 + rejected.ErrorField.Message = "invalid token " + secretToken + rejected.ErrorField.Metadata.Reason = tc.reason + fetchResolverUIDFn = func(context.Context, *controld.UtilityOrgRequest, string, bool) (*controld.ResolverConfig, error) { + return nil, rejected + } + + if got := cdUIDFromProvToken(); got != "" { + t.Errorf("cdUIDFromProvToken() = %q, want empty on failure", got) + } + if *exitCode != provisionExitCodeForCode[tc.wantCode] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[tc.wantCode]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(tc.wantCode) { + t.Errorf("code = %q, want %q", r.Code, tc.wantCode) + } + if r.Stage != string(provisionStageBootstrap) { + t.Errorf("stage = %q, want bootstrap", r.Stage) + } + if tc.wantContains != "" && !strings.Contains(r.Message, tc.wantContains) { + t.Errorf("message = %q, want it to contain %q", r.Message, tc.wantContains) + } + if strings.Contains(r.Message, secretToken) { + t.Errorf("token leaked into result message: %q", r.Message) + } + }) + } +} + func stubProvisionGlobals(t *testing.T) (exitCode *int, notified *bool) { t.Helper() oldCdUID, oldCdOrg := cdUID, cdOrg @@ -179,6 +315,109 @@ func TestHandleAPIPreflightFailure(t *testing.T) { }) } +// TestDoValidateCdRemoteConfigClassifiesAPIFailure covers the fatal=true +// caller: a fetch failure must classify on the provisioning boundary with +// the same per-class codes as the daemon-side preflight, instead of a bare +// fatal. +func TestDoValidateCdRemoteConfigClassifiesAPIFailure(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldFetch := fetchResolverConfig + t.Cleanup(func() { fetchResolverConfig = oldFetch }) + + deviceInvalid := func() error { + e := &controld.ErrorResponse{StatusCode: http.StatusNotFound} + e.ErrorField.Code = controld.InvalidConfigCode + e.ErrorField.Message = "device does not exist" + return e + } + rejected := func() error { + e := &controld.ErrorResponse{StatusCode: http.StatusUnauthorized} + e.ErrorField.Message = "bad token" + return e + } + + tests := []struct { + name string + err error + wantCode provisionFailureCode + }{ + {name: "device invalid", err: deviceInvalid(), wantCode: provisionCodeAPIDeviceInvalid}, + {name: "permanent rejection", err: rejected(), wantCode: provisionCodeAPIRejected}, + {name: "unreachable", err: retryableNetworkErr(), wantCode: provisionCodeAPIUnreachable}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fetchResolverConfig = func(context.Context, *controld.ResolverConfigRequest, bool) (*controld.ResolverConfig, error) { + return nil, tc.err + } + if err := doValidateCdRemoteConfig("device-uid-123", true); err == nil { + t.Error("doValidateCdRemoteConfig() error = nil, want the fetch error back") + } + if *exitCode != provisionExitCodeForCode[tc.wantCode] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[tc.wantCode]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(tc.wantCode) { + t.Errorf("code = %q, want %q", r.Code, tc.wantCode) + } + }) + } +} + +// TestDoValidateCdRemoteConfigNonFatalReturnsError proves the restart path +// (fatal=false) is unaffected: it still just warns and hands the error back, +// with no process exit and no result file. +func TestDoValidateCdRemoteConfigNonFatalReturnsError(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldFetch := fetchResolverConfig + t.Cleanup(func() { fetchResolverConfig = oldFetch }) + + wantErr := errors.New("network unreachable") + fetchResolverConfig = func(context.Context, *controld.ResolverConfigRequest, bool) (*controld.ResolverConfig, error) { + return nil, wantErr + } + + if err := doValidateCdRemoteConfig("device-uid-123", false); !errors.Is(err, wantErr) { + t.Errorf("doValidateCdRemoteConfig() error = %v, want %v", err, wantErr) + } + if *exitCode != -1 { + t.Errorf("provisionExit called with %d, want no exit", *exitCode) + } + if _, err := readProvisionResult(); err == nil { + t.Error("expected no provision result written for the non-fatal path") + } +} + +// TestDoValidateCdRemoteConfigDoesNotSelfUninstall proves a device-invalid +// failure on this path never triggers self-uninstall: unlike the daemon-side +// bootstrap preflight, callers passing fatal=true run before any service is +// installed, so there is nothing to remove. +func TestDoValidateCdRemoteConfigDoesNotSelfUninstall(t *testing.T) { + _, _ = stubProvisionGlobals(t) + oldFetch, oldUninstall := fetchResolverConfig, uninstallInvalidCdUIDFn + t.Cleanup(func() { fetchResolverConfig, uninstallInvalidCdUIDFn = oldFetch, oldUninstall }) + + uninstallCalled := false + uninstallInvalidCdUIDFn = func(*prog, zerolog.Logger, bool) bool { + uninstallCalled = true + return true + } + e := &controld.ErrorResponse{StatusCode: http.StatusNotFound} + e.ErrorField.Code = controld.InvalidConfigCode + fetchResolverConfig = func(context.Context, *controld.ResolverConfigRequest, bool) (*controld.ResolverConfig, error) { + return nil, e + } + + _ = doValidateCdRemoteConfig("device-uid-123", true) + + if uninstallCalled { + t.Error("doValidateCdRemoteConfig triggered self-uninstall; nothing is installed yet on this path") + } +} + func TestCdUIDFromProvTokenFailureEmitsCode(t *testing.T) { exitCode, _ := stubProvisionGlobals(t) oldFetch, oldHostname := fetchResolverUIDFn, customHostname diff --git a/cmd/cli/commands.go b/cmd/cli/commands.go index 6ebf5c71..9b057b44 100644 --- a/cmd/cli/commands.go +++ b/cmd/cli/commands.go @@ -342,9 +342,20 @@ NOTE: running "ctrld start" without any arguments will start already installed c return nil }, Run: func(cmd *cobra.Command, args []string) { - checkStrFlagEmpty(cmd, cdUidFlagName) - checkStrFlagEmpty(cmd, cdOrgFlagName) - validateCdAndNextDNSFlags() + // Clear before any check runs, not just before doTasksE: a result from a + // previous attempt must never survive to mislead diag/postinstall on this + // one, even if this attempt fails before reaching doTasksE. + clearProvisionResult() + + if !checkStrFlagEmpty(cmd, cdUidFlagName) { + return + } + if !checkStrFlagEmpty(cmd, cdOrgFlagName) { + return + } + if !validateCdAndNextDNSFlags() { + return + } sc := &service.Config{} *sc = *svcConfig osArgs := os.Args[2:] @@ -359,8 +370,8 @@ NOTE: running "ctrld start" without any arguments will start already installed c // Without this, a typo like "--intercept-mode fds" would install the service, // the child process would Fatal() on the invalid value, and the parent would // then uninstall — confusing and destructive. - if interceptMode != "" && !validInterceptMode(interceptMode) { - mainLog.Load().Fatal().Msgf("invalid --intercept-mode value %q: must be 'off', 'dns', or 'hard'", interceptMode) + if !validateInterceptModeFlag(interceptMode) { + return } p := &prog{ @@ -370,6 +381,9 @@ NOTE: running "ctrld start" without any arguments will start already installed c s, err := newService(p, sc) if err != nil { mainLog.Load().Error().Msg(err.Error()) + // A bare return would exit 0 with no result file, so support + // could not tell this failure from a start that never ran. + failProvisionUnclassified("initialize service: "+err.Error(), nil) return } p.preRun() @@ -397,7 +411,8 @@ NOTE: running "ctrld start" without any arguments will start already installed c // An explicit "off" argument must override a previously persisted config // value while the service clears that value on startup. if err := removeServiceFlag("--intercept-mode"); err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to remove existing intercept mode from service arguments") + failRunUnclassified(mainLog.Load().Error().Err(err), fmt.Sprintf("failed to remove existing intercept mode from service arguments: %v", err), nil) + return } if interceptMode == "off" { @@ -406,10 +421,12 @@ NOTE: running "ctrld start" without any arguments will start already installed c mainLog.Load().Notice().Msgf("Existing service detected — appending --intercept-mode %s to service arguments", interceptMode) } if err := appendServiceFlag("--intercept-mode"); err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to append intercept flag to service arguments") + failRunUnclassified(mainLog.Load().Error().Err(err), fmt.Sprintf("failed to append intercept flag to service arguments: %v", err), nil) + return } if err := appendServiceFlag(interceptMode); err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to append intercept mode value to service arguments") + failRunUnclassified(mainLog.Load().Error().Err(err), fmt.Sprintf("failed to append intercept mode value to service arguments: %v", err), nil) + return } // Stop the service if running (bypasses ctrld pin — this is an @@ -538,7 +555,8 @@ NOTE: running "ctrld start" without any arguments will start already installed c if startOnly && isCtrldInstalled { tryReadingConfigWithNotice(false, true) if err := v.Unmarshal(&cfg); err != nil { - mainLog.Load().Fatal().Msgf("failed to unmarshal config: %v", err) + failRunUnclassified(mainLog.Load().Error(), fmt.Sprintf("failed to unmarshal config: %v", err), nil) + return } // if already running, dont restart @@ -565,8 +583,6 @@ NOTE: running "ctrld start" without any arguments will start already installed c {s.Start, true, "Start"}, {noticeWritingControlDConfig, false, "Notice writing ControlD config"}, } - // Any result found later must come from this attempt, not a stale run. - clearProvisionResult() startAttemptAt := time.Now() mainLog.Load().Notice().Msg("Starting existing ctrld service") failedTask, taskErr := doTasksE(tasks) @@ -575,12 +591,15 @@ NOTE: running "ctrld start" without any arguments will start already installed c failProvision(newProvisionResult(code, serviceTaskErrorSummary(failedTask, taskErr), nil, provisionSecrets()...), nil) return } - os.Exit(1) + // Not a service-stage task. doTasksE already logged the cause; classify + // UNCLASSIFIED instead of the old silent fall-through that exited 1. + failProvisionUnclassified(serviceTaskErrorSummary(failedTask, taskErr), nil) + return } sockDir, err := socketDir() if err != nil { - mainLog.Load().Warn().Err(err).Msg("Failed to get socket directory") - os.Exit(1) + failRunUnclassified(mainLog.Load().Error(), fmt.Sprintf("failed to get socket directory: %v", err), nil) + return } // The daemon can start and still fail provisioning (for example a @@ -623,11 +642,14 @@ NOTE: running "ctrld start" without any arguments will start already installed c sc.Arguments = append(sc.Arguments, "--cd="+cdUID) } if cdUID != "" { - validateCdUpstreamProtocol() + if !validateCdUpstreamProtocol(nil) { + return + } } if err := p.router.ConfigureService(sc); err != nil { - mainLog.Load().Fatal().Err(err).Msg("failed to configure service on router") + failRunUnclassified(mainLog.Load().Error().Err(err), fmt.Sprintf("failed to configure service on router: %v", err), nil) + return } if configPath != "" { @@ -637,7 +659,8 @@ NOTE: running "ctrld start" without any arguments will start already installed c tryReadingConfigWithNotice(writeDefaultConfig, true) if err := v.Unmarshal(&cfg); err != nil { - mainLog.Load().Fatal().Msgf("failed to unmarshal config: %v", err) + failRunUnclassified(mainLog.Load().Error(), fmt.Sprintf("failed to unmarshal config: %v", err), nil) + return } initInteractiveLogging() @@ -682,8 +705,6 @@ NOTE: running "ctrld start" without any arguments will start already installed c // generated after s.Start, so we notice users here for consistent with nextdns mode. {noticeWritingControlDConfig, false, "Notice writing ControlD config"}, } - // Any result found later must come from this attempt, not a stale run. - clearProvisionResult() startAttemptAt := time.Now() mainLog.Load().Notice().Msg("Starting service") failedTask, taskErr := doTasksE(tasks) @@ -692,9 +713,9 @@ NOTE: running "ctrld start" without any arguments will start already installed c failProvision(newProvisionResult(code, serviceTaskErrorSummary(failedTask, taskErr), nil, provisionSecrets()...), nil) return } - // Not a service-stage task. doTasksE already logged the cause; exit - // non-zero instead of the old silent fall-through that exited 0. - os.Exit(1) + // Not a service-stage task. doTasksE already logged the cause; classify + // UNCLASSIFIED instead of the old silent fall-through that exited 0. + failProvisionUnclassified(serviceTaskErrorSummary(failedTask, taskErr), nil) return } diff --git a/cmd/cli/commands_diag.go b/cmd/cli/commands_diag.go new file mode 100644 index 00000000..566bc5f2 --- /dev/null +++ b/cmd/cli/commands_diag.go @@ -0,0 +1,473 @@ +package cli + +import ( + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "time" + "unicode/utf8" + + "github.com/kardianos/service" + "github.com/spf13/cobra" + + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// diag reports the facts that support asks for on every provisioning +// ticket. A customer or admin can then paste the output of one command +// instead of a hunt through logs and preference panes. It runs without +// root. A section that needs data the current user cannot read reports +// that, instead of a failure of the whole command. On an installed device +// those are the provisioning-result and the service-state sections, because +// the service wrote the file as root and the service manager answers only +// root. +const diagCmdLong = `Collect diagnostics for a provisioning failure. + +Reports the client version, MDM-managed preferences (macOS only), the last +provisioning result, service state, and whether the Control D API is +reachable. Safe to paste into a support ticket: it never prints the +provisioning token itself, only whether one is present. + +Run it with root or administrator rights on an installed device to include +the last provisioning result and the service state. Without those rights, +the two sections report permission denied.` + +// diagAPIProbeTimeout bounds the API reachability check. diag must return +// promptly even when the network is unreachable. +const diagAPIProbeTimeout = 5 * time.Second + +// diagOverallTimeout bounds the whole report. No single probe, however +// wedged, may keep "ctrld diag" from returning. +const diagOverallTimeout = 15 * time.Second + +// diagServiceStateTimeout bounds one service-state probe. systemctl or +// launchctl can hang against a wedged service manager; the kardianos +// service package gives us no way to cancel that call, so we race it +// against this timer in a goroutine instead. +const diagServiceStateTimeout = 5 * time.Second + +// diagFieldMaxLen bounds any single field pulled from outside ctrld's own +// control (a managed-preferences value), so a misconfigured profile cannot +// blow up the report's size. +const diagFieldMaxLen = 256 + +// managedPrefsDomain is the MDM-managed preferences domain ctrld reads its +// provisioning settings from. +const managedPrefsDomain = "/Library/Managed Preferences/com.controld.ctrld" + +// managedPrefsBin is invoked with an absolute path so diag never depends on +// PATH. +const managedPrefsBin = "/usr/bin/defaults" + +// managedPrefsSupported reports whether this platform has managed +// preferences to read. A var so tests can exercise the macOS-shaped report +// on any OS. +var managedPrefsSupported = func() bool { return runtime.GOOS == "darwin" } + +// managedPrefsRead runs `defaults read [key]` and returns the +// trimmed value, or ok=false if the domain or key does not exist. A var so +// tests seam it instead of shelling out for real. It honors the caller's +// ctx so a near-expired overall deadline cuts this short too. +var managedPrefsRead = func(ctx context.Context, domain, key string) (string, bool) { + args := []string{"read", domain} + if key != "" { + args = append(args, key) + } + probeCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + defer cancel() + out, err := exec.CommandContext(probeCtx, managedPrefsBin, args...).Output() + if err != nil { + return "", false + } + return strings.TrimSpace(string(out)), true +} + +// diagProbeReachability makes the one HTTPS probe api_reachability reports +// on. A var so tests replace it instead of hitting the network. +var diagProbeReachability = controld.ProbeReachability + +type diagReport struct { + ClientVersion string `json:"client_version"` + Commit string `json:"commit"` + ManagedPreferences diagManagedPreferences `json:"managed_preferences"` + ProvisionResult diagProvisionResult `json:"provision_result"` + ServiceState diagServiceState `json:"service_state"` + APIReachability diagAPIReachability `json:"api_reachability"` +} + +type diagManagedPreferences struct { + Applicable bool `json:"applicable"` + ProfilePresent bool `json:"profile_present"` + ProvisionToken string `json:"provision_token"` // "present" or "absent"; never the value + CustomHostname string `json:"custom_hostname"` + InterceptMode string `json:"intercept_mode"` + Note string `json:"note"` +} + +// diagProvisionResult mirrors the on-disk provision_result.json for report +// purposes. AgeSeconds is -1 when no age applies (nothing recorded, or the +// timestamp did not parse). Message and Attempts are re-bounded on read: the +// file was written bounded, but diag must not trust that a file on disk +// still is (a mismatched version, or hand-edited). +type diagProvisionResult struct { + Status string `json:"status"` // none, recorded, untrusted, unreadable, corrupt + Stage string `json:"stage"` + Code string `json:"code"` + ExitCode int `json:"exit_code"` + Message string `json:"message"` + Attempts []provisionBindAttempt `json:"attempts,omitempty"` + AgeSeconds int64 `json:"age_seconds"` +} + +type diagServiceState struct { + Status string `json:"status"` // running, stopped, not_installed, unknown + Note string `json:"note"` +} + +type diagAPIReachability struct { + Reachable bool `json:"reachable"` + ErrorClass string `json:"error_class"` // empty when reachable +} + +// diagProvisionResultPath locates the result file where the service wrote +// it. The shared home-dir resolver has a fallback to the home directory of +// the current user when /etc/controld is not writable, and every run +// without root hits that fallback. A diag that used it would report "none +// recorded" for a file it never looked at. A var so tests can point it at +// a temp dir. +var diagProvisionResultPath = func() string { + if homedir != "" { + return filepath.Join(homedir, provisionResultFileName) + } + dir, err := serviceHomeDir() + if err != nil { + return provisionResultPath() + } + return filepath.Join(dir, provisionResultFileName) +} + +// diagServiceStateFn collects service_state. A var so tests can supply a +// fixed state instead of depending on whatever service happens to be +// installed on the machine running the tests. +var diagServiceStateFn = collectServiceStateReal + +// buildDiagReport bounds the whole report at diagOverallTimeout: whatever +// deadline the caller passed in, a single probe wedging past this must not +// keep "ctrld diag" from returning. +func buildDiagReport(ctx context.Context) diagReport { + ctx, cancel := context.WithTimeout(ctx, diagOverallTimeout) + defer cancel() + return diagReport{ + ClientVersion: curVersion(), + Commit: commit, + ManagedPreferences: collectManagedPreferences(ctx), + ProvisionResult: collectProvisionResultDiag(), + ServiceState: collectServiceStateBounded(ctx), + APIReachability: collectAPIReachability(ctx), + } +} + +func collectManagedPreferences(ctx context.Context) diagManagedPreferences { + if !managedPrefsSupported() { + return diagManagedPreferences{Note: "not applicable on this platform"} + } + m := diagManagedPreferences{Applicable: true} + if _, ok := managedPrefsRead(ctx, managedPrefsDomain, ""); !ok { + m.Note = "configuration profile not found" + return m + } + m.ProfilePresent = true + // An empty value reads as absent: the postinstall refuses to provision + // on an empty ProvisionToken, so "present" would send support the wrong + // way. + if v, ok := managedPrefsRead(ctx, managedPrefsDomain, "ProvisionToken"); ok && v != "" { + m.ProvisionToken = "present" + } else { + m.ProvisionToken = "absent" + } + if v, ok := managedPrefsRead(ctx, managedPrefsDomain, "CustomHostname"); ok { + m.CustomHostname = boundedDiagField(v) + } + if v, ok := managedPrefsRead(ctx, managedPrefsDomain, "InterceptMode"); ok { + m.InterceptMode = boundedDiagField(v) + } + return m +} + +// boundedDiagField caps a value from outside ctrld's control to a size that +// keeps the report bounded, cutting on a rune boundary so it never splits a +// multi-byte character. +func boundedDiagField(s string) string { + if utf8.RuneCountInString(s) <= diagFieldMaxLen { + return s + } + runes := []rune(s) + return string(runes[:diagFieldMaxLen]) +} + +// collectProvisionResultDiag reads provision_result.json from the service +// home through the same trusted reader "ctrld start" uses, so diag never +// reports a code that contract validation would reject. A read the current +// user is not permitted to make is its own status: the file is not +// corrupt, the reader lacks root. +func collectProvisionResultDiag() diagProvisionResult { + r, err := readProvisionResultAt(diagProvisionResultPath()) + if err != nil { + if os.IsNotExist(err) { + return diagProvisionResult{Status: "none", AgeSeconds: -1} + } + if os.IsPermission(err) { + return diagProvisionResult{Status: "unreadable", AgeSeconds: -1} + } + return diagProvisionResult{Status: "corrupt", AgeSeconds: -1} + } + if !provisionResultTrusted(r) { + return diagProvisionResult{Status: "untrusted", AgeSeconds: -1} + } + age := int64(-1) + if ts, parseErr := time.Parse(time.RFC3339, r.Timestamp); parseErr == nil { + if d := time.Since(ts); d >= 0 { + age = int64(d.Round(time.Second).Seconds()) + } else { + age = 0 + } + } + var attempts []provisionBindAttempt + if r.Detail != nil { + attempts = boundedDiagAttempts(r.Detail.Attempts) + } + return diagProvisionResult{ + Status: "recorded", + Stage: r.Stage, + Code: r.Code, + ExitCode: r.ExitCode, + Message: boundedDiagField(r.Message), + Attempts: attempts, + AgeSeconds: age, + } +} + +// boundedDiagAttempts re-applies the same attempt-count cap the file was +// written with, and bounds each attempt's fields, so a result file from a +// mismatched or tampered version cannot make the report unbounded. +func boundedDiagAttempts(attempts []provisionBindAttempt) []provisionBindAttempt { + if len(attempts) > maxProvisionBindAttempts { + attempts = attempts[:maxProvisionBindAttempts] + } + bounded := make([]provisionBindAttempt, len(attempts)) + for i, a := range attempts { + bounded[i] = provisionBindAttempt{ + Addr: boundedDiagField(a.Addr), + Proto: boundedDiagField(a.Proto), + OSError: boundedDiagField(a.OSError), + } + } + return bounded +} + +// collectServiceStateReal is diagServiceStateFn's production implementation. +// It reuses the same service-manager wrapper "ctrld status" does, so a +// permission-limited run reports the OS's own permission error rather than a +// wrong status. +func collectServiceStateReal() diagServiceState { + s, err := newService(&prog{}, svcConfig) + if err != nil { + return diagServiceState{Status: "unknown", Note: "could not set up the service manager"} + } + status, statusErr := s.Status() + switch { + case errors.Is(statusErr, service.ErrNotInstalled): + return diagServiceState{Status: "not_installed"} + case statusErr != nil: + return diagServiceState{Status: "unknown", Note: boundedDiagField(statusErr.Error())} + case status == service.StatusRunning: + return diagServiceState{Status: "running"} + case status == service.StatusStopped: + return diagServiceState{Status: "stopped"} + default: + return diagServiceState{Status: "unknown"} + } +} + +// collectServiceStateBounded runs diagServiceStateFn in the background and +// races it against ctx and diagServiceStateTimeout, so a wedged service +// manager reports "timed out" instead of hanging the whole diag report. The +// goroutine is left running if the probe never returns; that is harmless +// since the process exits shortly after diag prints its report. +func collectServiceStateBounded(ctx context.Context) diagServiceState { + done := make(chan diagServiceState, 1) + go func() { done <- diagServiceStateFn() }() + + timer := time.NewTimer(diagServiceStateTimeout) + defer timer.Stop() + + select { + case s := <-done: + return s + case <-ctx.Done(): + return diagServiceState{Status: "unknown", Note: "timed out"} + case <-timer.C: + return diagServiceState{Status: "unknown", Note: "timed out"} + } +} + +func collectAPIReachability(ctx context.Context) diagAPIReachability { + probeCtx, cancel := context.WithTimeout(ctx, diagAPIProbeTimeout) + defer cancel() + if err := diagProbeReachability(probeCtx, cdDev); err != nil { + return diagAPIReachability{Reachable: false, ErrorClass: classifyReachabilityError(err)} + } + return diagAPIReachability{Reachable: true} +} + +// classifyReachabilityError turns a probe failure into a coarse class safe +// to print: no host, no address, no request details, just what kind of +// failure it was. +func classifyReachabilityError(err error) string { + if err == nil { + return "" + } + if errors.Is(err, context.DeadlineExceeded) { + return "timeout" + } + var netErr net.Error + if errors.As(err, &netErr) && netErr.Timeout() { + return "timeout" + } + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + return "dns" + } + var certErr *tls.CertificateVerificationError + if errors.As(err, &certErr) { + return "tls" + } + var opErr *net.OpError + if errors.As(err, &opErr) { + return "connection" + } + return "other" +} + +// diagElevateHint names the step that gets root or administrator rights on +// this OS, for the text report. +func diagElevateHint() string { + if runtime.GOOS == "windows" { + return "run again from an administrator prompt" + } + return "run again with sudo" +} + +func displayOrDefault(s, def string) string { + if s == "" { + return def + } + return s +} + +// renderDiagText writes the text-mode report in a fixed section order, so +// output stays stable across runs and safe to diff or paste into a ticket. +func renderDiagText(w io.Writer, r diagReport) { + fmt.Fprintf(w, "client version: %s (commit %s)\n\n", r.ClientVersion, r.Commit) + + fmt.Fprintln(w, "managed preferences:") + switch { + case !r.ManagedPreferences.Applicable: + fmt.Fprintf(w, " %s\n", displayOrDefault(r.ManagedPreferences.Note, "not applicable")) + case !r.ManagedPreferences.ProfilePresent: + fmt.Fprintf(w, " %s\n", r.ManagedPreferences.Note) + default: + fmt.Fprintf(w, " provision token: %s\n", r.ManagedPreferences.ProvisionToken) + fmt.Fprintf(w, " custom hostname: %s\n", displayOrDefault(r.ManagedPreferences.CustomHostname, "(not set)")) + fmt.Fprintf(w, " intercept mode: %s\n", displayOrDefault(r.ManagedPreferences.InterceptMode, "(not set)")) + } + fmt.Fprintln(w) + + fmt.Fprintln(w, "last provisioning result:") + switch r.ProvisionResult.Status { + case "none": + fmt.Fprintln(w, " none recorded") + case "untrusted": + fmt.Fprintln(w, " result file present but not trusted (contents ignored)") + case "unreadable": + fmt.Fprintf(w, " permission denied (%s)\n", diagElevateHint()) + case "corrupt": + fmt.Fprintln(w, " result file present but could not be read (contents ignored)") + default: + fmt.Fprintf(w, " stage: %s\n", r.ProvisionResult.Stage) + fmt.Fprintf(w, " code: %s\n", r.ProvisionResult.Code) + fmt.Fprintf(w, " exit code: %d\n", r.ProvisionResult.ExitCode) + fmt.Fprintf(w, " message: %s\n", r.ProvisionResult.Message) + for _, a := range r.ProvisionResult.Attempts { + fmt.Fprintf(w, " attempt: %s/%s: %s\n", a.Addr, a.Proto, a.OSError) + } + fmt.Fprintf(w, " age: %s\n", (time.Duration(r.ProvisionResult.AgeSeconds) * time.Second).String()) + } + fmt.Fprintln(w) + + fmt.Fprintln(w, "service state:") + fmt.Fprintf(w, " status: %s\n", r.ServiceState.Status) + if r.ServiceState.Note != "" { + fmt.Fprintf(w, " note: %s\n", r.ServiceState.Note) + } + fmt.Fprintln(w) + + fmt.Fprintln(w, "api reachability:") + fmt.Fprintf(w, " reachable: %t\n", r.APIReachability.Reachable) + if r.APIReachability.ErrorClass != "" { + fmt.Fprintf(w, " error class: %s\n", r.APIReachability.ErrorClass) + } +} + +func writeDiagJSON(w io.Writer, r diagReport) error { + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(r) +} + +// initDiagCmd registers "ctrld diag" alongside the other top-level commands. +func initDiagCmd() *cobra.Command { + diagCmd := newDiagCmd() + rootCmd.AddCommand(diagCmd) + return diagCmd +} + +// newDiagCmd builds the "diag" command without registering it anywhere, so +// tests can attach it to an isolated root command instead of the package's +// shared one. +func newDiagCmd() *cobra.Command { + var asJSON bool + diagCmd := &cobra.Command{ + Use: "diag", + Short: "Collect diagnostics for a provisioning failure", + Long: diagCmdLong, + Args: cobra.NoArgs, + // diag always exits 0 once it ran: a failure it finds is reported, + // not turned into a nonzero exit. RunE returning an error would exit + // 1 (see Main), so every branch below reports instead of erroring. + RunE: func(cmd *cobra.Command, args []string) error { + report := buildDiagReport(context.Background()) + if asJSON { + if err := writeDiagJSON(cmd.OutOrStdout(), report); err != nil { + // A closed pipe (e.g. `ctrld diag --json | head -1`) must + // not turn into a nonzero exit; the report already ran. + mainLog.Load().Debug().Err(err).Msg("could not write diag JSON report") + } + return nil + } + renderDiagText(cmd.OutOrStdout(), report) + return nil + }, + } + diagCmd.Flags().BoolVar(&asJSON, "json", false, "print the report as JSON") + return diagCmd +} diff --git a/cmd/cli/commands_diag_test.go b/cmd/cli/commands_diag_test.go new file mode 100644 index 00000000..d347fc3c --- /dev/null +++ b/cmd/cli/commands_diag_test.go @@ -0,0 +1,429 @@ +package cli + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + "unicode/utf8" + + "github.com/spf13/cobra" +) + +// fakeProvisionToken stands in for a real provisioning code. Tests assert it +// never reaches either output mode. +const fakeProvisionToken = "org-v1-FAKE00000000000000000000TOKEN" + +func withManagedPrefsSeam(t *testing.T, supported bool, values map[string]string) { + t.Helper() + oldSupported := managedPrefsSupported + oldRead := managedPrefsRead + managedPrefsSupported = func() bool { return supported } + managedPrefsRead = func(_ context.Context, _, key string) (string, bool) { + v, ok := values[key] + return v, ok + } + t.Cleanup(func() { + managedPrefsSupported = oldSupported + managedPrefsRead = oldRead + }) +} + +// overrideDiagProvisionResultPath points the writer and diag at one temp +// file, so a test can write a result and read it back through diag. +func overrideDiagProvisionResultPath(t *testing.T) string { + t.Helper() + path := overrideProvisionResultPath(t) + old := diagProvisionResultPath + diagProvisionResultPath = func() string { return path } + t.Cleanup(func() { diagProvisionResultPath = old }) + return path +} + +func withServiceStateSeam(t *testing.T, state diagServiceState) { + t.Helper() + old := diagServiceStateFn + diagServiceStateFn = func() diagServiceState { return state } + t.Cleanup(func() { diagServiceStateFn = old }) +} + +func withAPIProbeSeam(t *testing.T, err error) { + t.Helper() + old := diagProbeReachability + diagProbeReachability = func(context.Context, bool) error { return err } + t.Cleanup(func() { diagProbeReachability = old }) +} + +func seedTrustedProvisionResult(t *testing.T, age time.Duration) { + t.Helper() + overrideDiagProvisionResultPath(t) + r := newProvisionResult(provisionCodeTokenExpired, "the provisioning code has expired", nil) + r.Timestamp = time.Now().Add(-age).UTC().Format(time.RFC3339) + if err := writeProvisionResult(r); err != nil { + t.Fatal(err) + } +} + +// seedOversizedProvisionResult writes a trusted result file straight to disk +// (bypassing newProvisionResult's own bounding), standing in for a file left +// by a mismatched or tampered version of ctrld. +func seedOversizedProvisionResult(t *testing.T) { + t.Helper() + overrideDiagProvisionResultPath(t) + attempts := make([]provisionBindAttempt, maxProvisionBindAttempts*3) + for i := range attempts { + attempts[i] = provisionBindAttempt{ + Addr: "0.0.0.0:53", + Proto: "udp", + OSError: strings.Repeat("e", diagFieldMaxLen*3), + } + } + r := &provisionResult{ + Version: 1, + Timestamp: time.Now().UTC().Format(time.RFC3339), + Stage: string(provisionStageListener), + Code: string(provisionCodeListenerBindFailed), + ExitCode: provisionExitCodeForCode[provisionCodeListenerBindFailed], + Message: strings.Repeat("m", diagFieldMaxLen*3), + Detail: &provisionDetail{Attempts: attempts}, + } + if err := writeProvisionResult(r); err != nil { + t.Fatal(err) + } +} + +func TestDiagProvisionResultBoundsOversizedFields(t *testing.T) { + seedOversizedProvisionResult(t) + + got := collectProvisionResultDiag() + + if n := utf8.RuneCountInString(got.Message); n > diagFieldMaxLen { + t.Errorf("message length = %d, want <= %d", n, diagFieldMaxLen) + } + if len(got.Attempts) > maxProvisionBindAttempts { + t.Errorf("attempts length = %d, want <= %d", len(got.Attempts), maxProvisionBindAttempts) + } + for _, a := range got.Attempts { + if n := utf8.RuneCountInString(a.OSError); n > diagFieldMaxLen { + t.Errorf("attempt os_error length = %d, want <= %d", n, diagFieldMaxLen) + } + } +} + +func TestDiagTextReportNeverLeaksToken(t *testing.T) { + withManagedPrefsSeam(t, true, map[string]string{ + "": "", // domain probe: profile present + "ProvisionToken": fakeProvisionToken, + "CustomHostname": "corp-laptop.example.com", + "InterceptMode": "intercept-dns", + }) + seedTrustedProvisionResult(t, 3*time.Minute+12*time.Second) + withServiceStateSeam(t, diagServiceState{Status: "stopped"}) + withAPIProbeSeam(t, nil) + + report := buildDiagReport(context.Background()) + var buf bytes.Buffer + renderDiagText(&buf, report) + out := buf.String() + + if strings.Contains(out, fakeProvisionToken) { + t.Fatalf("text output leaked the provision token: %s", out) + } + wantLines := []string{ + "provision token: present", + "custom hostname: corp-laptop.example.com", + "intercept mode: intercept-dns", + "stage: bootstrap", + "code: TOKEN_EXPIRED", + "exit code: 34", + "status: stopped", + "reachable: true", + } + for _, want := range wantLines { + if !strings.Contains(out, want) { + t.Errorf("text output missing %q, got:\n%s", want, out) + } + } +} + +func TestDiagJSONReportNeverLeaksToken(t *testing.T) { + withManagedPrefsSeam(t, true, map[string]string{ + "": "", + "ProvisionToken": fakeProvisionToken, + "CustomHostname": "corp-laptop.example.com", + "InterceptMode": "standard", + }) + seedTrustedProvisionResult(t, time.Minute) + withServiceStateSeam(t, diagServiceState{Status: "running"}) + withAPIProbeSeam(t, errors.New("dial tcp: connect: connection refused")) + + report := buildDiagReport(context.Background()) + var buf bytes.Buffer + if err := writeDiagJSON(&buf, report); err != nil { + t.Fatal(err) + } + out := buf.String() + if strings.Contains(out, fakeProvisionToken) { + t.Fatalf("JSON output leaked the provision token: %s", out) + } + + var decoded diagReport + if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil { + t.Fatalf("could not decode JSON report: %v", err) + } + if decoded.ManagedPreferences.ProvisionToken != "present" { + t.Errorf("provision_token = %q, want present", decoded.ManagedPreferences.ProvisionToken) + } + if decoded.ManagedPreferences.CustomHostname != "corp-laptop.example.com" { + t.Errorf("custom_hostname = %q", decoded.ManagedPreferences.CustomHostname) + } + if decoded.ProvisionResult.Status != "recorded" || decoded.ProvisionResult.Code != "TOKEN_EXPIRED" { + t.Errorf("provision_result = %+v", decoded.ProvisionResult) + } + if decoded.ServiceState.Status != "running" { + t.Errorf("service_state = %+v", decoded.ServiceState) + } + if decoded.APIReachability.Reachable { + t.Error("api_reachability.reachable = true, want false") + } + if decoded.APIReachability.ErrorClass == "" { + t.Error("api_reachability.error_class empty for an unreachable API") + } +} + +func TestDiagEmptyMachine(t *testing.T) { + overrideDiagProvisionResultPath(t) // temp dir, no result file written + withManagedPrefsSeam(t, false, nil) + withServiceStateSeam(t, diagServiceState{Status: "not_installed"}) + withAPIProbeSeam(t, context.DeadlineExceeded) + + report := buildDiagReport(context.Background()) + + if report.ManagedPreferences.Applicable { + t.Error("managed preferences reported applicable with no profile on this platform") + } + if report.ProvisionResult.Status != "none" { + t.Errorf("provision result status = %q, want none", report.ProvisionResult.Status) + } + if report.ServiceState.Status != "not_installed" { + t.Errorf("service state = %q, want not_installed", report.ServiceState.Status) + } + if report.APIReachability.Reachable { + t.Error("api reachability reported reachable with a forced timeout") + } + if report.APIReachability.ErrorClass != "timeout" { + t.Errorf("error class = %q, want timeout", report.APIReachability.ErrorClass) + } + + var buf bytes.Buffer + renderDiagText(&buf, report) + if !strings.Contains(buf.String(), "none recorded") { + t.Errorf("text output missing 'none recorded': %s", buf.String()) + } +} + +// Without root, diag must look where the root-run service wrote the result +// file, not in the home directory of the current user. +func TestDiagProvisionResultPathIgnoresUserHome(t *testing.T) { + want := "/etc/controld/" + provisionResultFileName + if runtime.GOOS == "windows" { + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + want = filepath.Join(filepath.Dir(exe), provisionResultFileName) + } + if got := diagProvisionResultPath(); got != want { + t.Errorf("diag provision result path = %q, want %q", got, want) + } +} + +// Diag must read where a daemon started with --homedir wrote, so it obeys +// the same override as the writer. +func TestDiagProvisionResultPathHonorsHomedir(t *testing.T) { + old := homedir + homedir = t.TempDir() + t.Cleanup(func() { homedir = old }) + want := filepath.Join(homedir, provisionResultFileName) + if got := diagProvisionResultPath(); got != want { + t.Errorf("diag provision result path = %q, want %q", got, want) + } +} + +// A result file the current user cannot read must report that, not +// "corrupt": the file is fine, the reader lacks root. +func TestDiagProvisionResultUnreadable(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("file modes do not deny reads on Windows") + } + if os.Geteuid() == 0 { + t.Skip("root can read a 0000 file") + } + path := overrideDiagProvisionResultPath(t) + if err := os.WriteFile(path, []byte("{}"), 0o000); err != nil { + t.Fatal(err) + } + + r := collectProvisionResultDiag() + if r.Status != "unreadable" { + t.Errorf("status = %q, want unreadable", r.Status) + } + if r.AgeSeconds != -1 { + t.Errorf("age_seconds = %d, want -1", r.AgeSeconds) + } + + var buf bytes.Buffer + renderDiagText(&buf, diagReport{ProvisionResult: r}) + if !strings.Contains(buf.String(), "permission denied ("+diagElevateHint()+")") { + t.Errorf("text output does not name permission denied and the elevation step: %s", buf.String()) + } +} + +func TestDiagServiceStateHangYieldsTimeout(t *testing.T) { + old := diagServiceStateFn + // started closes the instant the background probe goroutine reads and + // invokes our stub. Cleanup waits for that before restoring the global: + // otherwise a slow-to-schedule goroutine can still be reading + // diagServiceStateFn when Cleanup writes to it, a data race on the shared + // package var (this test's stub is left running past the test's own + // return, same as production - see collectServiceStateBounded's doc). + started := make(chan struct{}) + diagServiceStateFn = func() diagServiceState { + close(started) + time.Sleep(2 * time.Second) // stand in for a wedged systemctl/launchctl + return diagServiceState{Status: "running"} + } + t.Cleanup(func() { + <-started + diagServiceStateFn = old + }) + + withManagedPrefsSeam(t, false, nil) + overrideDiagProvisionResultPath(t) + withAPIProbeSeam(t, nil) + + // A short deadline stands in for the overall 15s budget already having + // run low; the probe must still yield within it instead of hanging. + ctx, cancel := context.WithTimeout(context.Background(), 150*time.Millisecond) + defer cancel() + + start := time.Now() + report := buildDiagReport(ctx) + elapsed := time.Since(start) + + if elapsed > time.Second { + t.Fatalf("buildDiagReport took %s, want bounded well under the 2s hang", elapsed) + } + if report.ServiceState.Status != "unknown" { + t.Errorf("service state status = %q, want unknown", report.ServiceState.Status) + } + if !strings.Contains(report.ServiceState.Note, "timed out") { + t.Errorf("service state note = %q, want it to mention timing out", report.ServiceState.Note) + } +} + +func TestDiagManagedPrefsProfileAbsent(t *testing.T) { + withManagedPrefsSeam(t, true, map[string]string{}) // domain read fails: profile absent + m := collectManagedPreferences(context.Background()) + if m.ProfilePresent { + t.Error("profile reported present when the domain read failed") + } + if m.Note == "" { + t.Error("expected a note explaining the absent profile") + } +} + +func TestDiagManagedPrefsTokenAbsent(t *testing.T) { + withManagedPrefsSeam(t, true, map[string]string{"": ""}) // profile present, no keys set + m := collectManagedPreferences(context.Background()) + if !m.ProfilePresent { + t.Fatal("profile should be present") + } + if m.ProvisionToken != "absent" { + t.Errorf("provision token = %q, want absent", m.ProvisionToken) + } +} + +// An empty ProvisionToken value must read as absent: the postinstall refuses +// to provision on an empty token, so diag must not call it present. +func TestDiagManagedPrefsTokenEmpty(t *testing.T) { + withManagedPrefsSeam(t, true, map[string]string{"": "", "ProvisionToken": ""}) + m := collectManagedPreferences(context.Background()) + if !m.ProfilePresent { + t.Fatal("profile should be present") + } + if m.ProvisionToken != "absent" { + t.Errorf("provision token = %q, want absent", m.ProvisionToken) + } +} + +func TestClassifyReachabilityError(t *testing.T) { + if got := classifyReachabilityError(nil); got != "" { + t.Errorf("nil error class = %q, want empty", got) + } + if got := classifyReachabilityError(context.DeadlineExceeded); got != "timeout" { + t.Errorf("deadline exceeded class = %q, want timeout", got) + } +} + +func TestDiagCommandJSONFlag(t *testing.T) { + withManagedPrefsSeam(t, false, nil) + overrideDiagProvisionResultPath(t) + withServiceStateSeam(t, diagServiceState{Status: "not_installed"}) + withAPIProbeSeam(t, nil) + + testRootCmd := &cobra.Command{Use: "ctrld"} + testRootCmd.AddCommand(newDiagCmd()) + var buf bytes.Buffer + testRootCmd.SetOut(&buf) + testRootCmd.SetArgs([]string{"diag", "--json"}) + if err := testRootCmd.Execute(); err != nil { + t.Fatalf("diag --json returned error: %v", err) + } + var decoded diagReport + if err := json.Unmarshal(buf.Bytes(), &decoded); err != nil { + t.Fatalf("diag --json did not print valid JSON: %v\n%s", err, buf.String()) + } +} + +// writerFailingAfter accepts the first n bytes written to it, then fails +// every write after that - standing in for a pipe closed by a downstream +// reader (`ctrld diag --json | head -1`). +type writerFailingAfter struct { + n int + written int +} + +func (w *writerFailingAfter) Write(p []byte) (int, error) { + if w.written >= w.n { + return 0, errors.New("write: broken pipe") + } + remaining := w.n - w.written + if len(p) > remaining { + w.written += remaining + return remaining, errors.New("write: broken pipe") + } + w.written += len(p) + return len(p), nil +} + +func TestDiagJSONWriteErrorStillExitsZero(t *testing.T) { + withManagedPrefsSeam(t, false, nil) + overrideDiagProvisionResultPath(t) + withServiceStateSeam(t, diagServiceState{Status: "not_installed"}) + withAPIProbeSeam(t, nil) + + testRootCmd := &cobra.Command{Use: "ctrld"} + testRootCmd.AddCommand(newDiagCmd()) + testRootCmd.SetOut(&writerFailingAfter{n: 10}) + testRootCmd.SetArgs([]string{"diag", "--json"}) + if err := testRootCmd.Execute(); err != nil { + t.Fatalf("diag --json with a failing writer returned error %v, want nil per the always-exit-0 contract", err) + } +} diff --git a/cmd/cli/commands_start_test.go b/cmd/cli/commands_start_test.go index 882fac2f..2ac2ec40 100644 --- a/cmd/cli/commands_start_test.go +++ b/cmd/cli/commands_start_test.go @@ -1,10 +1,162 @@ package cli import ( + "go/ast" + "go/parser" + "go/token" + "os" + "strings" "testing" "time" ) +// startRunFuncSource extracts the source text of initStartCmd's "Run:" +// closure body from commands.go. Driving the closure itself end-to-end for +// every early-return branch is not practical in a unit test: within a few +// lines of any check failing, it reaches into the real OS service manager. +// Some invariants about its shape are cheaper and more reliable to pin by +// reading the source than by executing it. +func startRunFuncSource(t *testing.T) string { + t.Helper() + file := packageSourcePath(t, "commands.go") + fset := token.NewFileSet() + node, err := parser.ParseFile(fset, file, nil, 0) + if err != nil { + t.Fatalf("could not parse %s: %v", file, err) + } + var runLit *ast.FuncLit + ast.Inspect(node, func(n ast.Node) bool { + fn, ok := n.(*ast.FuncDecl) + if !ok || fn.Name.Name != "initStartCmd" { + return true + } + ast.Inspect(fn.Body, func(n ast.Node) bool { + kv, ok := n.(*ast.KeyValueExpr) + if !ok { + return true + } + // initStartCmd defines a second, unrelated "Run:" field further down + // for a start command alias; take only the first match, which is the + // real start command's closure this test cares about. + if runLit == nil { + if ident, ok := kv.Key.(*ast.Ident); ok && ident.Name == "Run" { + if lit, ok := kv.Value.(*ast.FuncLit); ok { + runLit = lit + } + } + } + return true + }) + return false + }) + if runLit == nil { + t.Fatalf("Run: func literal not found in initStartCmd in %s", file) + } + src, err := os.ReadFile(file) + if err != nil { + t.Fatalf("could not read %s: %v", file, err) + } + start := fset.Position(runLit.Body.Lbrace).Offset + end := fset.Position(runLit.Body.Rbrace).Offset + return string(src[start:end]) +} + +// TestStartCommandClearsProvisionResultBeforeAnyCheck pins the ordering fix: +// clearProvisionResult() must run before every check in the start command's +// Run closure that can fail or return early, not just before doTasksE. +// Without this, a check between the top of the closure and the old call +// sites could return early (whether by writing its own classified failure +// or, like the "service already running" and service-manager-init-error +// paths, by writing nothing at all) while a previous attempt's result file +// was still sitting there to mislead diag/postinstall on retry. +func TestStartCommandClearsProvisionResultBeforeAnyCheck(t *testing.T) { + body := startRunFuncSource(t) + + clearIdx := strings.Index(body, "clearProvisionResult()") + if clearIdx == -1 { + t.Fatal("start command no longer calls clearProvisionResult()") + } + + // Every check or step that can return out of the closure before reaching + // doTasksE. Each must appear after the entry clear. + earlyChecks := []string{ + "checkStrFlagEmpty(", + "validateCdAndNextDNSFlags(", + "validateInterceptModeFlag(", + "doTasksE(", + } + for _, check := range earlyChecks { + idx := strings.Index(body, check) + if idx == -1 { + t.Fatalf("expected the start command to still call %s", check) + } + if idx < clearIdx { + t.Errorf("%s appears before clearProvisionResult(): a failure there could leave a stale result file behind", check) + } + } +} + +// TestStartCommandClassifiesServiceInitFailure pins the fix for a bare +// return: when newService fails in the start command, the closure must fail +// through failProvisionUnclassified, so a result file and the identifier +// line exist, instead of a plain return that exits 0. +func TestStartCommandClassifiesServiceInitFailure(t *testing.T) { + body := startRunFuncSource(t) + initIdx := strings.Index(body, "newService(p, sc)") + if initIdx == -1 { + t.Fatal("start command no longer calls newService(p, sc)") + } + branchEnd := strings.Index(body[initIdx:], "p.preRun()") + if branchEnd == -1 { + t.Fatal("could not find the end of the service init branch") + } + if !strings.Contains(body[initIdx:initIdx+branchEnd], "failProvisionUnclassified(") { + t.Error("service init failure does not fail through failProvisionUnclassified") + } +} + +// TestStartCommandReplacesStaleResultOnEarlyClassifiedFailure is a +// behavioral companion to the structural test above: it drives the real +// start command through its earliest classified failure (an invalid +// --intercept-mode) and checks the file left behind names the new attempt, +// not a stale one seeded beforehand. +func TestStartCommandReplacesStaleResultOnEarlyClassifiedFailure(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldIntercept, oldCdUID, oldCdOrg, oldNextdns := interceptMode, cdUID, cdOrg, nextdns + t.Cleanup(func() { + interceptMode, cdUID, cdOrg, nextdns = oldIntercept, oldCdUID, oldCdOrg, oldNextdns + }) + + // initStartCmd binds these globals to flag defaults as it registers them + // (StringVarP writes the default straight into the pointer), so the + // command must exist before the test overrides the values it drives the + // closure with. + cmd := initStartCmd() + cdUID, cdOrg, nextdns = "", "", "" + interceptMode = "bogus" // fails validateInterceptModeFlag before any OS work + + if err := writeProvisionResult(newProvisionResult(provisionCodeServiceStartFailed, "a previous failed attempt", nil)); err != nil { + t.Fatal(err) + } + + cmd.Run(cmd, nil) + + wantExit := provisionExitCodeForCode[provisionCodeInterceptModeInvalid] + if *exitCode != wantExit { + t.Fatalf("exit = %d, want %d (validateInterceptModeFlag should have run)", *exitCode, wantExit) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code == string(provisionCodeServiceStartFailed) { + t.Fatal("stale result from a previous attempt survived the new attempt") + } + if r.Code != string(provisionCodeInterceptModeInvalid) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeInterceptModeInvalid) + } +} + func TestServiceStageFailureCode(t *testing.T) { tests := []struct { taskName string diff --git a/cmd/cli/hostname.go b/cmd/cli/hostname.go index d28435db..21f33dc7 100644 --- a/cmd/cli/hostname.go +++ b/cmd/cli/hostname.go @@ -1,6 +1,10 @@ package cli -import "regexp" +import ( + "fmt" + "regexp" + "strings" +) // validHostname reports whether hostname is a valid hostname. // A valid hostname contains 3 -> 64 characters and conform to RFC1123. @@ -12,3 +16,67 @@ func validHostname(hostname string) bool { validHostnameRfc1123 := regexp.MustCompile(`^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$`) return validHostnameRfc1123.MatchString(hostname) } + +// isHostnameChar reports whether r is part of validHostname's accepted +// charset (letters, digits, hyphen, dot). It does not check position, so a +// hostname can fail validHostname on structure (length, leading/trailing +// hyphen) while every one of its characters passes here. +func isHostnameChar(r rune) bool { + switch { + case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9': + return true + case r == '-' || r == '.': + return true + } + return false +} + +// offendingHostnameChars returns the distinct characters in hostname that +// validHostname's charset does not accept, in first-seen order. Empty when +// every character is accepted - a rejection can still come from structure +// alone (too short, too long, a leading or trailing hyphen). +func offendingHostnameChars(hostname string) string { + seen := make(map[rune]bool) + var bad []rune + for _, r := range hostname { + if isHostnameChar(r) || seen[r] { + continue + } + seen[r] = true + bad = append(bad, r) + } + return string(bad) +} + +// serverFoldedHostnameChars are the characters ControlD's +// DevicesTableModel.formatDeviceName folds to '-' (or strips) when it +// registers a device name server-side. Only '.' can actually reach +// hostnameMayBeFoldedByServer through validateCustomHostnameFlag's guarded +// path: validHostname runs first and already rejects any hostname +// containing a space or a '+' as CUSTOM_HOSTNAME_INVALID, so those two never +// get here from an explicit --custom-hostname value. They stay in this set +// for completeness: a mobile caller can set CustomHostname to an +// OS-derived default directly, without going through +// validateCustomHostnameFlag at all, so a space or '+' can still reach the +// API unvalidated by this client. +const serverFoldedHostnameChars = ". +" + +// hostnameMayBeFoldedByServer reports whether hostname contains a character +// ControlD may fold or strip when it registers the device, so the name ctrld +// accepted may not be the name the dashboard ends up showing. +func hostnameMayBeFoldedByServer(hostname string) bool { + return strings.ContainsAny(hostname, serverFoldedHostnameChars) +} + +// customHostnameFailureMessage names the field, the offending character(s) +// when there are any, and the allowed format for CUSTOM_HOSTNAME_INVALID. +// ctrld's accept/reject rule (validHostname) is unchanged - this only +// explains a rejection that used to be a bare fatal exit. +func customHostnameFailureMessage(hostname string) string { + const allowedFormat = "3-64 characters of letters, digits, hyphens, and dots (RFC1123 hostname format)" + reason := "is not a valid hostname" + if bad := offendingHostnameChars(hostname); bad != "" { + reason = fmt.Sprintf("contains characters a hostname cannot use: %q", bad) + } + return fmt.Sprintf("--custom-hostname (CustomHostname) %q %s; allowed format: %s", hostname, reason, allowedFormat) +} diff --git a/cmd/cli/hostname_test.go b/cmd/cli/hostname_test.go index f44b2313..b096215d 100644 --- a/cmd/cli/hostname_test.go +++ b/cmd/cli/hostname_test.go @@ -33,3 +33,65 @@ func Test_validHostname(t *testing.T) { }) } } + +// TestOffendingHostnameChars pins the characters surfaced in the +// CUSTOM_HOSTNAME_INVALID message, so the failure names what is actually +// wrong instead of a bare "invalid hostname". +func TestOffendingHostnameChars(t *testing.T) { + tests := []struct { + name string + hostname string + want string + }{ + {"single offender", "foo@bar", "@"}, + {"space", "foo bar", " "}, + {"dot is allowed", "foo.bar", ""}, + {"hyphen is allowed", "foo-bar", ""}, + {"distinct offenders in order", "a!b!c#d", "!#"}, + {"structurally invalid but no bad char", strings.Repeat("a", 65), ""}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := offendingHostnameChars(tc.hostname); got != tc.want { + t.Errorf("offendingHostnameChars(%q) = %q, want %q", tc.hostname, got, tc.want) + } + }) + } +} + +// TestHostnameMayBeFoldedByServer pins the characters ControlD's +// DevicesTableModel.formatDeviceName folds or strips when it registers a +// device, so a ctrld-accepted name using one gets a heads-up notice instead +// of silently registering under a different name. +func TestHostnameMayBeFoldedByServer(t *testing.T) { + tests := []struct { + name string + hostname string + want bool + }{ + {"dot", "foo.bar", true}, + {"space", "foo bar", true}, + {"plus", "foo+bar", true}, + {"plain", "foobar", false}, + {"hyphen only", "foo-bar", false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := hostnameMayBeFoldedByServer(tc.hostname); got != tc.want { + t.Errorf("hostnameMayBeFoldedByServer(%q) = %v, want %v", tc.hostname, got, tc.want) + } + }) + } +} + +// TestCustomHostnameFailureMessage pins the message shape the T6 contract +// requires: the flag/field name, the offending character(s), and the +// allowed format. +func TestCustomHostnameFailureMessage(t *testing.T) { + msg := customHostnameFailureMessage("foo@bar") + for _, want := range []string{"--custom-hostname", "CustomHostname", "@", "allowed format"} { + if !strings.Contains(msg, want) { + t.Errorf("customHostnameFailureMessage() = %q, want it to contain %q", msg, want) + } + } +} diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 082269e4..a8382210 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -637,6 +637,25 @@ func (p *prog) setupUpstream(cfg *ctrld.Config) { p.ptrNameservers = ptrNameservers } +// notifyExitToLogServer writes msgExit to the log connection, if one is +// open, so a waiting "ctrld start" sees this run ended instead of waiting +// out the full timeout. A terminal provisioning path calls this right +// before failing. +func (p *prog) notifyExitToLogServer() { + if p.logConn != nil { + _, _ = p.logConn.Write([]byte(msgExit)) + } +} + +// reportServeDNSFailure classifies a listener that bound successfully - the +// LISTENER_* codes already rule out a bind conflict - but failed to actually +// serve DNS. No dedicated code exists for this, so it falls back to +// UNCLASSIFIED. notifyExitToLogServer unblocks a waiting "ctrld start" +// before the process exits. +func (p *prog) reportServeDNSFailure(listenerNum string, err error) { + failRunUnclassified(mainLog.Load().Error().Err(err), fmt.Sprintf("unable to start dns proxy on listener.%s: %v", listenerNum, err), p.notifyExitToLogServer) +} + // run runs the ctrld main components. // // The reload boolean indicates that the function is run when ctrld first start @@ -754,7 +773,7 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { addr := net.JoinHostPort(listenerConfig.IP, strconv.Itoa(listenerConfig.Port)) mainLog.Load().Info().Msgf("starting DNS server on listener.%s: %s", listenerNum, addr) if err := p.serveDNS(listenerNum); err != nil { - mainLog.Load().Fatal().Err(err).Msgf("unable to start dns proxy on listener.%s", listenerNum) + p.reportServeDNSFailure(listenerNum, err) } mainLog.Load().Debug().Msgf("end of serveDNS listener.%s: %s", listenerNum, addr) }(listenerNum) @@ -927,8 +946,11 @@ var ( initializeOsResolverWithSystemNameserversFn = ctrld.InitializeOsResolverWithSystemNameservers setDnsForRunningIfaceFn = (*prog).setDnsForRunningIface resetDNSFn = (*prog).resetDNS - refuseFallbackFatal = func(format string, v ...any) { - mainLog.Load().Fatal().Msgf(format, v...) + // refuseFallbackFatal reports a startup failure the interface-DNS fallback + // cannot safely paper over, then exits. No dedicated code exists for this, + // so it falls back to UNCLASSIFIED. + refuseFallbackFatal = func(p *prog, format string, v ...any) { + failRunUnclassified(mainLog.Load().Error(), fmt.Sprintf(format, v...), p.notifyExitToLogServer) } ) @@ -1030,7 +1052,7 @@ func (p *prog) setDNS(systemNameservers []string) { // Leave the host resolvable: restore static settings or DHCP rather than // exiting with an interface still pointed at a ctrld that is not serving. resetDNSFn(p, false, true) - refuseFallbackFatal("Refusing to fall back to interface DNS: it cannot direct queries to %s:%d, which would leave this host with no working resolver. Free port 53 for ctrld, or resolve the intercept failure, then start again.", lc.IP, lc.Port) + refuseFallbackFatal(p, "Refusing to fall back to interface DNS: it cannot direct queries to %s:%d, which would leave this host with no working resolver. Free port 53 for ctrld, or resolve the intercept failure, then start again.", lc.IP, lc.Port) // Unreachable in production - the line above exits - but returning // explicitly keeps the refusal from depending on that, so nothing can // fall through to installing the fallback this just rejected. diff --git a/cmd/cli/prog_intercept_fallback_test.go b/cmd/cli/prog_intercept_fallback_test.go index 0cf98ab0..06f71248 100644 --- a/cmd/cli/prog_intercept_fallback_test.go +++ b/cmd/cli/prog_intercept_fallback_test.go @@ -135,7 +135,7 @@ func newInterceptFallbackHarness(t *testing.T, lc *ctrld.ListenerConfig) *interc return nil } resetDNSFn = func(_ *prog, _ bool, _ bool) { h.resetCalls++ } - refuseFallbackFatal = func(format string, v ...any) { + refuseFallbackFatal = func(_ *prog, format string, v ...any) { h.refusals = append(h.refusals, fmt.Sprintf(format, v...)) } diff --git a/cmd/cli/provision_boundary_guard_test.go b/cmd/cli/provision_boundary_guard_test.go new file mode 100644 index 00000000..a4585061 --- /dev/null +++ b/cmd/cli/provision_boundary_guard_test.go @@ -0,0 +1,243 @@ +package cli + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "runtime" + "testing" +) + +// packageSourcePath resolves a file in this package's source directory, or +// skips the test when the source is not available. The Windows CI job runs +// pre-built test binaries with no checkout, so neither the working directory +// nor the build-time path recorded by runtime.Caller reaches the source +// there. Skipping is safe for these static checks: they analyze the same +// source on every platform, and the jobs with a checkout still enforce them. +func packageSourcePath(t *testing.T, name string) string { + t.Helper() + if _, err := os.Stat(name); err == nil { + return name + } + if _, thisFile, _, ok := runtime.Caller(0); ok { + p := filepath.Join(filepath.Dir(thisFile), name) + if _, err := os.Stat(p); err == nil { + return p + } + } + t.Skipf("package source for %s not available (pre-built test binary without a checkout); this static check runs where source is present", name) + return "" +} + +// provisionBoundaryFiles are the files this lane's provisioning-hardening +// work audited: every terminal path they can reach must report a classified +// failure (a result file, an output line, a stage-scoped exit code), not a +// bare crash installer tooling cannot read. TestProvisioningTerminalPathAudit +// documents the audit by scenario; this test enforces it structurally so a +// later change cannot reintroduce a bare fatal without either classifying it +// or adding it here with a reason. +// +// v1.0 keeps the start command, the run command, and every other top-level +// command (log, stop, reload, status, uninstall, clients, upgrade) in one +// commands.go, where master splits each into its own file. That collapses +// several master file names into "commands.go" here, and it means the +// allowlist below has one entry per out-of-scope command instead of one +// excluded file per command. +// +// Scope: the check sees direct calls (".Fatal()", "os.Exit", "panic") in the +// files listed here. A helper defined in one of these files is caught once, +// at the body that holds the call, so an indirect exit through it needs no +// second rule. A helper defined outside the list, such as +// checkHasElevatedPrivilege in service.go wired as the "start" PreRun, is +// outside the reach of this check. TestProvisioningTerminalPathAudit lists +// those by hand in its exclusion comment. +var provisionBoundaryFiles = []string{ + "cli.go", + "commands.go", + "prog.go", + "provision_result.go", +} + +// provisionBoundaryFatalAllowlist is every call in the files above that +// still reaches a bare ".Fatal()" (on any logger chain), "os.Exit", or +// "panic", keyed +// by "file.go:FuncName" (FuncName includes the receiver type for methods, +// e.g. "prog.setDNS"). Each entry names why it is not a provisioning +// failure, or why it predates and sits outside the --cd-org boundary this +// lane hardened. +var provisionBoundaryFatalAllowlist = map[string]string{ + "cli.go:RunMobile": `panic on a nil AppConfig is a programming error in the mobile host ` + + `app, at an entry point the CLI command tree never calls. Not reachable from ` + + `"ctrld start --cd-org".`, + + "cli.go:run": `os.Exit(0) is the successful exit of the daemon-respawn launcher, right ` + + `after it starts the real background process. Not a failure.`, + + "cli.go:readConfigFile": `shared CLI config-parsing helper used the same way by every ` + + `invocation mode (config-file, no-config, nextdns, --cd, --cd-org). Predates and is ` + + `orthogonal to the --cd-org provisioning boundary.`, + + "cli.go:processNoConfigFlags": `shared CLI helper enforcing --listen/--primary_upstream in ` + + `no-config mode. Applies uniformly across every invocation mode, not specifically to ` + + `provisioning.`, + + "cli.go:processListenFlag": `shared CLI helper parsing --listen. Applies uniformly across ` + + `every invocation mode, not specifically to provisioning.`, + + "cli.go:readConfigWithNotice": `shared CLI helper (userHomeDir failure while locating the ` + + `default config file). Applies uniformly across every invocation mode, not specifically ` + + `to provisioning.`, + + "cli.go:checkStrFlagEmpty": `shared flag-emptiness helper. Only an explicit empty --cd-org ` + + `is a provisioning-token value, classified separately inside this function; every other ` + + `flag (currently --cd) keeps its bare fatal.`, + + "cli.go:uninstall": `the "ctrld uninstall" command's own router re-configuration step. ` + + `Outside the --cd-org provisioning boundary this lane hardened.`, + + "commands.go:initStartCmd": `two unrelated bare exits nested in the same top-level func: ` + + `the deactivation pin check's os.Exit(126) (out of scope for this lane by contract - ` + + `"pin-check 126 untouched"), and the hidden "router setup" alias command's own ` + + `os.Executable/re-exec failure handling (generic CLI plumbing for a re-exec helper, not ` + + `a provisioning outcome). Every check on the actual --cd-org boundary in this function ` + + `already classifies through failProvision/failRunUnclassified.`, + + "commands.go:initLogCmd": `the "ctrld log" command. Outside the --cd-org provisioning ` + + `boundary this lane hardened.`, + + "commands.go:initStopCmd": `the "ctrld stop" command. Outside the --cd-org provisioning ` + + `boundary this lane hardened.`, + + "commands.go:initReloadCmd": `the "ctrld reload"/"ctrld restart" commands. Outside the ` + + `--cd-org provisioning boundary this lane hardened.`, + + "commands.go:initStatusCmd": `the "ctrld status" command. Outside the --cd-org ` + + `provisioning boundary this lane hardened.`, + + "commands.go:initUninstallCmd": `the "ctrld uninstall" command. Outside the --cd-org ` + + `provisioning boundary this lane hardened.`, + + "commands.go:initClientsCmd": `the "ctrld clients" command. Outside the --cd-org ` + + `provisioning boundary this lane hardened.`, + + "commands.go:initUpgradeCmd": `the "ctrld upgrade" command. Outside the --cd-org ` + + `provisioning boundary this lane hardened.`, + + "prog.go:prog.Stop": `the deactivation pin check's os.Exit(126). Out of scope for this ` + + `lane by contract ("pin-check 126 untouched").`, + + "prog.go:prog.setDNS": `the runtime --intercept-mode validation (validInterceptMode), ` + + `distinct from the early "ctrld start" check that already reports ` + + `INTERCEPT_MODE_INVALID. Left alone per T6's own doc comment.`, +} + +// funcDeclKey names a function declaration the way +// provisionBoundaryFatalAllowlist keys it: "FuncName" for a plain function, +// "ReceiverType.FuncName" for a method. +func funcDeclKey(fn *ast.FuncDecl) string { + if fn.Recv == nil || len(fn.Recv.List) == 0 { + return fn.Name.Name + } + recvType := fn.Recv.List[0].Type + if star, ok := recvType.(*ast.StarExpr); ok { + recvType = star.X + } + if ident, ok := recvType.(*ast.Ident); ok { + return ident.Name + "." + fn.Name.Name + } + return fn.Name.Name +} + +// isBareFatalOrExitCall reports whether call is ".Fatal(...)" on any +// logger chain, "os.Exit(...)", or a bare "panic(...)". +func isBareFatalOrExitCall(call *ast.CallExpr) bool { + if ident, ok := call.Fun.(*ast.Ident); ok && ident.Name == "panic" { + return true + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return false + } + if sel.Sel.Name == "Fatal" { + return true + } + if sel.Sel.Name == "Exit" { + if ident, ok := sel.X.(*ast.Ident); ok && ident.Name == "os" { + return true + } + } + return false +} + +// TestProvisionBoundaryHasNoUnlistedBareFatal parses every provisioning +// boundary file and flags a bare Fatal/os.Exit that is not in the allowlist +// above. A new one appearing here means a change added a crash the +// provisioning boundary's contract does not allow: classify it through +// failProvision/failProvisionUnclassified, or add it to the allowlist with a +// reason if it genuinely sits outside the boundary. +func TestProvisionBoundaryHasNoUnlistedBareFatal(t *testing.T) { + fset := token.NewFileSet() + for _, file := range provisionBoundaryFiles { + node, err := parser.ParseFile(fset, packageSourcePath(t, file), nil, 0) + if err != nil { + t.Fatalf("could not parse %s: %v", file, err) + } + for _, decl := range node.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Body == nil { + continue + } + key := file + ":" + funcDeclKey(fn) + ast.Inspect(fn.Body, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok || !isBareFatalOrExitCall(call) { + return true + } + if _, allowed := provisionBoundaryFatalAllowlist[key]; !allowed { + pos := fset.Position(call.Pos()) + t.Errorf("%s:%d: unlisted bare fatal/exit in %s - classify it through "+ + "failProvision/failProvisionUnclassified, or add it to "+ + "provisionBoundaryFatalAllowlist with a reason", file, pos.Line, key) + } + return true + }) + } + } +} + +// TestProvisionBoundaryAllowlistHasNoStaleEntries is the converse check: every +// allowlisted key must still name a real bare Fatal/os.Exit. A stale entry +// (the call was removed or reclassified) would silently widen the allowlist +// and hide a real regression the next time this test runs. +func TestProvisionBoundaryAllowlistHasNoStaleEntries(t *testing.T) { + fset := token.NewFileSet() + found := make(map[string]bool, len(provisionBoundaryFatalAllowlist)) + for _, file := range provisionBoundaryFiles { + node, err := parser.ParseFile(fset, packageSourcePath(t, file), nil, 0) + if err != nil { + t.Fatalf("could not parse %s: %v", file, err) + } + for _, decl := range node.Decls { + fn, ok := decl.(*ast.FuncDecl) + if !ok || fn.Body == nil { + continue + } + key := file + ":" + funcDeclKey(fn) + ast.Inspect(fn.Body, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok || !isBareFatalOrExitCall(call) { + return true + } + found[key] = true + return true + }) + } + } + for key := range provisionBoundaryFatalAllowlist { + if !found[key] { + t.Errorf("provisionBoundaryFatalAllowlist[%q] no longer matches any bare fatal/exit; remove the stale entry", key) + } + } +} diff --git a/cmd/cli/provision_input_test.go b/cmd/cli/provision_input_test.go new file mode 100644 index 00000000..07139076 --- /dev/null +++ b/cmd/cli/provision_input_test.go @@ -0,0 +1,417 @@ +package cli + +import ( + "context" + "strings" + "testing" + + "github.com/rs/zerolog" + "github.com/spf13/cobra" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// captureMainLog swaps mainLog for a test-local buffer logger and restores +// the previous logger on cleanup. Asserting on the shared logOutput sink is +// order-dependent: a test that ran earlier can re-store mainLog (Windows' +// Test_validInterfaces calls initConsoleLogging), leaving the shared buffer +// stale for every later test. +func captureMainLog(t *testing.T) *syncBuffer { + t.Helper() + buf := &syncBuffer{} + l := zerolog.New(buf) + old := mainLog.Load() + mainLog.Store(&l) + t.Cleanup(func() { mainLog.Store(old) }) + return buf +} + +func TestProvisionTokenShapeValid(t *testing.T) { + tests := []struct { + name string + token string + want bool + }{ + {"too short", "abcde", false}, + {"minimum length", "abcdef", true}, + {"maximum length", strings.Repeat("a", 64), true}, + {"too long", strings.Repeat("a", 65), false}, + {"contains space", "org-v1- abc", false}, + {"contains tab", "org-v1-\tabc", false}, + {"contains newline", "org-v1-\nabc", false}, + {"contains control char", "org-v1-\x00abc", false}, + {"sane with prefix", "org-v1-abcdef123456", true}, + {"sane without prefix", "legacytoken123", true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := provisionTokenShapeValid(tc.token); got != tc.want { + t.Errorf("provisionTokenShapeValid(%q) = %v, want %v", tc.token, got, tc.want) + } + }) + } +} + +// TestCdUIDFromProvTokenMalformedToken proves a malformed --cd-org value is +// classified before any network attempt: no call reaches fetchResolverUIDFn, +// and the malformed value itself never appears in the persisted message. +func TestCdUIDFromProvTokenMalformedToken(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldFetch, oldHostname := fetchResolverUIDFn, customHostname + t.Cleanup(func() { fetchResolverUIDFn, customHostname = oldFetch, oldHostname }) + + cdUID = "" + const malformed = "ab cd" + cdOrg = malformed + customHostname = "" + + called := false + fetchResolverUIDFn = func(context.Context, *controld.UtilityOrgRequest, string, bool) (*controld.ResolverConfig, error) { + called = true + return nil, nil + } + + if got := cdUIDFromProvToken(); got != "" { + t.Errorf("cdUIDFromProvToken() = %q, want empty on failure", got) + } + if called { + t.Error("fetchResolverUIDFn was called; malformed token must be rejected before any network attempt") + } + if *exitCode != provisionExitCodeForCode[provisionCodeProvisionTokenMalformed] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeProvisionTokenMalformed]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeProvisionTokenMalformed) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeProvisionTokenMalformed) + } + if r.Stage != string(provisionStageInput) { + t.Errorf("stage = %q, want %q", r.Stage, provisionStageInput) + } + if strings.Contains(r.Message, malformed) { + t.Errorf("malformed token leaked into result message: %q", r.Message) + } +} + +// TestCdUIDFromProvTokenPrefixlessTokenProceeds proves a token missing the +// "org-v1-" prefix is not rejected - only warned about - since legacy codes +// may lack it. +func TestCdUIDFromProvTokenPrefixlessTokenProceeds(t *testing.T) { + _, _ = stubProvisionGlobals(t) + logBuf := captureMainLog(t) + oldFetch, oldHostname := fetchResolverUIDFn, customHostname + t.Cleanup(func() { fetchResolverUIDFn, customHostname = oldFetch, oldHostname }) + + cdUID = "" + cdOrg = "legacytoken123" + customHostname = "" + + called := false + fetchResolverUIDFn = func(context.Context, *controld.UtilityOrgRequest, string, bool) (*controld.ResolverConfig, error) { + called = true + return &controld.ResolverConfig{UID: "resolved-uid"}, nil + } + + if got := cdUIDFromProvToken(); got != "resolved-uid" { + t.Errorf("cdUIDFromProvToken() = %q, want resolved-uid", got) + } + if !called { + t.Error("fetchResolverUIDFn was not called; a prefixless-but-sane token must still proceed") + } + if !strings.Contains(logBuf.String(), "org-v1-") { + t.Error("expected a warning mentioning the org-v1- prefix") + } +} + +// TestCdUIDFromProvTokenInvalidCustomHostname proves an invalid +// --custom-hostname value is classified before any network attempt, with a +// message that names the field and format, and never leaks the provision +// token. +func TestCdUIDFromProvTokenInvalidCustomHostname(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldFetch, oldHostname := fetchResolverUIDFn, customHostname + t.Cleanup(func() { fetchResolverUIDFn, customHostname = oldFetch, oldHostname }) + + cdUID = "" + const secretToken = "org-v1-secret-token-42" + cdOrg = secretToken + customHostname = "foo@bar" + + called := false + fetchResolverUIDFn = func(context.Context, *controld.UtilityOrgRequest, string, bool) (*controld.ResolverConfig, error) { + called = true + return nil, nil + } + + if got := cdUIDFromProvToken(); got != "" { + t.Errorf("cdUIDFromProvToken() = %q, want empty on failure", got) + } + if called { + t.Error("fetchResolverUIDFn was called; invalid custom hostname must be rejected before any network attempt") + } + if *exitCode != provisionExitCodeForCode[provisionCodeCustomHostnameInvalid] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeCustomHostnameInvalid]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeCustomHostnameInvalid) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeCustomHostnameInvalid) + } + if !strings.Contains(r.Message, "allowed format") { + t.Errorf("message = %q, want it to contain the allowed format", r.Message) + } + if strings.Contains(r.Message, secretToken) { + t.Errorf("provision token leaked into result message: %q", r.Message) + } +} + +// TestCdUIDFromProvTokenFoldableHostnameLogsNotice proves a hostname that +// ctrld accepts, but ControlD's device-name formatting would fold or strip, +// gets a notice - not a failure - and provisioning still proceeds. +func TestCdUIDFromProvTokenFoldableHostnameLogsNotice(t *testing.T) { + _, _ = stubProvisionGlobals(t) + logBuf := captureMainLog(t) + oldFetch, oldHostname := fetchResolverUIDFn, customHostname + t.Cleanup(func() { fetchResolverUIDFn, customHostname = oldFetch, oldHostname }) + + cdUID = "" + cdOrg = "org-v1-abcdef123456" + customHostname = "foo.bar" + + fetchResolverUIDFn = func(context.Context, *controld.UtilityOrgRequest, string, bool) (*controld.ResolverConfig, error) { + return &controld.ResolverConfig{UID: "resolved-uid"}, nil + } + + if got := cdUIDFromProvToken(); got != "resolved-uid" { + t.Errorf("cdUIDFromProvToken() = %q, want resolved-uid", got) + } + if !strings.Contains(logBuf.String(), "foo.bar") { + t.Error("expected a notice naming the foldable hostname") + } +} + +func TestValidateInterceptModeFlag(t *testing.T) { + oldMode := interceptMode + t.Cleanup(func() { interceptMode = oldMode }) + + t.Run("valid values proceed", func(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + for _, mode := range []string{"", "off", "dns", "hard"} { + interceptMode = mode + if !validateInterceptModeFlag(mode) { + t.Errorf("validateInterceptModeFlag(%q) = false, want true", mode) + } + if *exitCode != -1 { + t.Errorf("mode %q: provisionExit called with %d, want no exit", mode, *exitCode) + } + } + }) + + t.Run("invalid value is classified", func(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + interceptMode = "fds" + if validateInterceptModeFlag("fds") { + t.Error("validateInterceptModeFlag(\"fds\") = true, want false") + } + if *exitCode != provisionExitCodeForCode[provisionCodeInterceptModeInvalid] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeInterceptModeInvalid]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeInterceptModeInvalid) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeInterceptModeInvalid) + } + for _, want := range []string{"off", "dns", "hard"} { + if !strings.Contains(r.Message, want) { + t.Errorf("message = %q, want it to contain %q", r.Message, want) + } + } + }) +} + +// TestCheckStrFlagEmptyClassifiesEmptyCdOrg proves an explicit empty --cd-org +// is classified as a malformed provisioning token rather than a bare fatal. +func TestCheckStrFlagEmptyClassifiesEmptyCdOrg(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + cmd := &cobra.Command{} + cmd.Flags().String(cdOrgFlagName, "", "") + if err := cmd.Flags().Set(cdOrgFlagName, ""); err != nil { + t.Fatal(err) + } + + if checkStrFlagEmpty(cmd, cdOrgFlagName) { + t.Error("checkStrFlagEmpty() = true, want false for an explicit empty --cd-org") + } + if *exitCode != provisionExitCodeForCode[provisionCodeProvisionTokenMalformed] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeProvisionTokenMalformed]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeProvisionTokenMalformed) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeProvisionTokenMalformed) + } + if r.Stage != string(provisionStageInput) { + t.Errorf("stage = %q, want %q", r.Stage, provisionStageInput) + } + if !strings.Contains(r.Message, cdOrgFlagName) { + t.Errorf("message = %q, want it to name --%s", r.Message, cdOrgFlagName) + } +} + +// TestCheckStrFlagEmptyProceedsWhenNotChangedOrNotEmpty proves the two cases +// that must not classify: the flag was never set, and it was set to a +// non-empty value. +func TestCheckStrFlagEmptyProceedsWhenNotChangedOrNotEmpty(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + + t.Run("flag never set", func(t *testing.T) { + cmd := &cobra.Command{} + cmd.Flags().String(cdOrgFlagName, "", "") + if !checkStrFlagEmpty(cmd, cdOrgFlagName) { + t.Error("checkStrFlagEmpty() = false, want true when the flag was never set") + } + }) + + t.Run("flag set to a non-empty value", func(t *testing.T) { + cmd := &cobra.Command{} + cmd.Flags().String(cdOrgFlagName, "", "") + if err := cmd.Flags().Set(cdOrgFlagName, "org-v1-abc"); err != nil { + t.Fatal(err) + } + if !checkStrFlagEmpty(cmd, cdOrgFlagName) { + t.Error("checkStrFlagEmpty() = false, want true for a non-empty value") + } + }) + + if *exitCode != -1 { + t.Errorf("provisionExit called with %d, want no exit", *exitCode) + } +} + +// TestValidateCdAndNextDNSFlagsClassifiesConflict proves --cd or --cd-org +// combined with --nextdns is classified as INVALID_FLAG_COMBINATION, naming +// every flag involved. +func TestValidateCdAndNextDNSFlagsClassifiesConflict(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldCdUID, oldCdOrg, oldNextdns := cdUID, cdOrg, nextdns + t.Cleanup(func() { cdUID, cdOrg, nextdns = oldCdUID, oldCdOrg, oldNextdns }) + + t.Run("non-conflicting combinations proceed", func(t *testing.T) { + cases := []struct{ cdUID, cdOrg, nextdns string }{ + {"", "", ""}, + {"uid123", "", ""}, + {"", "org-v1-abc", ""}, + {"", "", "nextdns-id"}, + } + for _, tc := range cases { + cdUID, cdOrg, nextdns = tc.cdUID, tc.cdOrg, tc.nextdns + if !validateCdAndNextDNSFlags() { + t.Errorf("validateCdAndNextDNSFlags() = false for cdUID=%q cdOrg=%q nextdns=%q, want true", tc.cdUID, tc.cdOrg, tc.nextdns) + } + } + if *exitCode != -1 { + t.Errorf("provisionExit called with %d, want no exit", *exitCode) + } + }) + + t.Run("cd-org with nextdns is classified", func(t *testing.T) { + cdUID, cdOrg, nextdns = "", "org-v1-abc", "nextdns-id" + if validateCdAndNextDNSFlags() { + t.Error("validateCdAndNextDNSFlags() = true, want false") + } + if *exitCode != provisionExitCodeForCode[provisionCodeInvalidFlagCombination] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeInvalidFlagCombination]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeInvalidFlagCombination) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeInvalidFlagCombination) + } + if r.Stage != string(provisionStageInput) { + t.Errorf("stage = %q, want %q", r.Stage, provisionStageInput) + } + for _, want := range []string{cdUidFlagName, cdOrgFlagName, nextdnsFlagName} { + if !strings.Contains(r.Message, want) { + t.Errorf("message = %q, want it to name --%s", r.Message, want) + } + } + }) + + t.Run("cd with nextdns is classified", func(t *testing.T) { + cdUID, cdOrg, nextdns = "uid123", "", "nextdns-id" + if validateCdAndNextDNSFlags() { + t.Error("validateCdAndNextDNSFlags() = true, want false") + } + if *exitCode != provisionExitCodeForCode[provisionCodeInvalidFlagCombination] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeInvalidFlagCombination]) + } + }) +} + +// TestValidateCdUpstreamProtocol proves an invalid --proto value is +// classified only once --cd is in play, and that notify runs when given. +func TestValidateCdUpstreamProtocol(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + oldCdUID, oldProto := cdUID, cdUpstreamProto + t.Cleanup(func() { cdUID, cdUpstreamProto = oldCdUID, oldProto }) + + t.Run("no --cd proceeds regardless of protocol", func(t *testing.T) { + cdUID = "" + cdUpstreamProto = "garbage" + if !validateCdUpstreamProtocol(nil) { + t.Error("validateCdUpstreamProtocol(nil) = false, want true when --cd is not set") + } + if *exitCode != -1 { + t.Errorf("provisionExit called with %d, want no exit", *exitCode) + } + }) + + t.Run("valid protocols proceed", func(t *testing.T) { + cdUID = "uid123" + for _, proto := range []string{ctrld.ResolverTypeDOH, ctrld.ResolverTypeDOH3} { + cdUpstreamProto = proto + if !validateCdUpstreamProtocol(nil) { + t.Errorf("validateCdUpstreamProtocol(nil) = false for proto %q, want true", proto) + } + } + if *exitCode != -1 { + t.Errorf("provisionExit called with %d, want no exit", *exitCode) + } + }) + + t.Run("invalid protocol is classified and notifies", func(t *testing.T) { + cdUID = "uid123" + cdUpstreamProto = "quic" + notified := false + if validateCdUpstreamProtocol(func() { notified = true }) { + t.Error("validateCdUpstreamProtocol() = true, want false") + } + if !notified { + t.Error("notify not called") + } + if *exitCode != provisionExitCodeForCode[provisionCodeInvalidFlagCombination] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeInvalidFlagCombination]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeInvalidFlagCombination) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeInvalidFlagCombination) + } + if !strings.Contains(r.Message, "quic") || !strings.Contains(r.Message, "doh") { + t.Errorf("message = %q, want it to name the given and allowed values", r.Message) + } + }) +} diff --git a/cmd/cli/provision_result.go b/cmd/cli/provision_result.go index bd513bed..606b8a41 100644 --- a/cmd/cli/provision_result.go +++ b/cmd/cli/provision_result.go @@ -8,6 +8,8 @@ import ( "strings" "time" "unicode/utf8" + + "github.com/rs/zerolog" ) // A terminal provisioning failure reports the same stable code on three @@ -20,6 +22,7 @@ import ( type provisionStage string const ( + provisionStageInput provisionStage = "input" provisionStageBootstrap provisionStage = "bootstrap" provisionStageListener provisionStage = "listener" provisionStageService provisionStage = "service" @@ -28,50 +31,87 @@ const ( type provisionFailureCode string const ( - provisionCodeAPIUnreachable provisionFailureCode = "API_UNREACHABLE" - provisionCodeAPIRejected provisionFailureCode = "API_REJECTED" - provisionCodeAPIDeviceInvalid provisionFailureCode = "API_DEVICE_INVALID" - provisionCodeListenerBindFailed provisionFailureCode = "LISTENER_BIND_FAILED" - provisionCodeListenerAddrUnavail provisionFailureCode = "LISTENER_CONFIGURED_ADDR_UNAVAILABLE" - provisionCodeServiceInstall provisionFailureCode = "SERVICE_INSTALL_FAILED" - provisionCodeServiceStartFailed provisionFailureCode = "SERVICE_START_FAILED" - provisionCodeServiceSelfCheck provisionFailureCode = "SERVICE_SELFCHECK_FAILED" + provisionCodeProvisionTokenMalformed provisionFailureCode = "PROVISION_TOKEN_MALFORMED" + provisionCodeCustomHostnameInvalid provisionFailureCode = "CUSTOM_HOSTNAME_INVALID" + provisionCodeInterceptModeInvalid provisionFailureCode = "INTERCEPT_MODE_INVALID" + provisionCodeInvalidFlagCombination provisionFailureCode = "INVALID_FLAG_COMBINATION" + provisionCodeAPIUnreachable provisionFailureCode = "API_UNREACHABLE" + provisionCodeAPIRejected provisionFailureCode = "API_REJECTED" + provisionCodeAPIDeviceInvalid provisionFailureCode = "API_DEVICE_INVALID" + provisionCodeTokenInvalid provisionFailureCode = "TOKEN_INVALID" + provisionCodeTokenExpired provisionFailureCode = "TOKEN_EXPIRED" + provisionCodeTokenLimitReached provisionFailureCode = "TOKEN_LIMIT_REACHED" + provisionCodeTokenDisabled provisionFailureCode = "TOKEN_DISABLED" + provisionCodeListenerBindFailed provisionFailureCode = "LISTENER_BIND_FAILED" + provisionCodeListenerAddrUnavail provisionFailureCode = "LISTENER_CONFIGURED_ADDR_UNAVAILABLE" + provisionCodeServiceInstall provisionFailureCode = "SERVICE_INSTALL_FAILED" + provisionCodeServiceStartFailed provisionFailureCode = "SERVICE_START_FAILED" + provisionCodeServiceSelfCheck provisionFailureCode = "SERVICE_SELFCHECK_FAILED" + provisionCodeUnclassified provisionFailureCode = "UNCLASSIFIED" ) var allProvisionFailureCodes = []provisionFailureCode{ + provisionCodeProvisionTokenMalformed, + provisionCodeCustomHostnameInvalid, + provisionCodeInterceptModeInvalid, + provisionCodeInvalidFlagCombination, provisionCodeAPIUnreachable, provisionCodeAPIRejected, provisionCodeAPIDeviceInvalid, + provisionCodeTokenInvalid, + provisionCodeTokenExpired, + provisionCodeTokenLimitReached, + provisionCodeTokenDisabled, provisionCodeListenerBindFailed, provisionCodeListenerAddrUnavail, provisionCodeServiceInstall, provisionCodeServiceStartFailed, provisionCodeServiceSelfCheck, + provisionCodeUnclassified, } var provisionStageForCode = map[provisionFailureCode]provisionStage{ - provisionCodeAPIUnreachable: provisionStageBootstrap, - provisionCodeAPIRejected: provisionStageBootstrap, - provisionCodeAPIDeviceInvalid: provisionStageBootstrap, - provisionCodeListenerBindFailed: provisionStageListener, - provisionCodeListenerAddrUnavail: provisionStageListener, - provisionCodeServiceInstall: provisionStageService, - provisionCodeServiceStartFailed: provisionStageService, - provisionCodeServiceSelfCheck: provisionStageService, -} - -// Exit codes are grouped by stage (bootstrap 30-39, listener 40-49, service -// 50-59) so the exit code alone names the failed stage. 0-3 belong to -// "ctrld status" and 126 to the deactivation pin check; never reuse those. + provisionCodeProvisionTokenMalformed: provisionStageInput, + provisionCodeCustomHostnameInvalid: provisionStageInput, + provisionCodeInterceptModeInvalid: provisionStageInput, + provisionCodeInvalidFlagCombination: provisionStageInput, + provisionCodeAPIUnreachable: provisionStageBootstrap, + provisionCodeAPIRejected: provisionStageBootstrap, + provisionCodeAPIDeviceInvalid: provisionStageBootstrap, + provisionCodeTokenInvalid: provisionStageBootstrap, + provisionCodeTokenExpired: provisionStageBootstrap, + provisionCodeTokenLimitReached: provisionStageBootstrap, + provisionCodeTokenDisabled: provisionStageBootstrap, + provisionCodeListenerBindFailed: provisionStageListener, + provisionCodeListenerAddrUnavail: provisionStageListener, + provisionCodeServiceInstall: provisionStageService, + provisionCodeServiceStartFailed: provisionStageService, + provisionCodeServiceSelfCheck: provisionStageService, + provisionCodeUnclassified: provisionStageService, +} + +// Exit codes are grouped by stage (input 20-29, bootstrap 30-39, listener +// 40-49, service 50-59) so the exit code alone names the failed stage. 0-3 +// belong to "ctrld status" and 126 to the deactivation pin check; never +// reuse those. var provisionExitCodeForCode = map[provisionFailureCode]int{ - provisionCodeAPIUnreachable: 30, - provisionCodeAPIRejected: 31, - provisionCodeAPIDeviceInvalid: 32, - provisionCodeListenerBindFailed: 41, - provisionCodeListenerAddrUnavail: 42, - provisionCodeServiceInstall: 51, - provisionCodeServiceStartFailed: 52, - provisionCodeServiceSelfCheck: 53, + provisionCodeProvisionTokenMalformed: 21, + provisionCodeCustomHostnameInvalid: 22, + provisionCodeInterceptModeInvalid: 23, + provisionCodeInvalidFlagCombination: 24, + provisionCodeAPIUnreachable: 30, + provisionCodeAPIRejected: 31, + provisionCodeAPIDeviceInvalid: 32, + provisionCodeTokenInvalid: 33, + provisionCodeTokenExpired: 34, + provisionCodeTokenLimitReached: 35, + provisionCodeTokenDisabled: 36, + provisionCodeListenerBindFailed: 41, + provisionCodeListenerAddrUnavail: 42, + provisionCodeServiceInstall: 51, + provisionCodeServiceStartFailed: 52, + provisionCodeServiceSelfCheck: 53, + provisionCodeUnclassified: 59, } const ( @@ -213,7 +253,11 @@ func writeProvisionResult(r *provisionResult) error { } func readProvisionResult() (*provisionResult, error) { - buf, err := os.ReadFile(provisionResultPath()) + return readProvisionResultAt(provisionResultPath()) +} + +func readProvisionResultAt(path string) (*provisionResult, error) { + buf, err := os.ReadFile(path) if err != nil { return nil, err } @@ -245,3 +289,24 @@ func failProvision(r *provisionResult, notify func()) { } provisionExit(r.ExitCode) } + +// failProvisionUnclassified persists an UNCLASSIFIED result (service stage, +// exit 59) and exits. It is the fallback for a terminal path that predates a +// stable code - a config unmarshal, a file-system or environment failure - +// so support still gets a result file and a stage-scoped exit code instead +// of a bare crash with nothing to read. No terminal path on the +// provisioning boundary may bypass this or an existing classified code. +func failProvisionUnclassified(message string, notify func()) { + failProvision(newProvisionResult(provisionCodeUnclassified, message, nil, provisionSecrets()...), notify) +} + +// failRunUnclassified logs msg on ev, then fails provisioning as UNCLASSIFIED +// and unblocks a waiting "ctrld start" via notify (nil if none). Callers +// return immediately after this call: provisionExit is stubbed out under +// test, so nothing stops execution from falling through otherwise. ev +// carries whatever the caller already chained onto it (for example .Err()), +// so each call site keeps its own log fields. +func failRunUnclassified(ev *zerolog.Event, msg string, notify func()) { + ev.Msg(msg) + failProvisionUnclassified(msg, notify) +} diff --git a/cmd/cli/provision_result_test.go b/cmd/cli/provision_result_test.go index 30039041..f7f95d3c 100644 --- a/cmd/cli/provision_result_test.go +++ b/cmd/cli/provision_result_test.go @@ -20,8 +20,21 @@ func overrideProvisionResultPath(t *testing.T) string { return path } +// The result file must obey the same homedir override as the log and the +// config file, so a daemon started with --homedir writes it next to them. +func TestProvisionResultPathHonorsHomedir(t *testing.T) { + old := homedir + homedir = t.TempDir() + t.Cleanup(func() { homedir = old }) + want := filepath.Join(homedir, provisionResultFileName) + if got := provisionResultPath(); got != want { + t.Errorf("provisionResultPath() = %q, want %q", got, want) + } +} + func TestProvisionCodesMapToOneStageAndInRangeExit(t *testing.T) { stageRanges := map[provisionStage][2]int{ + provisionStageInput: {20, 29}, provisionStageBootstrap: {30, 39}, provisionStageListener: {40, 49}, provisionStageService: {50, 59}, @@ -55,8 +68,8 @@ func TestProvisionCodesMapToOneStageAndInRangeExit(t *testing.T) { } seenExits[exit] = code } - if len(allProvisionFailureCodes) != 8 { - t.Errorf("expected 8 codes, got %d", len(allProvisionFailureCodes)) + if len(allProvisionFailureCodes) != 17 { + t.Errorf("expected 17 codes, got %d", len(allProvisionFailureCodes)) } } @@ -166,14 +179,17 @@ func TestFailureCodeDocTableMatchesConstants(t *testing.T) { t.Fatalf("could not read the failure-code doc: %v", err) } doc := string(buf) + // Package-stage rows document identifiers scripts/pkg/postinstall emits + // on its own; they are not in this binary's failure-code registry, so + // they sit outside the one-row-per-code count below. rows := 0 for _, line := range strings.Split(doc, "\n") { - if strings.HasPrefix(line, "| `") { + if strings.HasPrefix(line, "| `") && !strings.Contains(line, "| package |") { rows++ } } if rows != len(allProvisionFailureCodes) { - t.Errorf("doc table has %d code rows, want %d", rows, len(allProvisionFailureCodes)) + t.Errorf("doc table has %d non-package code rows, want %d", rows, len(allProvisionFailureCodes)) } for _, code := range allProvisionFailureCodes { row := "| `" + string(code) + "` | " + string(provisionStageForCode[code]) + " | " + strconv.Itoa(provisionExitCodeForCode[code]) + " |" diff --git a/cmd/cli/provision_terminal_paths_test.go b/cmd/cli/provision_terminal_paths_test.go new file mode 100644 index 00000000..7e592221 --- /dev/null +++ b/cmd/cli/provision_terminal_paths_test.go @@ -0,0 +1,244 @@ +package cli + +import ( + "errors" + "net" + "strings" + "testing" + "time" +) + +// TestFailProvisionUnclassified pins the fallback every terminal path on the +// provisioning boundary uses when it predates a stable code: it persists +// UNCLASSIFIED, calls notify, and exits at the code's exit number. +func TestFailProvisionUnclassified(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + notified := false + + failProvisionUnclassified("something failed with no dedicated code", func() { notified = true }) + + if !notified { + t.Error("notify not called") + } + if *exitCode != provisionExitCodeForCode[provisionCodeUnclassified] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeUnclassified]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeUnclassified) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeUnclassified) + } + if r.Stage != string(provisionStageService) { + t.Errorf("stage = %q, want %q", r.Stage, provisionStageService) + } + if !strings.Contains(r.Message, "something failed") { + t.Errorf("message = %q, want it to contain the given message", r.Message) + } +} + +// TestNotifyExitToLogServer pins the method every run()-side failure uses to +// unblock a waiting "ctrld start": it writes msgExit to the log connection +// exactly when one is set, and is a no-op otherwise. +func TestNotifyExitToLogServer(t *testing.T) { + t.Run("nil connection is a no-op", func(t *testing.T) { + p := &prog{} + p.notifyExitToLogServer() // must not panic + }) + + t.Run("writes msgExit to a set connection", func(t *testing.T) { + server, client := net.Pipe() + t.Cleanup(func() { _ = server.Close(); _ = client.Close() }) + p := &prog{logConn: client} + + received := make(chan string, 1) + go func() { + buf := make([]byte, 64) + n, _ := server.Read(buf) + received <- string(buf[:n]) + }() + + p.notifyExitToLogServer() + + select { + case got := <-received: + if got != msgExit { + t.Errorf("wrote %q, want %q", got, msgExit) + } + case <-time.After(2 * time.Second): + t.Fatal("timed out waiting for notifyExitToLogServer to write") + } + }) +} + +// TestReportServeDNSFailure covers the one prog.go path where a listener +// bound successfully (LISTENER_* already ruled that out) but the proxy +// itself failed to start serving - a distinct failure with no dedicated +// code. +func TestReportServeDNSFailure(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + server, client := net.Pipe() + t.Cleanup(func() { _ = server.Close(); _ = client.Close() }) + go func() { + buf := make([]byte, 64) + _, _ = server.Read(buf) + }() + p := &prog{logConn: client} + + p.reportServeDNSFailure("0", errors.New("bind lost mid-flight")) + + if *exitCode != provisionExitCodeForCode[provisionCodeUnclassified] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeUnclassified]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeUnclassified) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeUnclassified) + } + if !strings.Contains(r.Message, "listener.0") { + t.Errorf("message = %q, want it to name the listener", r.Message) + } +} + +// TestRefuseFallbackFatalDefault covers the production refuseFallbackFatal +// closure (not the test-harness override other tests install): it +// classifies UNCLASSIFIED and notifies through the given prog's log +// connection. +func TestRefuseFallbackFatalDefault(t *testing.T) { + exitCode, _ := stubProvisionGlobals(t) + server, client := net.Pipe() + t.Cleanup(func() { _ = server.Close(); _ = client.Close() }) + go func() { + buf := make([]byte, 64) + _, _ = server.Read(buf) + }() + p := &prog{logConn: client} + + refuseFallbackFatal(p, "cannot fall back: %s", "port 53 is unavailable") + + if *exitCode != provisionExitCodeForCode[provisionCodeUnclassified] { + t.Errorf("exit = %d, want %d", *exitCode, provisionExitCodeForCode[provisionCodeUnclassified]) + } + r, err := readProvisionResult() + if err != nil { + t.Fatalf("no provision result written: %v", err) + } + if r.Code != string(provisionCodeUnclassified) { + t.Errorf("code = %q, want %q", r.Code, provisionCodeUnclassified) + } + if !strings.Contains(r.Message, "port 53 is unavailable") { + t.Errorf("message = %q, want it to contain the formatted detail", r.Message) + } +} + +// terminalPathCase documents one terminal path this lane audited under +// "ctrld start --cd-org": where it lives, what triggers it, and the code it +// now reports. reachable names a test that exercises the real call site +// end-to-end (through the seams this package already exposes); a path deep +// inside the start command's Run closure / run() with no seam to trigger it +// in isolation is documented here with reachable == "" and relies on the +// shared helper tests above (TestFailProvisionUnclassified, +// TestNotifyExitToLogServer, TestReportServeDNSFailure, +// TestRefuseFallbackFatalDefault) to prove the wiring it calls into. +type terminalPathCase struct { + file string + scenario string + code provisionFailureCode + reachable string // name of the test proving this exact site, or "" if only the shared helper is exercised +} + +// TestProvisioningTerminalPathAudit is the committed enumeration required for +// "ctrld start --cd-org": every Fatal/os.Exit found by re-deriving the +// requirement (grepping commands.go's initStartCmd, cli.go's run() and its +// callers cdUIDFromProvToken/handleAPIPreflightFailure, and prog.go's +// pre-daemonization paths), reconciled against a code. None of them may +// reach a bare Fatal/os.Exit any more; each row's code is asserted against +// the known contract maps so a typo'd code name fails this test. +func TestProvisioningTerminalPathAudit(t *testing.T) { + cases := []terminalPathCase{ + // commands.go initStartCmd (parent "ctrld start" process). v1.0 keeps + // what master splits into commands_service_start.go in the same + // commands.go as every other command. + {"commands.go (initStartCmd)", "--intercept-mode is not off/dns/hard", provisionCodeInterceptModeInvalid, "TestValidateInterceptModeFlag"}, + {"commands.go (initStartCmd) / cli.go RunCobraCommand", "explicit empty --cd-org", provisionCodeProvisionTokenMalformed, "TestCheckStrFlagEmptyClassifiesEmptyCdOrg"}, + {"commands.go (initStartCmd)", "--cd/--cd-org used together with --nextdns", provisionCodeInvalidFlagCombination, "TestValidateCdAndNextDNSFlagsClassifiesConflict"}, + {"commands.go (initStartCmd)", "newService fails building the service handle", provisionCodeUnclassified, "TestStartCommandClassifiesServiceInitFailure"}, + {"commands.go (initStartCmd) / cli.go run()", "--proto is not doh/doh3 once --cd is set", provisionCodeInvalidFlagCombination, "TestValidateCdUpstreamProtocol"}, + {"commands.go (initStartCmd)", "removeServiceFlag fails while upgrading an existing service's intercept mode", provisionCodeUnclassified, ""}, + {"commands.go (initStartCmd)", "appendServiceFlag(\"--intercept-mode\") fails during the same upgrade", provisionCodeUnclassified, ""}, + {"commands.go (initStartCmd)", "appendServiceFlag(mode) fails during the same upgrade", provisionCodeUnclassified, ""}, + {"commands.go (initStartCmd)", "config unmarshal fails restarting an already-installed service", provisionCodeUnclassified, ""}, + {"commands.go (initStartCmd)", "doTasksE fails on a non-service-stage task restarting an existing service", provisionCodeUnclassified, ""}, + {"commands.go (initStartCmd)", "socketDir() fails restarting an existing service", provisionCodeUnclassified, ""}, + // v1.0-only: the DNS intercept-mode router abstraction (predates and is + // unrelated to issue-595) configures itself before a fresh install; + // already routed through failRunUnclassified, not a bare fatal. + {"commands.go (initStartCmd)", "router.ConfigureService fails configuring the router before a fresh install", provisionCodeUnclassified, ""}, + {"commands.go (initStartCmd)", "config unmarshal fails on a fresh install", provisionCodeUnclassified, ""}, + {"commands.go (initStartCmd)", "doTasksE fails on a non-service-stage task on a fresh install", provisionCodeUnclassified, ""}, + + // cli.go run() (the "ctrld run" child process --cd-org spawns, or a direct "ctrld run --cd-org" invocation) + {"cli.go run()", "called with a nil stop channel", provisionCodeUnclassified, ""}, + {"cli.go run()", "--daemon on windows", provisionCodeUnclassified, ""}, + {"cli.go run()", "newService fails building the background service handle", provisionCodeUnclassified, ""}, + {"cli.go run()", "readBase64Config fails", provisionCodeUnclassified, ""}, + {"cli.go run()", "config unmarshal fails", provisionCodeUnclassified, ""}, + {"cli.go run()", "network is not up", provisionCodeUnclassified, ""}, + // v1.0-only: same router abstraction as above, on the daemon side. + {"cli.go run()", "router.PreRun fails performing the router pre-run check", provisionCodeUnclassified, ""}, + {"cli.go run()", "writeConfigFile fails", provisionCodeUnclassified, ""}, + {"cli.go run()", "validateConfig fails", provisionCodeUnclassified, ""}, + {"cli.go run()", "os.Executable fails in daemon respawn", provisionCodeUnclassified, ""}, + {"cli.go run()", "os.Getwd fails in daemon respawn", provisionCodeUnclassified, ""}, + {"cli.go run()", "cmd.Start fails in daemon respawn", provisionCodeUnclassified, ""}, + + // prog.go (pre-daemonization: listener startup and DNS-intercept setup) + {"prog.go (p *prog) run()", "serveDNS fails after the listener already bound", provisionCodeUnclassified, "TestReportServeDNSFailure"}, + {"prog.go (p *prog) setDNS()", "DNS intercept fails and the interface-DNS fallback cannot reach a non-53 listener", provisionCodeUnclassified, "TestRefuseFallbackFatalDefault"}, + } + + for _, tc := range cases { + if _, ok := provisionStageForCode[tc.code]; !ok { + t.Errorf("%s: %s: code %s is not a known contract code", tc.file, tc.scenario, tc.code) + } + if tc.reachable != "" { + continue + } + t.Logf("documented (exercised only via shared helper wiring, not a standalone seam): %s: %s -> %s", tc.file, tc.scenario, tc.code) + } + + // Out of scope, left as bare Fatal/os.Exit deliberately, with reasons: + // + // - cli.go run(): stopCh nil check is still bare-Fatal-free (converted + // above) but is genuinely unreachable from any CLI invocation - the + // CLI always constructs a fresh channel. Converted anyway above for + // consistency, not because it is reachable. + // - prog.go (p *prog) setDNS() line ~962's own --intercept-mode check: + // explicitly left alone per T6 - it is the "runtime validation in + // prog.go" validInterceptMode's own doc comment calls out as distinct + // from "the early start command check" that got INTERCEPT_MODE_INVALID. + // - prog.go (p *prog) Stop() and commands.go initStartCmd's own + // deactivation-pin os.Exit: explicitly out of scope per the lane + // contract ("pin-check 126 untouched"). + // - cli.go: doValidateCdRemoteConfig's Fatal is not reachable from the + // --cd-org start path at all on v1.0 - the start command defers --cd + // validation to processCDFlags in the "ctrld run" child instead (see + // the comment in initStartCmd), a divergence from master that predates + // and is unrelated to this lane. General CLI/config-parsing helpers + // (readConfigFile, processNoConfigFlags, processListenFlag, + // readConfigWithNotice's userHomeDir failure): these predate and apply + // uniformly across every ctrld invocation mode (config-file mode, + // no-config mode, nextdns mode), not specifically to --cd-org + // provisioning. Out of scope for this lane's --cd-org enumeration. + // - checkStrFlagEmpty for --cd (cdUidFlagName) keeps its bare fatal: + // only the --cd-org case above is a provisioning-token value. + // - cli.go uninstall()'s bare fatal on a router re-configuration + // failure: the "ctrld uninstall" command, not "ctrld start --cd-org". + // + // No firewall-mode flag exists on v1.0, so master's INTERCEPT_MODE_INVALID- + // adjacent "--firewall-mode is not off/on" rows (parent and daemon-side) + // have no equivalent here. +} diff --git a/docs/provisioning-failure-codes.md b/docs/provisioning-failure-codes.md index 6d8b299d..41954bcd 100644 --- a/docs/provisioning-failure-codes.md +++ b/docs/provisioning-failure-codes.md @@ -13,9 +13,10 @@ stable code on three surfaces: `provisioning failed: stage= code= (exit )`. The macOS pkg `postinstall` extracts exactly this line into the installer log, so MDM consoles see it without any ctrld log configuration. -- **Exit code** — stage-scoped: bootstrap 30–39, listener 40–49, - service 50–59. Unrelated existing contracts are unchanged - (`ctrld status` exits 0–3; invalid deactivation pin exits 126). +- **Exit code** — stage-scoped: input 20–29, bootstrap 30–39, + listener 40–49, service 50–59. This does not change other existing + exit-code contracts: `ctrld status` still exits 0–3, and an invalid + deactivation pin still exits 126. A customer or administrator only needs to report the code (or the whole output line). The table below is the maintained support mapping; it must @@ -23,16 +24,109 @@ stay in sync with `cmd/cli/provision_result.go` and changes in the same MR. ## Codes +For every code below, first run `sudo ctrld diag` (`sudo ctrld diag --json` +gives a paste-safe, machine-readable copy). Get that output before you do +anything else. The command needs no running service. Without root, the +last-provisioning-result and service-state sections report permission +denied, so run it with `sudo`. It repeats the same stage, code, and message +the customer already reported. +The **Next action / evidence** column lists what to look at next. + | Code | Stage | Exit | Failure scenario | Next action / evidence | |---|---|---|---|---| -| `API_UNREACHABLE` | bootstrap | 30 | The Control D API could not be reached or answered with a retryable error (network failure, proxy interference, 5xx, timeout) and retries ran out. The service manager may retry the service later. | Check the device's network path to `api.controld.com` (DNS, proxy, firewall, captive portal). Ask for the result file's `message` and whether other TLS traffic works. | -| `API_REJECTED` | bootstrap | 31 | The API answered and permanently rejected the configuration (4xx other than 408/429): bad or revoked token, malformed request. ctrld exits without burning service-manager restarts because retrying cannot change the answer. | Verify the provision token / org configuration in the Control D dashboard. Re-push after fixing credentials. Evidence: HTTP status in the result file `message`. | -| `API_DEVICE_INVALID` | bootstrap | 32 | The API reports the device/resolver no longer exists (error code 40402). ctrld self-uninstalls its service because the identity is gone server-side. | Confirm the device was deleted or re-provisioned in the dashboard; re-provision with a current token. No local evidence needed beyond the code. | -| `LISTENER_BIND_FAILED` | listener | 41 | No listen address could be bound after all fallbacks (configured address, 0.0.0.0:53, localhost:53, port 5354, random) were exhausted. `detail.attempts` records each tried address with the UDP/TCP OS error, e.g. `address already in use` (another DNS service owns the port) or `can't assign requested address` (address not on any interface). | Read `detail.attempts`: `address already in use` → find the process owning the port (`sudo lsof -i :53 -nP`); `can't assign requested address` → the configured IP is not present on the device. Then fix the conflict or the listener config. | +| `PROVISION_TOKEN_MALFORMED` | input | 21 | The `--cd-org` value is not a provisioning code: it is empty, too short, too long, or it contains whitespace or control characters. ctrld checks this before any network call. A missing `org-v1-` prefix is not this code. That case only logs a warning, because legacy codes can lack the prefix. | Make sure that the provisioning code was copied in full, with no extra whitespace. The result file never echoes the value. | +| `CUSTOM_HOSTNAME_INVALID` | input | 22 | The `--custom-hostname` value fails ctrld's own hostname rule (`validHostname`): it is not 3 to 64 characters, or it is not in RFC1123 hostname format. ctrld checks this before any network call. The API itself does not reject a bad hostname during provisioning. When ControlD registers the device name, it folds or strips characters like space, `+`, and `.`. Because of this, a value ctrld accepts can still register under an adjusted name. That case only logs a notice, not a failure. | Fix `--custom-hostname` following the message: it names the offending characters and the allowed format. | +| `INTERCEPT_MODE_INVALID` | input | 23 | The `--intercept-mode` value is not one of `off`, `dns`, or `hard`. ctrld checks this before it installs the service. | Re-run with a valid `--intercept-mode` value. | +| `INVALID_FLAG_COMBINATION` | input | 24 | `--cd` or `--cd-org` is used together with `--nextdns`, or `--proto` is set to anything other than `doh`/`doh3` while `--cd` is in play. ctrld checks this before any network call or service install. | Re-run without the conflicting flag, or fix the invalid value. The message names the exact flags involved. | +| `API_UNREACHABLE` | bootstrap | 30 | The Control D API could not be reached or answered with a retryable error (network failure, proxy interference, 5xx, timeout) and retries ran out. The service manager may retry the service later. | Make sure that the device has network access to `api.controld.com` (DNS, proxy, firewall, captive portal). `ctrld diag`'s `api_reachability` block already ran this same probe. Ask whether other TLS traffic works. | +| `API_REJECTED` | bootstrap | 31 | The API answered and permanently rejected the configuration (4xx other than 408/429): bad or revoked token, malformed request. ctrld exits without burning service-manager restarts because retrying cannot change the answer. This is also the fallback for the four `TOKEN_*` codes below. Those codes need the API response to carry `error.metadata.reason`. An API deployment that does not yet populate that field reports every provisioning-code rejection as `API_REJECTED` instead. | Verify the provision token / org configuration in the Control D dashboard. Re-push after fixing credentials. Evidence: HTTP status in the result file `message`. | +| `API_DEVICE_INVALID` | bootstrap | 32 | The API reports the device/resolver no longer exists (error code 40402). When the daemon's own bootstrap preflight discovers this, it self-uninstalls, because the identity is gone server-side. A direct install-time config fetch can also hit this code before the service exists. On that path there is nothing to uninstall yet. | Confirm the device was deleted or re-provisioned in the dashboard; re-provision with a current token. No local evidence needed beyond the code. | +| `TOKEN_INVALID` | bootstrap | 33 | The API rejected the `--cd-org` provisioning code with reason `token_invalid`: the code is not recognized. This code needs the deployed API to send `error.metadata.reason: "token_invalid"`. See `API_REJECTED` above for the fallback when it does not. | Make sure that the code is correct. Re-enter it exactly as given. | +| `TOKEN_EXPIRED` | bootstrap | 34 | The API rejected the `--cd-org` provisioning code with reason `token_expired`. This has the same API-reason dependency as `TOKEN_INVALID`. | Get a new provisioning code from your administrator. | +| `TOKEN_LIMIT_REACHED` | bootstrap | 35 | The API rejected the `--cd-org` provisioning code with reason `token_limit_reached`: it reached its device limit. This has the same API-reason dependency as `TOKEN_INVALID`. | Free up a device slot or use a different provisioning code. | +| `TOKEN_DISABLED` | bootstrap | 36 | The API rejected the `--cd-org` provisioning code with reason `token_disabled`: the code was invalidated. This has the same API-reason dependency as `TOKEN_INVALID`. | Download a profile from an active provisioning code. | +| `LISTENER_BIND_FAILED` | listener | 41 | No listen address could be bound after all fallbacks (configured address, 0.0.0.0:53, localhost:53, random) were exhausted. `detail.attempts` records each tried address with the UDP/TCP OS error, e.g. `address already in use` (another DNS service owns the port) or `can't assign requested address` (address not on any interface). | Read `detail.attempts`: `address already in use` → find the process owning the port (`sudo lsof -i :53 -nP`); `can't assign requested address` → the configured IP is not present on the device. Then fix the conflict or the listener config. | | `LISTENER_CONFIGURED_ADDR_UNAVAILABLE` | listener | 42 | An explicitly configured listener address could not be bound and configuration checks forbid falling back to another address, or (macOS intercept mode) the required explicit address is unavailable. | The configured `ip:port` in the listener config is wrong for this device or occupied. Verify the address exists on an interface and nothing else binds it; correct the config rather than expecting fallback. | | `SERVICE_INSTALL_FAILED` | service | 51 | The OS service manager refused to install the service (launchd/systemd/SCM registration failed). | Check OS-level constraints: permissions/elevation, MDM policy blocking daemon installation, corrupted previous install. Evidence: result file `message` (service manager error), plus `launchctl print system/ctrld` / `systemctl status ctrld` / SCM state. | | `SERVICE_START_FAILED` | service | 52 | The service installed but the service manager could not start it. | Check the service manager's own log for the start error, then the ctrld home dir `ctrld.log`. Often permissions or a binary quarantined by security tooling. | -| `SERVICE_SELFCHECK_FAILED` | service | 53 | The service started but never became healthy: no fresher failure was reported by the daemon, and the post-install DNS self-check failed. The just-installed service is rolled back (uninstalled). If the daemon itself recorded a more specific failure (e.g. a listener code), that code is reported instead of this one. | Ask for the drained service log printed by `ctrld start` and the result file. If the service was running but unreachable, check host firewall rules intercepting DNS to the listener. | +| `SERVICE_SELFCHECK_FAILED` | service | 53 | The service started but never became healthy: no fresher failure was reported by the daemon, and the post-install DNS self-check failed. On a fresh install or an upgrade, the just-installed service is rolled back (uninstalled). A restart of an already installed service keeps that service installed. If the daemon itself recorded a more specific failure (e.g. a listener code), that code is reported instead of this one. | Ask for the drained service log printed by `ctrld start` and the result file. If the service was running but unreachable, check host firewall rules intercepting DNS to the listener. | +| `UNCLASSIFIED` | service | 59 | This code covers terminal failures on the provisioning boundary that predate a dedicated code. Examples include a config unmarshal, and a file-system or environment failure such as writing the config file, reading `socketDir`, or respawning as a daemon. Other examples are a service-argument update failing mid-upgrade, and a router pre-run or configuration failure. A further example is a DNS-intercept failure the interface-DNS fallback cannot safely take over from. Every one of these used to be a bare crash with nothing to read. Now they all persist a result file. | Read the result file `message`: it names the specific operation that failed and the underlying OS error. When you report it, treat this the same as any other stage-scoped failure. | + +The macOS installer script (`scripts/pkg/postinstall`) emits the package rows +below, not ctrld itself, before ctrld ever runs. They only ever exit 1 (the +script's own exit code), and they never appear in `provision_result.json`. +The same script also reuses `PROVISION_TOKEN_MALFORMED` and +`CUSTOM_HOSTNAME_INVALID` from the table above, for its own pre-flight +checks on the managed-prefs `ProvisionToken`/`CustomHostname` values. It +uses the same rule, and also logs it as `stage=package (exit 1)`. So those +two codes can appear with either stage, depending on which side rejected +the value. + +| `PROFILE_PREFS_MISSING` | package | 1 | The MDM configuration profile (managed prefs domain `com.controld.ctrld`) has no `ProvisionToken`, or the value is empty. This shows up after the postinstall script's wait loop ends: 12 attempts, 10 seconds apart, about 2 minutes. | Scope the `com.controld.ctrld` configuration profile to the device. If the profile is already on the device, set its `ProvisionToken` value. Then reinstall the package, or run `sudo ctrld start --cd-org=` by hand. | +| `INTERCEPT_MODE_INVALID` | package | 1 | On a fresh install, the profile's `InterceptMode` is set, but it is not one of the managed-prefs values (empty, `standard`, `intercept-dns`). A fresh install fails visibly, instead of silently falling back, because a managed deployment must not provision into an unintended mode. On an upgrade, the same bad value only logs a warning and keeps the existing service mode. So a bad profile edit cannot break an already-working fleet. | Fix `InterceptMode` in the profile to one of the allowed values. Then reinstall the package. | +| `SERVICE_RELOAD_FAILED` | package | 1 | An upgrade reused the existing service mode (no managed `InterceptMode` override), and `launchctl load` on the installed plist failed. | Run `sudo launchctl load /Library/LaunchDaemons/ctrld.plist` by hand, and read the error it gives. | +| `TEMP_FILE_UNAVAILABLE` | package | 1 | The script failed to create the private temp file it uses to capture ctrld's output (`/tmp` or `TMPDIR` is full or unwritable). Because ctrld never runs, the script has no diagnostics to add. | Free space in `/tmp` or `TMPDIR`, or fix its permissions. Then reinstall the package. | + +## Customer-visible message shape + +Every failure shows the same two-part shape: the fixed identifier line, then +one free-text line specific to the code. Neither part ever contains the +provision token, the resolver/device ID, or config contents. + +- **Input stage** — names the offending flag or value and the rule it + broke. For example: `--cd-org provisioning code is malformed: must be + 6-64 characters with no whitespace or control characters`. For + `CUSTOM_HOSTNAME_INVALID`, the message names the specific offending + characters and the allowed format. +- **Bootstrap, `TOKEN_*`** — one fixed sentence per code (see the table + above). For example: `the provisioning code has expired; get a new + provisioning code from your administrator`. +- **Bootstrap, `API_UNREACHABLE`/`API_REJECTED`/`API_DEVICE_INVALID`** — + When the failure surfaces during a running install's periodic resolver + refresh, the message is `ControlD API rejected this configuration (HTTP + status )`. When it surfaces during the initial provisioning call, the + message is `provision token exchange failed: ` or `failed to fetch + resolver config: `. Neither ever includes the request or response + body. +- **Listener** — names the address or addresses tried. `detail.attempts` in + the result file carries the OS error for each address and protocol. +- **Service** — For `SERVICE_INSTALL_FAILED` or `SERVICE_START_FAILED`, the + message is ` failed: ` (`` is `Install` or `Start`). + When the daemon itself reported no fresher failure, `SERVICE_SELFCHECK_FAILED` + shows one of three messages, depending on what the self-check saw. When + there is no error and the service is not running, the message is `ctrld + service did not pass its post-start self-check`. When the test query + itself errored, the message is `An error occurred while performing test + query: `. When the service was running but the query silently + failed, the message is `ctrld service was running, but a DNS query could + not be sent to its listener; check firewall rules + blocking/intercepting/redirecting DNS queries`. +- **Package** — the plain-language second line the installer script logs + right after the identifier line. Each package row's *Next action* cell + above quotes this line verbatim, because the script has no separate + result file to carry it. + +## Where each code surfaces + +- **Every ctrld-native code** (every row above except `package`): the CLI + output / persisted `ctrld.log`, `provision_result.json`, and `ctrld diag` + (which reads that same file). +- **Package-stage codes** (`PROFILE_PREFS_MISSING`, `SERVICE_RELOAD_FAILED`, + `TEMP_FILE_UNAVAILABLE`, the package's own `INTERCEPT_MODE_INVALID` check + on the managed-prefs value, and the package pre-flight's reused + `PROVISION_TOKEN_MALFORMED` and `CUSTOM_HOSTNAME_INVALID`): only + `/var/log/install.log`. ctrld never ran, so no result file exists yet. + `sudo ctrld diag` run right after this reports "none recorded" for the last + provisioning result. That is still useful: it shows that the package + script's own gate rejected the profile before ctrld had a chance to run. +- **Any ctrld-native code can also reach `/var/log/install.log`.** The + postinstall script invokes ctrld in two cases: to provision on a fresh + install, or to reapply a managed intercept mode on an upgrade. Either + time, it greps ctrld's own identifier line out of the captured output. + It re-logs that line, and appends `(ctrld exit )`. So a bootstrap or + listener code, not only the three package-only codes, can appear in the + MDM install log. Right after any installer-script failure, it also + appends a fresh `sudo ctrld diag` run, each line prefixed `diag:`. ## Reading the result file @@ -50,12 +144,37 @@ The file sits in the same directory as the persisted internal log (`ctrld.log`) for the user the service runs as. On a healthy install the file is absent. +## Support: `ctrld diag` + +Run `sudo ctrld diag` (or `sudo ctrld diag --json`) to collect the facts +support needs for a provisioning ticket, in one copy-paste-safe command. It +collects the client version and the MDM-managed preferences (macOS only, +the token reported only as present or absent, an empty value counts as +absent). It also collects the last provisioning result, the service state, +and whether the Control D API is reachable. It needs no config. It also +runs without root, but then the last-provisioning-result and the +service-state sections report permission denied, and the text report names +the step that gets the rights it needs. Even when it finds a failure, it +still exits 0: that failure is part of the report, not a command failure. +It never asks for or prints the provisioning token itself. + +The `provision_result.status` field in the JSON report is one of `none` +(no result file), `recorded`, `untrusted` (a file that fails the contract +check), `unreadable` (permission denied), or `corrupt` (a file that could +not be parsed). + ## Rules for maintainers - Codes are append-only once released. Never rename, renumber, or reuse a code or exit number; add a new one and note the deprecation here. +- API rejection reasons are append-only too: `tokenFailureCodeForReason` + maps a fixed set of `error.metadata.reason` strings to their `TOKEN_*` + code. An unrecognized or absent reason falls back to `API_REJECTED` (see + `apiFailureCode`). So a new reason string does nothing until both a + matching code and a row here exist. - Every code added in `cmd/cli/provision_result.go` needs a row here in the same MR. Tests enforce the code/stage/exit maps and that this table has - exactly one row per code. + exactly one `stage != package` row per ctrld code. The `package` rows + belong to `scripts/pkg/postinstall` and are not part of that count. - Detail must stay bounded and free of secrets: the constructor strips the provision token and cd UID and caps sizes; do not bypass it. diff --git a/internal/controld/config.go b/internal/controld/config.go index 4b7ad07b..5efbcc44 100644 --- a/internal/controld/config.go +++ b/internal/controld/config.go @@ -38,6 +38,16 @@ const ( sendLogTimeout = 300 * time.Second ) +// Provisioning-token rejection reasons the API sends in error.metadata.reason +// (HTTP 400, code 40003). This list can grow; a value outside it is not an +// error, just one cmd/cli does not classify yet. +const ( + ReasonTokenInvalid = "token_invalid" + ReasonTokenExpired = "token_expired" + ReasonTokenLimitReached = "token_limit_reached" + ReasonTokenDisabled = "token_disabled" +) + // ResolverConfig represents Control D resolver data. type ResolverConfig struct { DOH string `json:"doh"` @@ -58,10 +68,41 @@ type utilityResponse struct { } `json:"body"` } +// errorMetadata carries additive, optional detail on top of Code/Message. +// Older API deployments omit it, so it must decode to its zero value rather +// than fail the whole response. Its custom UnmarshalJSON gives the same +// tolerance to a malformed value: a metadata that is not an object, or a +// Reason that is not a string (a number, an object, or null), degrades to +// the zero value rather than failing the response that contains it. +type errorMetadata struct { + // Reason is a machine-readable rejection reason sent on provisioning-token + // errors (HTTP 400, code 40003): token_invalid, token_expired, + // token_limit_reached, or token_disabled. Empty when absent or malformed; + // callers must treat any other value as unknown rather than reject the + // response. + Reason string `json:"reason"` +} + +func (m *errorMetadata) UnmarshalJSON(data []byte) error { + var raw struct { + Reason json.RawMessage `json:"reason"` + } + // Best-effort: a metadata that is not an object, or a reason that is not + // a string (number, object, null), leaves the zero value instead of + // failing this decode. Code and Message still classify the failure. + if err := json.Unmarshal(data, &raw); err != nil { + *m = errorMetadata{} + return nil + } + _ = json.Unmarshal(raw.Reason, &m.Reason) + return nil +} + type ErrorResponse struct { ErrorField struct { - Message string `json:"message"` - Code int `json:"code"` + Message string `json:"message"` + Code int `json:"code"` + Metadata errorMetadata `json:"metadata"` } `json:"error"` // StatusCode is the HTTP status the API answered with. It is not part of the JSON // body: this type is built for *any* non-200 whose body decodes, so the body alone diff --git a/internal/controld/config_test.go b/internal/controld/config_test.go index 5973de0c..16a102fe 100644 --- a/internal/controld/config_test.go +++ b/internal/controld/config_test.go @@ -96,3 +96,113 @@ func TestAPIErrorRecordsHTTPStatus(t *testing.T) { } }) } + +// TestAPIErrorDecodesRejectionReason pins the additive metadata.reason field the +// API sends on provisioning-token rejections. cmd/cli maps known reasons to their +// own failure codes, and must fall back cleanly when the field is absent or holds +// a value this build does not recognize yet. +func TestAPIErrorDecodesRejectionReason(t *testing.T) { + tests := []struct { + name string + body string + wantReason string + }{ + { + name: "known reason", + body: `{"error":{"message":"invalid token","code":40003,"metadata":{"reason":"token_disabled"}}}`, + wantReason: "token_disabled", + }, + { + name: "reason absent", + body: `{"error":{"message":"invalid token","code":40003}}`, + wantReason: "", + }, + { + name: "unknown reason value", + body: `{"error":{"message":"invalid token","code":40003,"metadata":{"reason":"something_new"}}}`, + wantReason: "something_new", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + d := json.NewDecoder(strings.NewReader(tc.body)) + errResp, err := apiErrorFromResponse(http.StatusBadRequest, d) + if err != nil { + t.Fatalf("unexpected decode error: %v", err) + } + if errResp.ErrorField.Metadata.Reason != tc.wantReason { + t.Errorf("reason = %q, want %q", errResp.ErrorField.Metadata.Reason, tc.wantReason) + } + }) + } +} + +// TestAPIErrorToleratesMalformedRejectionReason pins the fix for a decode error +// confined to metadata.reason: a reason sent as the wrong JSON type must not +// discard the rest of the response. Before this fix, apiErrorFromResponse +// returned the raw decode error and nothing else, which cmd/cli's +// apiFailureCode cannot recognize as an *ErrorResponse - it falls back to +// API_UNREACHABLE (a retryable bootstrap failure) instead of the permanent +// rejection the HTTP status and code actually describe. +func TestAPIErrorToleratesMalformedRejectionReason(t *testing.T) { + tests := []struct { + name string + body string + }{ + {name: "reason as a number", body: `{"error":{"message":"invalid token","code":40003,"metadata":{"reason":12345}}}`}, + {name: "reason as an object", body: `{"error":{"message":"invalid token","code":40003,"metadata":{"reason":{"inner":"value"}}}}`}, + {name: "reason as null", body: `{"error":{"message":"invalid token","code":40003,"metadata":{"reason":null}}}`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + d := json.NewDecoder(strings.NewReader(tc.body)) + errResp, err := apiErrorFromResponse(http.StatusBadRequest, d) + if err != nil { + t.Fatalf("a malformed reason must not fail the whole decode: %v", err) + } + if errResp.ErrorField.Message != "invalid token" { + t.Errorf("message = %q, want it to survive the malformed reason", errResp.ErrorField.Message) + } + if errResp.ErrorField.Code != 40003 { + t.Errorf("code = %d, want it to survive the malformed reason", errResp.ErrorField.Code) + } + if errResp.ErrorField.Metadata.Reason != "" { + t.Errorf("reason = %q, want empty for a malformed value", errResp.ErrorField.Metadata.Reason) + } + }) + } +} + +// TestAPIErrorToleratesMalformedMetadata pins the same tolerance one level +// up: a metadata field that is not a JSON object must not discard the rest +// of the response. Code and Message still classify the failure, and Reason +// stays empty. +func TestAPIErrorToleratesMalformedMetadata(t *testing.T) { + tests := []struct { + name string + body string + }{ + {name: "metadata as a string", body: `{"error":{"message":"invalid token","code":40003,"metadata":"foo"}}`}, + {name: "metadata as a number", body: `{"error":{"message":"invalid token","code":40003,"metadata":7}}`}, + {name: "metadata as an array", body: `{"error":{"message":"invalid token","code":40003,"metadata":["reason"]}}`}, + {name: "metadata as a boolean", body: `{"error":{"message":"invalid token","code":40003,"metadata":true}}`}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + d := json.NewDecoder(strings.NewReader(tc.body)) + errResp, err := apiErrorFromResponse(http.StatusBadRequest, d) + if err != nil { + t.Fatalf("unexpected decode error: %v", err) + } + if errResp.ErrorField.Code != 40003 { + t.Errorf("code = %d, want 40003", errResp.ErrorField.Code) + } + if errResp.ErrorField.Message != "invalid token" { + t.Errorf("message = %q, want %q", errResp.ErrorField.Message, "invalid token") + } + if errResp.ErrorField.Metadata.Reason != "" { + t.Errorf("reason = %q, want empty for a malformed metadata", errResp.ErrorField.Metadata.Reason) + } + }) + } +} diff --git a/internal/controld/reachability.go b/internal/controld/reachability.go new file mode 100644 index 00000000..6c0a8fcd --- /dev/null +++ b/internal/controld/reachability.go @@ -0,0 +1,33 @@ +package controld + +import ( + "context" + "io" + "net/http" +) + +// ProbeReachability makes one lightweight request to the ControlD API host, +// reusing the same transport and IP-fallback logic real provisioning traffic +// takes. Any HTTP response, even an error status, counts as reachable: this +// checks the network path, not whether the endpoint accepts the request. +// +// The caller controls how long to wait via ctx; there is no timeout here +// beyond what ctx enforces. +func ProbeReachability(ctx context.Context, cdDev bool) error { + apiURL := apiURLCom + if cdDev { + apiURL = apiURLDev + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, apiURL, nil) + if err != nil { + return err + } + client := &http.Client{Transport: apiTransport(cdDev)} + resp, err := doWithFallback(client, req, apiServerIP(cdDev)) + if err != nil { + return err + } + defer resp.Body.Close() + _, _ = io.Copy(io.Discard, resp.Body) + return nil +} diff --git a/internal/controld/reachability_test.go b/internal/controld/reachability_test.go new file mode 100644 index 00000000..9f39ec75 --- /dev/null +++ b/internal/controld/reachability_test.go @@ -0,0 +1,14 @@ +package controld + +import ( + "context" + "testing" +) + +func TestProbeReachabilityRespectsCancelledContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := ProbeReachability(ctx, false); err == nil { + t.Error("expected an error for an already-cancelled context") + } +} diff --git a/tasks/plan.md b/tasks/plan.md deleted file mode 100644 index e1303fc3..00000000 --- a/tasks/plan.md +++ /dev/null @@ -1,213 +0,0 @@ -# Plan: provisioning failure codes (issue #586) - -Spec: SPEC.md. Baseline: `ac0e6aed` on `v1.0`. -Branches: `issue-586` (off `v1.0`), `issue-586-master` (off `master`). -Two MRs, both referencing #586; the `v1.0` MR carries `Closes #586`. - -## Shared contract (fixed here so parallel tasks cannot diverge) - -| Code | Stage | Exit | -|---|---|---| -| `API_UNREACHABLE` | bootstrap | 30 | -| `API_REJECTED` | bootstrap | 31 | -| `API_DEVICE_INVALID` | bootstrap | 32 | -| `LISTENER_BIND_FAILED` | listener | 41 | -| `LISTENER_CONFIGURED_ADDR_UNAVAILABLE` | listener | 42 | -| `SERVICE_INSTALL_FAILED` | service | 51 | -| `SERVICE_START_FAILED` | service | 52 | -| `SERVICE_SELFCHECK_FAILED` | service | 53 | - -- Result file: `provision_result.json` in the ctrld home dir (same - resolution as the persisted internal log: `absHomeDir` on v1.0, the - `userHomeDir`-based equivalent on master). Atomic write (temp + - rename in the same dir). Cleared when provisioning succeeds. -- Result schema (version 1): `version`, `timestamp` (RFC3339, UTC), - `stage`, `code`, `exit_code`, `message`, optional `detail.attempts[]` - of `{addr, proto, os_error}`; attempts capped at 12 entries, every - string capped at 256 chars. -- Identifier line, exact format (greppable, token-free by - construction): `provisioning failed: stage= code= (exit )`. -- Redaction: results are built through a constructor that takes the - secrets in scope (cd UID, provision token) and strips them from every - field. Messages come from our own summaries plus OS error strings, - never raw config or API bodies. -- Exit seam: `provisionExit = os.Exit` package var so tests can stub - process exit. Emission helper `failProvision(...)` writes the file, - logs the identifier line, calls the notify func, then exits with the - stage code. Nonzero exit is preserved everywhere the daemon exits - nonzero today; the deliberate clean return on permanent API rejection - stays a clean return (result file only). - -## Dependency graph - -``` -A1 (contract module + doc, v1.0) D1 (master port) - ├─► B1 daemon emissions (cli.go) depends on: contract table - ├─► B2 start-side (commands.go+service.go) (from A1) + C1 verified - └─► B3 postinstall (scripts, tests) implementation as reference - └─► C1 v1.0 checkpoint ────────────► D1 ─► E1 final checkpoint -``` - -## Group A — serial, runs inline (1 task) - -### A1. Contract foundation on `issue-586` -Create branch `issue-586` from `v1.0`. New files: -`cmd/cli/provision_result.go`, `cmd/cli/provision_result_test.go`, -`docs/provisioning-failure-codes.md`. - -Module contents: stage type + the 8 code constants + exit-code map; -`ProvisionResult` struct per schema; bounded/redacting constructor; -atomic `writeProvisionResult` / `readProvisionResult` / -`clearProvisionResult`; identifier-line formatter; `provisionExit` -seam; `failProvision` helper. Doc: full table — code, stage, exit, -failure scenario, next safe troubleshooting action / evidence request. - -Tests (RED first): every code maps to exactly one stage and one -in-range exit code (30–39/40–49/50–59); no collision with 0–3 -(`ctrld status`) or 126 (pin); file round-trip; atomic overwrite; -clear; redaction (a result built from inputs containing a fake token -and cd UID serializes without them); attempts/string caps enforced; -identifier line matches the exact format. - -Acceptance: `go build ./...` and `go test ./cmd/cli/` green; doc rows -exactly match the constants. - -## Group B — parallel Workflow fan-out, one subagent per task, worktree isolation, branched from `issue-586` after A1 - -### B1. Daemon emissions in `cli.go` -- Bootstrap branches in `run()` (`cli.go:339-372`): - - permanent rejection (`permanentAPIRejection`): write `API_REJECTED` - result (HTTP status + our own summary, no raw API body), keep the - existing clean return and its comment. - - invalid device (`controld.InvalidConfigCode`): write - `API_DEVICE_INVALID` before `uninstallInvalidCdUID`. - - fatal fetch: write `API_UNREACHABLE`, replace - `cdLogger.Fatal()` with error log + identifier line + - `failProvision` exit 30 (still nonzero for the service manager). -- Listener (`tryUpdateListenerConfig`, `tryUpdateListenerConfigIntercept`): - - record every failed bind attempt `{addr, proto, os_error}` — - capture UDP and TCP errors separately in `tryListen` (keep - `errors.Join` for control flow), cap per contract. - - exhaustion fatal (`cli.go:1639`) and converged-random fatal - (`cli.go:1720`) → `LISTENER_BIND_FAILED` exit 41 with attempts. - - no-fallback-allowed fatal (`cli.go:1652`) and intercept-mode fatals - (`cli.go:1452,1467`) → `LISTENER_CONFIGURED_ADDR_UNAVAILABLE` - exit 42 (fallback-exhausted intercept fatal stays - `LISTENER_BIND_FAILED`). - - all fatals keep calling the notify func first; final message - includes the code string. -- Clear the result file at the point provisioning is known good - (after `updateListenerConfig` succeeds in `run()`). - -Tests (RED first): occupy a UDP+TCP port, drive the listener path to -exhaustion with the exit seam stubbed, assert the result file has -`LISTENER_BIND_FAILED`, the attempted address, both protocols' -`os_error` (`address already in use` class); pure mapping test -API error → code (permanent 4xx → `API_REJECTED`, 40402 → -`API_DEVICE_INVALID`, network error → `API_UNREACHABLE`) following -`cli_preflight_test.go` patterns. - -Acceptance: only `cmd/cli/cli.go` + new/extended tests touched; -`go build ./... && go test ./cmd/cli/` green. - -### B2. `ctrld start` reporting in `commands.go` + `service.go` -- Add `doTasksE` (returns failed task name + error; `doTasks` keeps - its signature and delegates). -- Fresh-install path (`commands.go:618`): failed `Install` task → - `SERVICE_INSTALL_FAILED` (write result, print identifier line, exit - 51); failed `Start` task → `SERVICE_START_FAILED` (exit 52). This - fixes the current fall-through that exits 0 on install failure. -- Existing-service path (`commands.go:528-543`): failure → 52 with the - same reporting (replaces bare `os.Exit(1)`). -- Self-check failure branch (`commands.go:627-664`): keep the log - drain and `uninstall(p, s)`; then read the daemon's result file — - if present and stamped after this start attempt began, report its - stage/code/exit (daemon identity wins: e.g. `LISTENER_BIND_FAILED`); - otherwise write and report `SERVICE_SELFCHECK_FAILED` exit 53. - Extract this into a testable helper (fabricated result files + - stubbed exit seam). -- On successful start (self-check ok), clear any stale result file. - -Tests (RED first): `doTasksE` failure attribution; helper precedence -(fresh daemon result wins; stale/missing falls back to 53); exit-code -selection per failed task. - -Acceptance: only `cmd/cli/commands.go`, `cmd/cli/service.go` + tests -touched; build and package tests green. - -### B3. postinstall MDM surface -- `scripts/pkg/postinstall`: capture `ctrld start` output to a - `mktemp` file (chmod 600) instead of `/dev/null`; keep the plist - check as the success gate; on failure, `grep -m1 '^provisioning - failed: '` from the capture into the install log together with the - exit code; delete the capture file always; never echo any other - output line (token safety preserved by extracting only the - fixed-format line). -- Shell test `test-scripts/darwin/test-postinstall-provision-failure.sh` - (matching existing script conventions): stub `$CTRLD` that prints a - fake token plus a valid identifier line and exits 41; assert the - logged output contains stage/code/exit and not the token; assert - success path unchanged. Runs without root. -- Update `docs/macos-pkg-mdm.md` where it documents the discard - behavior/failure triage, and add the failure-code doc link. - -Acceptance: shell test passes locally (`sh test-scripts/darwin/...`); -only `scripts/pkg/postinstall`, `test-scripts/darwin/`, `docs/` -touched. - -## Checkpoint C1 — serial, after Group B merges - -Merge order: B1, B2, B3 into `issue-586`. Then: `go build ./...`, -`go vet ./...`, `go test ./cmd/cli/...` (and full `./...`), run the B3 -shell test, verify doc table == constants, and verify each spec -acceptance criterion has an implementation + test. Fix-forward any -merge fallout before Group D starts. - -## Group D — serial (1 task, own worktree off `master`) - -### D1. Master port on `issue-586-master` -Create `git worktree` with branch `issue-586-master` from -`origin/master`/`master`. Port with the v1.0 implementation as -reference, adapted to master's structure (zap-shaped logging idiom, -no `commands.go`): - -- `cmd/cli/provision_result.go` + tests: identical contract table. -- Bootstrap: `run()` branches at master `cli.go:340-374` (same - permanent-rejection clean return, invalid-device, fatal fetch). -- Listener: `tryUpdateListenerConfig` fatals at master - `cli.go:1657/1667/1719`; intercept variant at `cli.go:1487/1502`; - per-attempt capture (bind errors currently logged at Debug, - `cli.go:1665`). -- Start side: `commands_service_start.go` — both `doTasks` call sites, - self-check `default:` arm (`os.Exit(1)` ~line 370), same fall-through - audit, same precedence logic; `doTasksE` in `service.go`. -- `docs/provisioning-failure-codes.md`: same table (omit - pkg/postinstall-specific notes; master has no `scripts/pkg`). -- No postinstall work on master. - -Tests mirrored from v1.0 where the structure allows. - -Acceptance: in the master worktree, `go build ./...`, -`go test ./cmd/cli/...` green; constants table semantically identical -to `issue-586`. - -## Checkpoint E1 — serial, final - -- Cross-branch contract equality: compare code constants, exit codes, - identifier-line format, result schema between the two branches. -- Full test suites on both branches. -- Both branches committed (per-task Conventional Commits); no pushes, - no MRs yet — `/draft-review` is the next pipeline step. - -## Execution notes - -- Each parallel group runs as one Workflow fan-out, one subagent per - task, `isolation: 'worktree'` so parallel edits never conflict; - serial tasks (A1, C1, D1, E1) run inline (D1 manages its own - master-based worktree). -- Every subagent follows RED → GREEN → regression → build and commits - in its worktree; the orchestrator merges in dependency order and - runs the full suite before the next group. -- Subagent prompts are self-contained: they carry the contract table - and file anchors from this plan, not references to SPEC.md (worktree - copies may not include untracked files). diff --git a/tasks/todo.md b/tasks/todo.md deleted file mode 100644 index 10a26935..00000000 --- a/tasks/todo.md +++ /dev/null @@ -1,21 +0,0 @@ -# TODO: issue #586 provisioning failure codes - -Groups run in order; tasks inside a parallel group run as one Workflow -fan-out (one subagent per task, worktree isolation). - -## Group A (serial) -- [x] A1: contract module `cmd/cli/provision_result.go` + tests + `docs/provisioning-failure-codes.md` on branch `issue-586` - -## Group B (parallel after A1) -- [x] B1: daemon emissions — bootstrap + listener paths in `cmd/cli/cli.go` + tests -- [x] B2: `ctrld start` reporting — `cmd/cli/commands.go`, `cmd/cli/service.go` (doTasksE, exit-0 fall-through fix, self-check precedence) + tests -- [x] B3: postinstall MDM surface — `scripts/pkg/postinstall`, shell test, `docs/macos-pkg-mdm.md` - -## Checkpoint C1 (serial) -- [x] C1: merge B1→B2→B3 into `issue-586`, full build/vet/test, shell test, doc/constants parity, spec AC audit - -## Group D (serial) -- [x] D1: master port on `issue-586-master` (contract module, cli.go emissions, commands_service_start.go, docs) + tests - -## Checkpoint E1 (serial) -- [x] E1: cross-branch contract equality, full suites on both branches, commits tidy — stop before push/MR (`/draft-review` next) diff --git a/test-scripts/darwin/test-pkg-intercept-mode.sh b/test-scripts/darwin/test-pkg-intercept-mode.sh index d1029cd7..f23e930b 100755 --- a/test-scripts/darwin/test-pkg-intercept-mode.sh +++ b/test-scripts/darwin/test-pkg-intercept-mode.sh @@ -36,14 +36,6 @@ printf 'launchctl %s\n' "$*" >>"$CALLS" exit 0 EOF -cat >"$bin/mktemp" <<'EOF' -#!/bin/sh -[ "${1:-}" = "-t" ] && [ -n "${2:-}" ] || exit 1 -path="$FAKE_TMPDIR/$2.capture" -: >"$path" -printf '%s\n' "$path" -EOF - cat >"$bin/ctrld" <<'EOF' #!/bin/sh printf 'ctrld %s\n' "$*" >>"$CALLS" @@ -55,7 +47,7 @@ fi exit "${FAKE_CTRLD_EXIT:-0}" EOF -chmod +x "$bin/defaults" "$bin/launchctl" "$bin/mktemp" "$bin/ctrld" +chmod +x "$bin/defaults" "$bin/launchctl" "$bin/ctrld" assert_contains() { expected=$1 @@ -91,6 +83,8 @@ run_case() { calls="$case_dir/calls" output="$case_dir/output" postinstall="$case_dir/postinstall" + case_tmpdir="$case_dir/tmp" + mkdir -p "$case_tmpdir" : >"$calls" if [ "$existing" = "1" ]; then : >"$plist" @@ -105,6 +99,7 @@ run_case() { status=0 PATH="$bin:$PATH" \ + TMPDIR="$case_tmpdir" \ CALLS="$calls" \ FAKE_TOKEN_PRESENT=1 \ FAKE_TOKEN=test-token \ @@ -112,7 +107,6 @@ run_case() { FAKE_MODE="$mode" \ FAKE_CTRLD_EXIT="$ctrld_exit" \ FAKE_PLIST="$plist" \ - FAKE_TMPDIR="$case_dir" \ "$postinstall" >"$output" 2>&1 || status=$? if [ "$status" -ne "$expected_status" ]; then @@ -149,9 +143,9 @@ case_dir=$(run_case upgrade-intercept 1 1 intercept-dns) assert_contains 'ctrld start --intercept-mode dns' "$case_dir/calls" assert_not_contains 'launchctl load' "$case_dir/calls" -case_dir=$(run_case fresh-invalid 0 1 invalid) -assert_contains 'WARNING: unsupported InterceptMode in managed preferences; using standard mode' "$case_dir/output" -assert_not_contains '--intercept-mode' "$case_dir/calls" +case_dir=$(run_case fresh-invalid 0 1 invalid 0 1) +assert_contains 'ERROR: provisioning failed: stage=package code=INTERCEPT_MODE_INVALID (exit 1)' "$case_dir/output" +assert_not_contains 'ctrld start' "$case_dir/calls" case_dir=$(run_case upgrade-invalid 1 1 invalid) assert_contains 'WARNING: unsupported InterceptMode in managed preferences; preserving existing service mode' "$case_dir/output" diff --git a/test-scripts/darwin/test-postinstall-provision-failure.sh b/test-scripts/darwin/test-postinstall-provision-failure.sh index d9f6eede..2cc454ec 100755 --- a/test-scripts/darwin/test-postinstall-provision-failure.sh +++ b/test-scripts/darwin/test-postinstall-provision-failure.sh @@ -1,45 +1,168 @@ #!/bin/sh -# Test: the MDM pkg postinstall script surfaces ctrld's provisioning -# failure identifier in its output without leaking the provision token, -# and still reports success once the plist exists. +# Test: the MDM pkg postinstall script validates managed prefs before +# running ctrld, classifies every failure (pre-flight or post-invoke) with +# a stable 'stage=package code=' identifier line, best-effort appends +# 'ctrld diag' output on failure, and never leaks the provision token. # -# Self-contained and root-free: every path postinstall touches is -# redirected into a throwaway temp directory via the -# CTRLD_POSTINSTALL_{PLIST,CTRLD,PREFS} overrides, and 'defaults' is -# stubbed on PATH so the profile-wait loop resolves on its first attempt. +# Self-contained and root-free: each case runs a copy of the real +# postinstall script with its PLIST/CTRLD/PREFS assignments sed-rewritten +# to fixture paths (same seam as test-pkg-intercept-mode.sh), with +# 'defaults', 'launchctl', and 'ctrld' stubbed on PATH. No macOS-only API is +# used, so this runs as plain POSIX sh on Linux too. # -# Out of scope: the upgrade path (plist already exists) calls the real -# launchctl and is not exercised here. +# With POSTINSTALL_TEST_REAL_DEFAULTS=1 on macOS, 'defaults' is not stubbed: +# each case writes its managed-prefs values into a real plist under the +# fixture dir with 'defaults write', and the postinstall copy reads them +# with the real binary. This tests the one boundary a stub cannot: what +# 'defaults read' prints and exits with for an empty string, a boolean, a +# control character, or a missing key. ctrld and launchctl stay stubbed, so +# the run never touches the services of the machine. +# +# The wait loop between profile attempts is shortened to 0 seconds in the +# copy under test. The attempt count and the log lines stay as they are. # # Run: sh test-postinstall-provision-failure.sh -SCRIPT_DIR=$(cd "$(dirname "$0")" && pwd) -POSTINSTALL="$SCRIPT_DIR/../../scripts/pkg/postinstall" +set -eu -WORKDIR=$(mktemp -d -t ctrld-postinstall-test) || { - echo "FAIL: could not create test work directory" >&2 - exit 1 -} -trap 'rm -rf "$WORKDIR"' EXIT +repo_root=$(CDPATH= cd -- "$(dirname -- "$0")/../.." && pwd) +postinstall_source="$repo_root/scripts/pkg/postinstall" + +fixture=$(mktemp -d "${TMPDIR:-/tmp}/ctrld-postinstall-test.XXXXXX") +trap 'rm -rf "$fixture"' EXIT HUP INT TERM -FAKE_TOKEN="FAKE-PROVISION-TOKEN-DO-NOT-LEAK-93af0c" -export FAKE_TOKEN +# A well-formed fake token used for cases that must reach ctrld (success, +# ctrld-emitted-identifier passthrough). Obviously fake, but still checked +# for non-leakage like a real one would be. +GOOD_TOKEN="org-v1-FAKE93af0cDONOTLEAK" +# A present-but-malformed value (contains whitespace) for the +# PROVISION_TOKEN_MALFORMED case. +MALFORMED_TOKEN="FAKE TOKEN WITH SPACE DO NOT LEAK" +# A second, differently-shaped org-v1- token: never the value read from +# managed prefs, so it can only be caught by the pattern-based redaction, +# not the exact-match one. +OTHER_ORG_TOKEN="org-v1-DIFFERENTshape001" +export GOOD_TOKEN MALFORMED_TOKEN OTHER_ORG_TOKEN + +# Real mktemp, resolved before the fixture bin/ overrides PATH, so the +# mktemp stub can fall through to it when a test case is not forcing a +# failure. +REAL_MKTEMP=$(command -v mktemp) +export REAL_MKTEMP + +bin="$fixture/bin" +mkdir -p "$bin" + +real_defaults=${POSTINSTALL_TEST_REAL_DEFAULTS:-0} +if [ "$real_defaults" = "1" ] && [ "$(uname -s)" != "Darwin" ]; then + echo "POSTINSTALL_TEST_REAL_DEFAULTS=1 needs macOS: the real 'defaults' binary is the point of this mode" >&2 + exit 2 +fi -STUBBIN="$WORKDIR/stubbin" -mkdir -p "$STUBBIN" +# --- Stubs ----------------------------------------------------------- + +if [ "$real_defaults" != "1" ]; then +cat >"$bin/defaults" <<'EOF' +#!/bin/sh +# Stand-in for macOS 'defaults read '. Every key answers from +# a FAKE_*_PRESENT/FAKE_* pair the test case exports beforehand; a key with +# no *_PRESENT=1 comes back unset, like a profile that never set it. +key=${3:-} +case "$key" in + ProvisionToken) + [ "${FAKE_TOKEN_PRESENT:-0}" = "1" ] || exit 1 + printf '%s\n' "${FAKE_TOKEN_VALUE:-}" + ;; + CustomHostname) + [ "${FAKE_HOST_PRESENT:-0}" = "1" ] || exit 1 + printf '%s\n' "${FAKE_HOST:-}" + ;; + InterceptMode) + [ "${FAKE_MODE_PRESENT:-0}" = "1" ] || exit 1 + printf '%s\n' "${FAKE_MODE:-}" + ;; + UseDevEnvironment) + [ "${FAKE_DEVENV_PRESENT:-0}" = "1" ] || exit 1 + printf '%s\n' "${FAKE_DEVENV:-}" + ;; + *) + exit 1 + ;; +esac +EOF +chmod +x "$bin/defaults" +fi + +# write_real_prefs materializes the FAKE_* managed-prefs values of the +# current case into a real plist at "$1", the path the postinstall copy +# reads as its PREFS domain. A key with no *_PRESENT=1 is not written, like +# a profile that never set it. +write_real_prefs() { + domain=$1 + [ "${FAKE_TOKEN_PRESENT:-0}" = "1" ] && defaults write "$domain" ProvisionToken -string "${FAKE_TOKEN_VALUE:-}" + [ "${FAKE_HOST_PRESENT:-0}" = "1" ] && defaults write "$domain" CustomHostname -string "${FAKE_HOST:-}" + [ "${FAKE_MODE_PRESENT:-0}" = "1" ] && defaults write "$domain" InterceptMode -string "${FAKE_MODE:-}" + [ "${FAKE_DEVENV_PRESENT:-0}" = "1" ] && defaults write "$domain" UseDevEnvironment -string "${FAKE_DEVENV:-}" + return 0 +} + +cat >"$bin/launchctl" <<'EOF' +#!/bin/sh +[ -n "${CALLS:-}" ] && printf 'launchctl %s\n' "$*" >>"$CALLS" +exit "${FAKE_LAUNCHCTL_EXIT:-0}" +EOF +chmod +x "$bin/launchctl" -cat > "$STUBBIN/defaults" <<'STUB' +cat >"$bin/ctrld" <<'EOF' #!/bin/sh -# Stand-in for macOS 'defaults read ': answers ProvisionToken -# immediately, like a profile that only sets that one key, so the -# postinstall wait loop never has to sleep. -if [ "$1" = "read" ] && [ "$3" = "ProvisionToken" ]; then - echo "$FAKE_TOKEN" +[ -n "${CALLS:-}" ] && printf 'ctrld %s\n' "$*" >>"$CALLS" + +if [ "$1" = "diag" ]; then + if [ "${FAKE_DIAG_TRAPS_TERM:-0}" = "1" ]; then + # Ignores the polite signal, so only SIGKILL can end it. Proves + # postinstall's wait stays bounded either way. + trap '' TERM + sleep 30 + exit 0 + fi + if [ "${FAKE_DIAG_WORKS:-0}" = "1" ]; then + echo "resolver: healthy" + echo "listener: 127.0.0.1:53 bound" + [ -n "${FAKE_DIAG_LEAK_LINE:-}" ] && echo "$FAKE_DIAG_LEAK_LINE" + exit 0 + fi + echo "ctrld: unknown command \"diag\"" >&2 + exit 1 +fi + +if [ "${FAKE_CTRLD_EXIT:-0}" = "0" ]; then + : >"$FAKE_PLIST" exit 0 fi -exit 1 -STUB -chmod +x "$STUBBIN/defaults" +# The service manager writes the plist at install time, before the start +# step, so a start failure leaves it behind. Reproduced so the test can +# prove postinstall does not read that leftover as success. +[ "${FAKE_CTRLD_CREATE_PLIST:-0}" = "1" ] && : >"$FAKE_PLIST" +# A real ctrld may echo the raw token on error; reproduced here so the test +# can prove postinstall never lets it reach install.log. +echo "$GOOD_TOKEN" +[ -n "${FAKE_CTRLD_IDENTIFIER:-}" ] && echo "$FAKE_CTRLD_IDENTIFIER" +exit "$FAKE_CTRLD_EXIT" +EOF +chmod +x "$bin/ctrld" + +cat >"$bin/mktemp" <<'EOF' +#!/bin/sh +# Stand-in for mktemp: forces the postinstall script's own mktemp failure +# branch on demand, otherwise falls through to the real mktemp. +if [ "${FAKE_MKTEMP_FAIL:-0}" = "1" ]; then + exit 1 +fi +exec "$REAL_MKTEMP" "$@" +EOF +chmod +x "$bin/mktemp" + +# --- Assertion helpers ------------------------------------------------- FAILURES=0 @@ -71,78 +194,469 @@ assert_not_contains() { esac } -# run_postinstall runs postinstall with the given plist/ctrld overrides and -# a private TMPDIR, so the caller can check that the postinstall's own -# capture file (created inside that TMPDIR via mktemp) does not survive. -run_postinstall() { +assert_not_contains "$(cat "$postinstall_source")" "CTRLD_POSTINSTALL_" "production script must not depend on removed env-var overrides" + +# --- Fixture plumbing --------------------------------------------------- + +# make_postinstall_copy writes a copy of the real postinstall script with +# its PLIST/CTRLD/PREFS assignments rewritten to fixture paths, so the test +# drives the actual production logic instead of a re-implementation. +make_postinstall_copy() { plist_override=$1 ctrld_override=$2 - capture_tmpdir=$3 - output=$(PATH="$STUBBIN:$PATH" \ - TMPDIR="$capture_tmpdir" \ - CTRLD_POSTINSTALL_PLIST="$plist_override" \ - CTRLD_POSTINSTALL_CTRLD="$ctrld_override" \ - CTRLD_POSTINSTALL_PREFS="/does/not/matter" \ - sh "$POSTINSTALL" 2>&1) - exit_code=$? + prefs_override=$3 + dest=$4 + sed \ + -e "s|^PLIST=\"/Library/LaunchDaemons/ctrld.plist\"\$|PLIST=\"$plist_override\"|" \ + -e "s|^CTRLD=\"/usr/local/bin/ctrld\"\$|CTRLD=\"$ctrld_override\"|" \ + -e "s|^PREFS=\"/Library/Managed Preferences/com.controld.ctrld\"\$|PREFS=\"$prefs_override\"|" \ + -e "s|^ sleep 10\$| sleep 0|" \ + "$postinstall_source" >"$dest" + chmod +x "$dest" + # A substitution that stops matching is silent: the copy keeps the real + # system paths and the case runs against the installed machine. Check + # that each redirection landed. + for expected in "PLIST=\"$plist_override\"" "CTRLD=\"$ctrld_override\"" "PREFS=\"$prefs_override\""; do + if ! grep -Fq -- "$expected" "$dest"; then + printf 'FAIL: fixture substitution did not apply (%s); postinstall paths changed?\n' "$expected" >&2 + exit 1 + fi + done + if grep -Eq '^(PLIST|CTRLD|PREFS)="(/Library|/usr/local)' "$dest"; then + printf 'FAIL: a real system path survived substitution\n' >&2 + grep -E '^(PLIST|CTRLD|PREFS)=' "$dest" >&2 + exit 1 + fi } -# --- Failure case: ctrld reports a listener bind failure --------------- +# reset_fakes clears every FAKE_*/CALLS override so one case's env can never +# leak into the next. +reset_fakes() { + unset FAKE_TOKEN_PRESENT FAKE_TOKEN_VALUE FAKE_HOST_PRESENT FAKE_HOST \ + FAKE_MODE_PRESENT FAKE_MODE FAKE_DEVENV_PRESENT FAKE_DEVENV \ + FAKE_CTRLD_EXIT FAKE_CTRLD_IDENTIFIER FAKE_CTRLD_CREATE_PLIST \ + FAKE_DIAG_WORKS FAKE_DIAG_LEAK_LINE FAKE_DIAG_TRAPS_TERM \ + FAKE_LAUNCHCTL_EXIT FAKE_MKTEMP_FAIL CALLS 2>/dev/null || true +} -failure_dir="$WORKDIR/failure" -failure_tmp="$failure_dir/tmp" -mkdir -p "$failure_tmp" -failure_plist="$failure_dir/ctrld.plist" -failure_ctrld="$failure_dir/ctrld" +# run_case sets up a fresh fixture dir (plist, prefs, private TMPDIR, a +# calls log) for "$name", runs a fixture-rewritten postinstall copy with +# every currently-exported FAKE_* var visible to the stubs, and leaves +# $output/$exit_code/$case_dir set for the caller's assertions. +run_case() { + name=$1 + existing_plist=${2:-0} + case_dir="$fixture/$name" + case_tmp="$case_dir/tmp" + mkdir -p "$case_tmp" + plist="$case_dir/ctrld.plist" + prefs="$case_dir/prefs" + calls="$case_dir/calls" + output_file="$case_dir/output" + postinstall="$case_dir/postinstall" + : >"$calls" + [ "$existing_plist" = "1" ] && : >"$plist" + [ "$real_defaults" = "1" ] && write_real_prefs "$prefs" -cat > "$failure_ctrld" <<'STUB' -#!/bin/sh -# Stands in for a ctrld that fails to bind its listener: echoes the raw -# token (as ctrld's own error output may) plus the fixed-format failure -# identifier behind a log-style prefix, then exits with the stage code. -echo "$FAKE_TOKEN" -echo "2024-01-01T00:00:00Z ERR ctrld: provisioning failed: stage=listener code=LISTENER_BIND_FAILED (exit 41)" -exit 41 -STUB -chmod +x "$failure_ctrld" - -run_postinstall "$failure_plist" "$failure_ctrld" "$failure_tmp" - -assert_eq "$exit_code" "1" "failure case: postinstall exit code" -assert_contains "$output" "stage=listener" "failure case: output names the stage" -assert_contains "$output" "LISTENER_BIND_FAILED" "failure case: output names the code" -assert_contains "$output" "41" "failure case: output names the exit code" -assert_not_contains "$output" "$FAKE_TOKEN" "failure case: output must not contain the provision token" - -leftover=$(ls -A "$failure_tmp" 2>/dev/null) -assert_eq "$leftover" "" "failure case: capture temp file removed" - -# --- Success case: ctrld provisions and writes the plist ---------------- - -success_dir="$WORKDIR/success" -success_tmp="$success_dir/tmp" -mkdir -p "$success_tmp" -success_plist="$success_dir/ctrld.plist" -success_ctrld="$success_dir/ctrld" - -cat > "$success_ctrld" < "$success_plist" -exit 0 -STUB -chmod +x "$success_ctrld" + make_postinstall_copy "$plist" "$bin/ctrld" "$prefs" "$postinstall" + + status=0 + PATH="$bin:$PATH" \ + TMPDIR="$case_tmp" \ + FAKE_PLIST="$plist" \ + CALLS="$calls" \ + "$postinstall" >"$output_file" 2>&1 || status=$? + + exit_code=$status + output=$(cat "$output_file") + calls_out=$(cat "$calls") +} + +# --- Case: missing prefs -> PROFILE_PREFS_MISSING ----------------------- +# Runs postinstall's real wait loop to exhaustion (~110s): no seam shortens +# it, since that timing is untouched by this change. + +reset_fakes +run_case missing-prefs + +assert_eq "$exit_code" "1" "missing-prefs: exit code" +assert_contains "$output" "stage=package code=PROFILE_PREFS_MISSING (exit 1)" "missing-prefs: identifier line" +assert_contains "$output" "Scope the com.controld.ctrld configuration profile" "missing-prefs: remediation hint kept" +assert_not_contains "$calls_out" "ctrld start" "missing-prefs: ctrld start never invoked" + +# --- Case: malformed token -> PROVISION_TOKEN_MALFORMED ----------------- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$MALFORMED_TOKEN" +run_case malformed-token + +assert_eq "$exit_code" "1" "malformed-token: exit code" +assert_contains "$output" "stage=package code=PROVISION_TOKEN_MALFORMED (exit 1)" "malformed-token: identifier line" +assert_not_contains "$output" "$MALFORMED_TOKEN" "malformed-token: token value must not leak" +assert_not_contains "$calls_out" "ctrld start" "malformed-token: ctrld start never invoked" + +# --- Case: bad CustomHostname -> CUSTOM_HOSTNAME_INVALID ----------------- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_HOST_PRESENT=1 FAKE_HOST="bad host!" +run_case bad-hostname + +assert_eq "$exit_code" "1" "bad-hostname: exit code" +assert_contains "$output" "stage=package code=CUSTOM_HOSTNAME_INVALID (exit 1)" "bad-hostname: identifier line" +assert_contains "$output" "bad host!" "bad-hostname: message names the hostname" +assert_not_contains "$output" "$GOOD_TOKEN" "bad-hostname: token value must not leak" +assert_not_contains "$calls_out" "ctrld start" "bad-hostname: ctrld start never invoked" + +# --- Case: token pasted into CustomHostname -> message is redacted -------- +# An admin can fill any managed-prefs key with the token by mistake. The +# CUSTOM_HOSTNAME_INVALID message echoes the value, so the log line must +# carry the redaction marker and never the token. + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_HOST_PRESENT=1 FAKE_HOST="${GOOD_TOKEN}!" +run_case token-in-hostname + +assert_eq "$exit_code" "1" "token-in-hostname: exit code" +assert_contains "$output" "stage=package code=CUSTOM_HOSTNAME_INVALID (exit 1)" "token-in-hostname: identifier line" +assert_contains "$output" "[redacted-token]" "token-in-hostname: message carries the redaction marker" +assert_not_contains "$output" "$GOOD_TOKEN" "token-in-hostname: token value must not leak" + +# --- Case: token pasted into UseDevEnvironment -> warning is redacted ------ + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_DEVENV_PRESENT=1 FAKE_DEVENV="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=0 +run_case token-in-dev-env + +assert_eq "$exit_code" "0" "token-in-dev-env: exit code" +assert_contains "$output" "WARNING: UseDevEnvironment value" "token-in-dev-env: still warns on the bad value" +assert_contains "$output" "[redacted-token]" "token-in-dev-env: warning carries the redaction marker" +assert_not_contains "$output" "$GOOD_TOKEN" "token-in-dev-env: token value must not leak" + +# --- Case: CustomHostname structural failures (RFC1123 mirror) ----------- +# Same rule as cmd/cli/hostname.go's validHostname: 3-64 characters, then +# dot-separated labels that start and end on a letter or digit (no +# leading/trailing hyphen or dot, no empty label). None of these contain a +# disallowed character, so only the structural check can catch them. + +bad_hostname_idx=0 +for bad_host in '-' '.abc' 'abc-' 'a'; do + bad_hostname_idx=$((bad_hostname_idx + 1)) + reset_fakes + export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" + export FAKE_HOST_PRESENT=1 FAKE_HOST="$bad_host" + run_case "bad-hostname-struct-$bad_hostname_idx" + assert_eq "$exit_code" "1" "bad-hostname-struct '$bad_host': exit code" + assert_contains "$output" "stage=package code=CUSTOM_HOSTNAME_INVALID (exit 1)" "bad-hostname-struct '$bad_host': identifier line" + assert_not_contains "$calls_out" "ctrld start" "bad-hostname-struct '$bad_host': ctrld start never invoked" +done + +long_host="" +i=0 +while [ "$i" -lt 90 ]; do + long_host="${long_host}a" + i=$((i + 1)) +done + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_HOST_PRESENT=1 FAKE_HOST="$long_host" +run_case bad-hostname-too-long + +assert_eq "$exit_code" "1" "bad-hostname-too-long: exit code" +assert_contains "$output" "stage=package code=CUSTOM_HOSTNAME_INVALID (exit 1)" "bad-hostname-too-long: identifier line" +assert_not_contains "$calls_out" "ctrld start" "bad-hostname-too-long: ctrld start never invoked" + +# --- Case: dotted CustomHostname is valid, passes through ---------------- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_HOST_PRESENT=1 FAKE_HOST="foo.bar" +export FAKE_CTRLD_EXIT=0 +run_case good-hostname-dotted -run_postinstall "$success_plist" "$success_ctrld" "$success_tmp" +assert_eq "$exit_code" "0" "good-hostname-dotted: exit code" +assert_contains "$calls_out" "--custom-hostname=foo.bar" "good-hostname-dotted: passed through to ctrld" -assert_eq "$exit_code" "0" "success case: postinstall exit code" -assert_contains "$output" "provisioning complete" "success case: output mentions success" +# --- Case: bad InterceptMode, fresh install -> fails, INTERCEPT_MODE_INVALID + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_MODE_PRESENT=1 FAKE_MODE="bogus" +run_case bad-mode-fresh + +assert_eq "$exit_code" "1" "bad-mode-fresh: exit code" +assert_contains "$output" "stage=package code=INTERCEPT_MODE_INVALID (exit 1)" "bad-mode-fresh: identifier line" +assert_not_contains "$output" "$GOOD_TOKEN" "bad-mode-fresh: token value must not leak" +assert_not_contains "$calls_out" "ctrld start" "bad-mode-fresh: ctrld start never invoked" + +# --- Case: bad InterceptMode, upgrade -> warns, keeps mode, exit 0 ------- + +reset_fakes +export FAKE_MODE_PRESENT=1 FAKE_MODE="bogus" +run_case bad-mode-upgrade 1 + +assert_eq "$exit_code" "0" "bad-mode-upgrade: exit code" +assert_contains "$output" "WARNING: unsupported InterceptMode in managed preferences; preserving existing service mode" "bad-mode-upgrade: warns" +assert_contains "$calls_out" "launchctl load" "bad-mode-upgrade: falls back to launchctl load" +assert_not_contains "$calls_out" "ctrld " "bad-mode-upgrade: ctrld start never invoked" + +# --- Case: upgrade reload fails -> SERVICE_RELOAD_FAILED ----------------- +# No managed InterceptMode override, so postinstall falls back to +# reloading the existing service; launchctl itself fails. + +reset_fakes +export FAKE_LAUNCHCTL_EXIT=1 +run_case reload-fails 1 + +assert_eq "$exit_code" "1" "reload-fails: exit code" +assert_contains "$output" "stage=package code=SERVICE_RELOAD_FAILED (exit 1)" "reload-fails: identifier line" +assert_contains "$output" "sudo launchctl load" "reload-fails: remediation hint kept" +assert_contains "$calls_out" "launchctl load" "reload-fails: launchctl load was attempted" +assert_contains "$calls_out" "ctrld diag" "reload-fails: diag block attempted" + +# --- Case: unrecognized UseDevEnvironment value -> warns, still installs - + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_DEVENV_PRESENT=1 FAKE_DEVENV="banana" +export FAKE_CTRLD_EXIT=0 +run_case bad-dev-env + +assert_eq "$exit_code" "0" "bad-dev-env: exit code" +assert_contains "$output" 'WARNING: UseDevEnvironment value "banana" is not recognized; ignoring' "bad-dev-env: warns on the bad value" +assert_contains "$output" "provisioning complete, service installed" "bad-dev-env: install still proceeds" +assert_not_contains "$calls_out" "--dev" "bad-dev-env: unrecognized value never turns on --dev" +assert_not_contains "$output" "$GOOD_TOKEN" "bad-dev-env: token value must not leak" + +# --- Case: ctrld-emitted identifier passthrough -------------------------- +# The token and hostname both pass postinstall's own shape checks; ctrld +# itself rejects the token and reports its own fixed-format identifier, +# which postinstall must surface unmodified. + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=22 +export FAKE_CTRLD_IDENTIFIER="provisioning failed: stage=input code=PROVISION_TOKEN_MALFORMED (exit 22)" +run_case ctrld-identifier-passthrough + +assert_eq "$exit_code" "1" "identifier-passthrough: exit code" +assert_contains "$output" "provisioning failed: stage=input code=PROVISION_TOKEN_MALFORMED (exit 22)" "identifier-passthrough: surfaces ctrld's own identifier" +assert_contains "$output" "ctrld exit 22" "identifier-passthrough: names ctrld's exit code" +assert_not_contains "$output" "$GOOD_TOKEN" "identifier-passthrough: token value must not leak" + +# --- Case: ctrld-emitted listener-stage identifier, capture cleaned up ---- +# A different stage/code than the input-stage example above (a listener +# bind conflict once ctrld actually ran), and explicitly checks the private +# capture temp file left no trace - the original regression this whole +# suite grew from. + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=41 +export FAKE_CTRLD_IDENTIFIER="provisioning failed: stage=listener code=LISTENER_BIND_FAILED (exit 41)" +run_case ctrld-listener-bind-failure + +assert_eq "$exit_code" "1" "listener-bind-failure: exit code" +assert_contains "$output" "stage=listener" "listener-bind-failure: output names the stage" +assert_contains "$output" "LISTENER_BIND_FAILED" "listener-bind-failure: output names the code" +assert_contains "$output" "ctrld exit 41" "listener-bind-failure: names ctrld's exit code" +assert_not_contains "$output" "$GOOD_TOKEN" "listener-bind-failure: token value must not leak" + +leftover=$(find "$fixture/ctrld-listener-bind-failure/tmp" -type f 2>/dev/null) +assert_eq "$leftover" "" "listener-bind-failure: capture temp file removed" + +# --- Case: service start fails after install wrote the plist ------------- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=52 FAKE_CTRLD_CREATE_PLIST=1 +export FAKE_CTRLD_IDENTIFIER="provisioning failed: stage=service code=SERVICE_START_FAILED (exit 52)" +run_case start-fails-plist-present + +assert_eq "$exit_code" "1" "start-fails-plist-present: exit code" +assert_contains "$output" "stage=service code=SERVICE_START_FAILED (exit 52)" "start-fails-plist-present: identifier line" +assert_not_contains "$output" "provisioning complete" "start-fails-plist-present: leftover plist is not success" +assert_not_contains "$output" "$GOOD_TOKEN" "start-fails-plist-present: token value must not leak" + +# --- Case: empty ProvisionToken -> PROFILE_PREFS_MISSING with its own hint -- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="" +run_case empty-token + +assert_eq "$exit_code" "1" "empty-token: exit code" +assert_contains "$output" "stage=package code=PROFILE_PREFS_MISSING (exit 1)" "empty-token: identifier line" +assert_contains "$output" "ProvisionToken value is empty" "empty-token: names the empty value, not a missing profile" +assert_not_contains "$calls_out" "ctrld start" "empty-token: ctrld start never invoked" + +# --- Case: control character in token arrives escaped, passes preflight ---- +# 'defaults read' prints a non-whitespace control character as escaped text +# (a byte 0x01 becomes the four characters \001), so the preflight sees a +# plain 6-64 character value and hands it to ctrld unchanged. The stub emits +# that escaped text; real-defaults mode stores the raw byte and lets the +# real binary escape it. + +reset_fakes +if [ "$real_defaults" = "1" ]; then + export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$(printf 'org-v1-abc\001def')" +else + export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE='org-v1-abc\001def' +fi +export FAKE_CTRLD_EXIT=0 +run_case control-char-token-escaped + +assert_eq "$exit_code" "0" "control-char-token-escaped: preflight passes the escaped text" +assert_contains "$calls_out" 'ctrld start --cd-org=org-v1-abc\001def' "control-char-token-escaped: ctrld receives the escaped text unchanged" +assert_not_contains "$output" "PROVISION_TOKEN_MALFORMED" "control-char-token-escaped: preflight does not classify escaped text as malformed" + +# --- Case: UseDevEnvironment false -> no warning, no --dev ----------------- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_DEVENV_PRESENT=1 FAKE_DEVENV="0" +export FAKE_CTRLD_EXIT=0 +run_case dev-env-false + +assert_eq "$exit_code" "0" "dev-env-false: exit code" +assert_not_contains "$output" "WARNING: UseDevEnvironment" "dev-env-false: a boolean false is a recognized value" +assert_not_contains "$calls_out" "--dev" "dev-env-false: false never turns on --dev" + +# --- Case: success -------------------------------------------------------- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=0 +export FAKE_DIAG_WORKS=1 +run_case success + +assert_eq "$exit_code" "0" "success: exit code" +assert_contains "$output" "provisioning complete, service installed" "success: reports success" +assert_not_contains "$output" "diag:" "success: no diag block on the success path" +assert_not_contains "$output" "$GOOD_TOKEN" "success: token value must not leak" + +leftover=$(find "$fixture/success/tmp" -type f 2>/dev/null) +assert_eq "$leftover" "" "success: capture temp file removed" + +# --- Case: diag present on failure ---------------------------------------- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=22 +export FAKE_CTRLD_IDENTIFIER="provisioning failed: stage=input code=PROVISION_TOKEN_MALFORMED (exit 22)" +export FAKE_DIAG_WORKS=1 +run_case diag-present + +assert_eq "$exit_code" "1" "diag-present: exit code" +assert_contains "$output" "ctrld postinstall: diag: resolver: healthy" "diag-present: diag output appended" +assert_contains "$output" "ctrld postinstall: diag: listener: 127.0.0.1:53 bound" "diag-present: every diag line prefixed" +assert_not_contains "$output" "$GOOD_TOKEN" "diag-present: token value must not leak" + +leftover=$(find "$fixture/diag-present/tmp" -type f 2>/dev/null) +assert_eq "$leftover" "" "diag-present: capture and diag temp files removed" + +# --- Case: diag output redacts the token, fresh install ------------------ +# diag leaks both the real (managed-prefs) token and an unrelated org-v1- +# shaped string; the first is caught by the exact-match swap, the second +# only by the pattern-based one. The rest of the leaked line must survive. + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=22 +export FAKE_CTRLD_IDENTIFIER="provisioning failed: stage=input code=PROVISION_TOKEN_MALFORMED (exit 22)" +export FAKE_DIAG_WORKS=1 +export FAKE_DIAG_LEAK_LINE="leaked: $GOOD_TOKEN and $OTHER_ORG_TOKEN" +run_case diag-redacts-token + +assert_eq "$exit_code" "1" "diag-redacts-token: exit code" +assert_not_contains "$output" "$GOOD_TOKEN" "diag-redacts-token: real token value must not leak" +assert_not_contains "$output" "$OTHER_ORG_TOKEN" "diag-redacts-token: unrelated org-v1- string must not leak" +assert_contains "$output" "leaked: [redacted-token] and org-v1-[redacted]" "diag-redacts-token: both redacted, rest of the line intact" +assert_contains "$output" "ctrld postinstall: diag: resolver: healthy" "diag-redacts-token: unrelated diag lines untouched" + +# --- Case: diag output redacts the token, upgrade path -------------------- +# Same redaction, but on the upgrade branch that applies a managed +# InterceptMode: it never reads ProvisionToken for its own purposes, so +# this proves redact_token is still populated there. + +reset_fakes +export FAKE_MODE_PRESENT=1 FAKE_MODE="standard" +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=22 +export FAKE_CTRLD_IDENTIFIER="provisioning failed: stage=input code=PROVISION_TOKEN_MALFORMED (exit 22)" +export FAKE_DIAG_WORKS=1 +export FAKE_DIAG_LEAK_LINE="leaked: $GOOD_TOKEN and $OTHER_ORG_TOKEN" +run_case diag-redacts-token-upgrade 1 + +assert_eq "$exit_code" "1" "diag-redacts-token-upgrade: exit code" +assert_contains "$calls_out" "ctrld start --intercept-mode off" "diag-redacts-token-upgrade: applied managed InterceptMode" +assert_not_contains "$output" "$GOOD_TOKEN" "diag-redacts-token-upgrade: real token value must not leak" +assert_not_contains "$output" "$OTHER_ORG_TOKEN" "diag-redacts-token-upgrade: unrelated org-v1- string must not leak" +assert_contains "$output" "leaked: [redacted-token] and org-v1-[redacted]" "diag-redacts-token-upgrade: both redacted, rest of the line intact" + +# --- Case: diag absent/failing -> identifier still logged (fallback) ----- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=1 +run_case diag-absent + +assert_eq "$exit_code" "1" "diag-absent: exit code" +assert_contains "$output" "provisioning failed with no identifier; see /var/log/install.log and run ctrld diag" "diag-absent: falls back when ctrld reports no identifier" +assert_not_contains "$output" "invalid/expired token, or no network" "diag-absent: old speculative wording is gone" +assert_not_contains "$output" "ctrld postinstall: diag:" "diag-absent: no diag block when diag fails" +assert_not_contains "$output" "$GOOD_TOKEN" "diag-absent: token value must not leak" + +leftover=$(find "$fixture/diag-absent/tmp" -type f 2>/dev/null) +assert_eq "$leftover" "" "diag-absent: capture temp file removed" + +# --- Case: diag ignores SIGTERM -> SIGKILL escalation bounds the wait ---- +# Without the escalation this would hang for the diag stub's full 30s +# sleep (or longer, on a real ignore-forever daemon); the grace loop above +# already spends up to 10s, so 20s is a generous bound that still catches +# a regression back to a plain SIGTERM-and-wait. + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_CTRLD_EXIT=22 +export FAKE_CTRLD_IDENTIFIER="provisioning failed: stage=input code=PROVISION_TOKEN_MALFORMED (exit 22)" +export FAKE_DIAG_TRAPS_TERM=1 +start_ts=$(date +%s) +run_case diag-ignores-term +end_ts=$(date +%s) +elapsed=$((end_ts - start_ts)) + +assert_eq "$exit_code" "1" "diag-ignores-term: exit code" +if [ "$elapsed" -gt 20 ]; then + fail "diag-ignores-term: took ${elapsed}s, expected the SIGKILL escalation to keep this well under 30s" +fi +assert_not_contains "$output" "$GOOD_TOKEN" "diag-ignores-term: token value must not leak" + +# --- Case: mktemp failure -> TEMP_FILE_UNAVAILABLE, no diag attempted ---- + +reset_fakes +export FAKE_TOKEN_PRESENT=1 FAKE_TOKEN_VALUE="$GOOD_TOKEN" +export FAKE_MKTEMP_FAIL=1 +run_case tempfile-unavailable + +assert_eq "$exit_code" "1" "tempfile-unavailable: exit code" +assert_contains "$output" "stage=package code=TEMP_FILE_UNAVAILABLE (exit 1)" "tempfile-unavailable: identifier line" +assert_contains "$output" "free space in /tmp or TMPDIR" "tempfile-unavailable: remediation hint" +assert_not_contains "$calls_out" "ctrld " "tempfile-unavailable: ctrld never invoked" +assert_not_contains "$output" "ctrld postinstall: diag:" "tempfile-unavailable: no diag block without a capture file" +assert_not_contains "$output" "$GOOD_TOKEN" "tempfile-unavailable: token value must not leak" if [ "$FAILURES" -gt 0 ]; then echo "$FAILURES assertion(s) failed" >&2 exit 1 fi -echo "OK: postinstall surfaces provisioning failure codes without leaking the token" +if [ "$real_defaults" = "1" ]; then + echo "OK (real defaults): postinstall classifies failures, surfaces diagnostics, and never leaks the token" +else + echo "OK (stubbed defaults): postinstall classifies failures, surfaces diagnostics, and never leaks the token" +fi exit 0 From c75989a6eeae7adfe558cfae03e0ffcccadb1672 Mon Sep 17 00:00:00 2001 From: Local Scribe Date: Tue, 15 Sep 2026 16:03:10 +0000 Subject: [PATCH 05/29] darwin: route Zscaler's DNS echo name through the OS resolver --- cmd/cli/dns_proxy.go | 115 +++++++++++- cmd/cli/dns_proxy_test.go | 367 +++++++++++++++++++++++++++++++++++++ docs/dns-intercept-mode.md | 66 +++++++ 3 files changed, 540 insertions(+), 8 deletions(-) diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index 35a2c717..29fd4408 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -31,6 +31,17 @@ import ( const ( staleTTL = 60 * time.Second localTTL = 3600 * time.Second + // zpaDNSEchoDomain is Zscaler Client Connector's fixed public health-check + // name. Client Connector must see the answer to it arrive on its own DNS + // path before it will synthesize addresses for Private Access applications. + zpaDNSEchoDomain = "dnsechotest.zscaler.com" + // zpaDNSEchoPolicyName is reported as the matched policy when the built-in + // ZPA DNS echo bypass routes a query, so log lines name the reason. + zpaDNSEchoPolicyName = "Built-in ZPA DNS echo bypass" + // noRuleMatched is the matchedRule placeholder used when no listener domain + // rule matched the query. It is compared, not only logged: the built-in ZPA + // echo bypass yields to an explicit domain rule for the same name. + noRuleMatched = "no rule" // EDNS0_OPTION_MAC is dnsmasq EDNS0 code for adding mac option. // https://thekelleys.org.uk/gitweb/?p=dnsmasq.git;a=blob;f=src/dns-protocol.h;h=76ac66a8c28317e9c121a74ab5fd0e20f6237dc8;hb=HEAD#l81 // This is also dns.EDNS0LOCALSTART, but define our own constant here for clarification. @@ -40,6 +51,19 @@ const ( selfUninstallMaxQueries = 32 ) +// zpaDNSEchoBypassEnabled limits the built-in ZPA DNS echo bypass to macOS, +// the only platform where pf interception is known to break Zscaler Private +// Access. Tests override it. +var zpaDNSEchoBypassEnabled = runtime.GOOS == "darwin" + +// zpaDNSEchoBypassActive reports whether the built-in Zscaler DNS echo bypass +// applies. Like every other VPN accommodation it is skipped in +// --intercept-mode hard, where the operator has asked for all DNS to go +// through ctrld with no split routing. +func zpaDNSEchoBypassActive() bool { + return zpaDNSEchoBypassEnabled && dnsIntercept && !hardIntercept +} + var osUpstreamConfig = &ctrld.UpstreamConfig{ Name: "OS resolver", Type: ctrld.ResolverTypeOS, @@ -82,6 +106,10 @@ type upstreamForResult struct { matchedRule string matched bool srcAddr string + // zpaDNSEcho reports that the built-in Zscaler DNS echo bypass chose the + // upstreams. proxy() keeps this query out of the response cache so Client + // Connector's health check reaches the OS resolver on every poll. + zpaDNSEcho bool } func (p *prog) addCachedResponse(key dnscache.Key, answer *dns.Msg) { @@ -316,16 +344,68 @@ func (p *prog) upstreamFor(ctx context.Context, defaultUpstreamNum string, lc *c upstreams := []string{upstreamPrefix + defaultUpstreamNum} matchedPolicy := "no policy" matchedNetwork := "no network" - matchedRule := "no rule" + matchedRule := noRuleMatched matched := false + zpaDNSEcho := false res = &upstreamForResult{srcAddr: addr.String()} defer func() { + // Zscaler Private Access: under macOS Intercept Mode, pf routes Client + // Connector's health-check query for zpaDNSEchoDomain into ctrld, which + // answers it from the Control D upstream. Client Connector never sees the + // answer on its own DNS path, so it keeps Private Access disabled. + // + // Route only that one exact name the way a Control D "bypass" rule does: + // an empty upstream list, which proxy() resolves through upstream.os — + // the system resolver set, which under Intercept Mode still includes + // Client Connector's own DNS servers. Matching on the name alone covers + // every record type, the same as a bypass rule. + // + // This runs after policy evaluation, and never touches `matched`. That + // flag is also the source authorization bit: serveDNS refuses a query on + // a Restricted listener when it is false, so forcing it true here would + // let an unauthorized client resolve this name through such a listener. + // + // The scope is one fixed public health-check name: every other domain, + // including all Private Access application domains, stays intercepted and + // filtered by Control D. --intercept-mode hard opts out, the same as it + // opts out of VPN DNS split routing. + if zpaDNSEchoBypassActive() && canonicalName(domain) == zpaDNSEchoDomain { + switch { + case matchedRule == noRuleMatched: + // The listener policy says nothing about this name, so the + // built-in route decides. Network- and MAC-policy targets route + // every name from a source and so state nothing about this one: + // override them, and the default upstream, and label the result + // so logs name the reason. + upstreams = nil + matchedPolicy = zpaDNSEchoPolicyName + matchedRule = zpaDNSEchoDomain + zpaDNSEcho = true + case len(upstreams) == 0: + // An explicit domain rule already selects the OS path: a rule + // with empty targets, which is how a Control D profile's bypass + // list reaches us (see the cfg.Listener rules built from + // resolverConfig.Exclude). That is the documented pre-fix + // workaround for this very bug, so keep the operator's own + // labels but still classify it as the health check — otherwise + // anyone who keeps the workaround after upgrading would route + // correctly and then have the answer served from our cache, + // losing the reconnect fix. + zpaDNSEcho = true + default: + // Any other explicit domain rule is a deliberate non-OS route + // for this exact name. That is operator intent, and the + // supported way to opt out without leaving Intercept Mode, so + // leave the policy's decision untouched. + } + } res.upstreams = upstreams res.matched = matched res.matchedPolicy = matchedPolicy res.matchedNetwork = matchedNetwork res.matchedRule = matchedRule + res.zpaDNSEcho = zpaDNSEcho }() if lc.Policy == nil { @@ -501,6 +581,21 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { serveStaleCache := p.cache != nil && p.cfg.Service.CacheServeStale upstreamConfigs := p.upstreamConfigsFromUpstreamNumbers(upstreams) + // Inverse queries must not be cached: + // https://www.rfc-editor.org/rfc/rfc1035#section-7.4 + // + // Neither must the Zscaler health check. Its whole purpose is to be observed + // arriving on Client Connector's own DNS path, so it has to reach the OS + // resolver on every poll — a locally served answer looks like a successful + // lookup to us and like silence to Client Connector. This matters most on + // reconnect: an answer cached while ZPA was disconnected would otherwise + // outlive InitializeOsResolver() and keep Private Access disabled until the + // entry's TTL (or Service.CacheTTLOverride) expired. + cacheable := p.cache != nil && req.msg.Question[0].Qtype != dns.TypePTR && !req.ufr.zpaDNSEcho + if req.ufr.zpaDNSEcho { + serveStaleCache = false + } + if len(upstreamConfigs) == 0 { upstreamConfigs = []*ctrld.UpstreamConfig{osUpstreamConfig} upstreams = []string{upstreamOS} @@ -530,6 +625,11 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { ctrld.Log(ctx, mainLog.Load().Debug(), "%s, %s, %s -> %v", req.ufr.matchedPolicy, req.ufr.matchedNetwork, req.ufr.matchedRule, upstreams) } else { switch { + case req.ufr.zpaDNSEcho: + // Reached when no policy matched the source, so the branch above did + // not log the bypass. Named explicitly rather than falling into the + // "no explicit policy matched" default. + ctrld.Log(ctx, mainLog.Load().Debug(), "%s, %s -> %v", req.ufr.matchedPolicy, req.ufr.matchedRule, upstreams) case isSrvLanLookup(req.msg): upstreams = []string{upstreamOS} upstreamConfigs = []*ctrld.UpstreamConfig{osUpstreamConfig} @@ -561,8 +661,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { } } - // Inverse query should not be cached: https://www.rfc-editor.org/rfc/rfc1035#section-7.4 - if p.cache != nil && req.msg.Question[0].Qtype != dns.TypePTR { + if cacheable { dns64Prefix, dns64Active := netip.Prefix{}, false if req.msg.Question[0].Qtype == dns.TypeAAAA { dns64Prefix, dns64Active = p.activeDNS64Prefix() @@ -611,7 +710,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { if answer != nil { p.vpnDNS.VPNDNSReachable() ctrld.Log(ctx, mainLog.Load().Debug(), "VPN DNS query successful") - if p.cache != nil { + if cacheable { ttl := 60 * time.Second if len(answer.Answer) > 0 { ttl = time.Duration(answer.Answer[0].Header().Ttl) * time.Second @@ -808,7 +907,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { // set compression, as it is not set by default when unpacking answer.Compress = true - if p.cache != nil && req.msg.Question[0].Qtype != dns.TypePTR { + if cacheable { p.addCachedResponse(dnscache.NewKey(req.msg, upstreams[n]), answer) ctrld.Log(ctx, mainLog.Load().Debug(), "add cached response") } @@ -822,7 +921,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { var synthesizedPrefix netip.Prefix answer, synthesizedPrefix = p.maybeDNS64(ctx, req.msg, answer, func(aReq *dns.Msg) *dns.Msg { key := dnscache.NewKey(aReq, upstreams[n]) - if p.cache != nil { + if cacheable { if cachedValue := p.cache.Get(key); cachedValue != nil { now := time.Now() if cachedValue.Expire.After(now) { @@ -834,12 +933,12 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { } } resolved := resolve(upstreams[n], upstreamConfig, aReq) - if p.cache != nil && resolved != nil && sameQuestion(aReq, resolved) { + if cacheable && resolved != nil && sameQuestion(aReq, resolved) { p.addCachedResponse(key, resolved) } return resolved }) - if p.cache != nil && synthesizedPrefix.IsValid() { + if cacheable && synthesizedPrefix.IsValid() { p.addCachedResponse(dns64CacheKey(req.msg, upstreams[n], synthesizedPrefix), answer) ctrld.Log(ctx, mainLog.Load().Debug(), "dns64: add cached response variant") } diff --git a/cmd/cli/dns_proxy_test.go b/cmd/cli/dns_proxy_test.go index 5462ff04..92421c8a 100644 --- a/cmd/cli/dns_proxy_test.go +++ b/cmd/cli/dns_proxy_test.go @@ -3,6 +3,7 @@ package cli import ( "context" "net" + "sync" "testing" "time" @@ -142,6 +143,372 @@ func Test_prog_upstreamFor(t *testing.T) { } } +// withZPADNSEchoBypass sets the three gates that guard the built-in Zscaler +// DNS echo bypass: the macOS-only platform gate, Intercept Mode, and the +// hard-mode opt-out. +func withZPADNSEchoBypass(t *testing.T, onDarwin, intercept, hard bool) { + t.Helper() + oldBypass, oldIntercept, oldHard := zpaDNSEchoBypassEnabled, dnsIntercept, hardIntercept + zpaDNSEchoBypassEnabled = onDarwin + dnsIntercept = intercept + hardIntercept = hard + t.Cleanup(func() { + zpaDNSEchoBypassEnabled = oldBypass + dnsIntercept = oldIntercept + hardIntercept = oldHard + }) +} + +// listenerWithRules builds a listener whose policy carries only domain rules. +func listenerWithRules(rules ...ctrld.Rule) *ctrld.ListenerConfig { + return &ctrld.ListenerConfig{ + IP: "127.0.0.1", + Port: 53, + Policy: &ctrld.ListenerPolicyConfig{ + Name: "Test Policy", + Rules: rules, + }, + } +} + +// Test_prog_upstreamFor_ZPADNSEchoBypass pins the routing decision for issue +// 601: under macOS Intercept Mode the exact Zscaler health-check name — and +// nothing else — is routed like a Control D bypass rule, i.e. with an empty +// upstream list that proxy() then resolves through upstream.os. +func Test_prog_upstreamFor_ZPADNSEchoBypass(t *testing.T) { + cfg := testhelper.SampleConfig(t) + p := &prog{cfg: cfg} + for _, nc := range p.cfg.Network { + for _, cidr := range nc.Cidrs { + _, ipNet, err := net.ParseCIDR(cidr) + require.NoError(t, err) + nc.IPNets = append(nc.IPNets, ipNet) + } + } + + // A listener whose policy routes the echo name to a specific non-OS + // upstream. That is a deliberate route for this exact name, so it wins. + lcWithEchoRule := listenerWithRules(ctrld.Rule{zpaDNSEchoDomain: []string{"upstream.2"}}) + lcWithZscalerWildcardRule := listenerWithRules(ctrld.Rule{"*.zscaler.com": []string{"upstream.2"}}) + + // Empty targets are how a Control D profile's bypass list reaches ctrld + // (cfg.Listener rules built from resolverConfig.Exclude). This is the + // documented pre-fix workaround, so it must keep routing to the OS path AND + // stay classified as the health check, or upgrading a machine that still + // carries the workaround would lose the no-cache behavior. + lcWithEchoBypassRule := listenerWithRules(ctrld.Rule{zpaDNSEchoDomain: []string{}}) + lcWithZscalerWildcardBypassRule := listenerWithRules(ctrld.Rule{"*.zscaler.com": []string{}}) + + // network.0 targets, i.e. what an un-bypassed query from 192.168.0.1 gets. + notBypassed := []string{"upstream.1", "upstream.0"} + const ( + inNetwork0 = "192.168.0.1:0" // matches network.0, so listener.0's policy matches + outOfPolicy = "10.0.0.1:0" // matches no configured network + ) + + tests := []struct { + name string + onDarwin bool + intercept bool + hard bool + lc *ctrld.ListenerConfig + defaultNum string + srcAddr string + domain string + wantUpstreams []string + wantMatched bool + wantZPAEcho bool + }{ + {"exact echo name routes to the OS resolver path", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain, nil, true, true}, + {"echo name is matched case-insensitively and with a trailing dot", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "DNSEchoTest.ZScaler.COM.", nil, true, true}, + {"echo name is bypassed on a listener with no policy", true, true, false, p.cfg.Listener["1"], "1", inNetwork0, zpaDNSEchoDomain, nil, false, true}, + + // The bypass must not double as source authorization. serveDNS refuses a + // query on a Restricted listener when matched is false, so an + // unauthorized source must stay unmatched even for this name — while + // still being routed to the OS resolver path if the listener does serve + // it. See TestZPADNSEchoBypassDoesNotAuthorizeRestrictedListener. + {"unauthorized source is still unmatched", true, true, false, p.cfg.Listener["0"], "0", outOfPolicy, zpaDNSEchoDomain, nil, false, true}, + + // An explicit domain rule routing the name to a specific upstream + // outranks the built-in route; network/MAC policy targets, which say + // nothing about this name, do not. + {"explicit domain rule for the name wins", true, true, false, lcWithEchoRule, "0", inNetwork0, zpaDNSEchoDomain, []string{"upstream.2"}, true, false}, + {"explicit wildcard rule covering the name wins", true, true, false, lcWithZscalerWildcardRule, "0", inNetwork0, zpaDNSEchoDomain, []string{"upstream.2"}, true, false}, + + // An explicit rule that itself selects the OS path (empty targets, i.e. + // the Control D bypass-list shape) keeps its own labels but is still the + // health check, so proxy() keeps it out of the cache. + {"existing exact bypass rule stays classified as the health check", true, true, false, lcWithEchoBypassRule, "0", inNetwork0, zpaDNSEchoDomain, nil, true, true}, + {"existing wildcard bypass rule stays classified as the health check", true, true, false, lcWithZscalerWildcardBypassRule, "0", inNetwork0, zpaDNSEchoDomain, nil, true, true}, + + // Gating: the bypass exists only where the problem does. + {"not bypassed off macOS", false, true, false, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain, notBypassed, true, false}, + {"not bypassed outside Intercept Mode", true, false, false, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain, notBypassed, true, false}, + {"not bypassed in hard intercept mode", true, true, true, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain, notBypassed, true, false}, + + // Scope: only the one exact name, so no neighbouring name loses filtering. + {"subdomain of the echo name is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "foo." + zpaDNSEchoDomain, notBypassed, true, false}, + {"echo name as a prefix of another domain is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, zpaDNSEchoDomain + ".evil.example", notBypassed, true, false}, + {"name that merely ends with the echo name is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "evil" + zpaDNSEchoDomain, notBypassed, true, false}, + {"parent Zscaler domain is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "zscaler.com", notBypassed, true, false}, + {"ordinary domain is not bypassed", true, true, false, p.cfg.Listener["0"], "0", inNetwork0, "example.com", notBypassed, true, false}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + withZPADNSEchoBypass(t, tc.onDarwin, tc.intercept, tc.hard) + addr, err := net.ResolveUDPAddr("udp", tc.srcAddr) + require.NoError(t, err) + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + + ufr := p.upstreamFor(ctx, tc.defaultNum, tc.lc, addr, "", tc.domain) + + assert.Equal(t, tc.wantUpstreams, ufr.upstreams) + assert.Equal(t, tc.wantMatched, ufr.matched, "matched is the source-authorization bit") + assert.Equal(t, tc.wantZPAEcho, ufr.zpaDNSEcho) + if tc.wantZPAEcho { + // An empty upstream list is the whole mechanism: it is what + // proxy() turns into upstream.os. Guard the translation step + // that makes that substitution fire. + assert.Empty(t, p.upstreamConfigsFromUpstreamNumbers(ufr.upstreams)) + if tc.lc.Policy == nil || len(tc.lc.Policy.Rules) == 0 { + // Only relabelled when the built-in route is the one that + // made the decision; an operator's own matching rule keeps + // its own labels. + assert.Equal(t, zpaDNSEchoPolicyName, ufr.matchedPolicy) + assert.Equal(t, zpaDNSEchoDomain, ufr.matchedRule) + } + } + }) + } +} + +// TestZPADNSEchoBypassDoesNotAuthorizeRestrictedListener pins the serveDNS +// consequence of keeping `matched` out of the bypass: a Restricted listener +// answers only sources that match its policy, and the built-in echo route must +// not become a hole in that for every macOS DNS-intercept instance. +func TestZPADNSEchoBypassDoesNotAuthorizeRestrictedListener(t *testing.T) { + withZPADNSEchoBypass(t, true, true, false) + + cfg := testhelper.SampleConfig(t) + p := &prog{cfg: cfg} + for _, nc := range p.cfg.Network { + for _, cidr := range nc.Cidrs { + _, ipNet, err := net.ParseCIDR(cidr) + require.NoError(t, err) + nc.IPNets = append(nc.IPNets, ipNet) + } + } + lc := p.cfg.Listener["0"] + lc.Restricted = true + + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + for _, tc := range []struct { + name string + srcAddr string + wantRefused bool + }{ + {"source outside every network policy is refused", "10.0.0.1:0", true}, + {"source inside network.0 is served", "192.168.0.1:0", false}, + } { + t.Run(tc.name, func(t *testing.T) { + addr, err := net.ResolveUDPAddr("udp", tc.srcAddr) + require.NoError(t, err) + ufr := p.upstreamFor(ctx, "0", lc, addr, "", zpaDNSEchoDomain) + + // The route itself is unconditional; only the authorization differs. + require.True(t, ufr.zpaDNSEcho) + // This is the exact condition serveDNS uses to answer REFUSED. + assert.Equal(t, tc.wantRefused, !ufr.matched && lc.Restricted) + }) + } +} + +// osResolverStub stands in for the OS resolver during a proxy() test. It +// answers NOERROR for anything, and records what it was actually asked, so a +// test can tell an answer that came off the wire from one served locally. +type osResolverStub struct { + addr string + + mu sync.Mutex + queries []string +} + +// startOSResolverStub points osUpstreamConfig at a local DNS server for the +// duration of the test. It uses the legacy (plain UDP) resolver type because +// ResolverTypeOS resolves through process-wide state in the ctrld package that +// a test in this package cannot address. +func startOSResolverStub(t *testing.T) *osResolverStub { + t.Helper() + + pc, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1)}) + require.NoError(t, err) + + stub := &osResolverStub{addr: pc.LocalAddr().String()} + srv := &dns.Server{PacketConn: pc, Net: "udp"} + srv.Handler = dns.HandlerFunc(func(w dns.ResponseWriter, m *dns.Msg) { + stub.mu.Lock() + stub.queries = append(stub.queries, dns.TypeToString[m.Question[0].Qtype]+" "+m.Question[0].Name) + stub.mu.Unlock() + answer := new(dns.Msg) + answer.SetReply(m) + _ = w.WriteMsg(answer) + }) + + started := make(chan struct{}) + srv.NotifyStartedFunc = func() { close(started) } + go func() { _ = srv.ActivateAndServe() }() + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("timed out waiting for the OS resolver stub to start") + } + + old := osUpstreamConfig + osUpstreamConfig = &ctrld.UpstreamConfig{ + Name: "OS resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: stub.addr, + Timeout: 2000, + } + t.Cleanup(func() { + osUpstreamConfig = old + _ = srv.Shutdown() + }) + return stub +} + +func (s *osResolverStub) received() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.queries...) +} + +// TestZPADNSEchoBypassReachesOSResolverUncached is the regression test for the +// two ways this route could look correct while still leaving ZPA broken. +// +// 1. It must not be served from ctrld's response cache. Client Connector +// enables Private Access only after it observes the echo answer arrive on +// its own DNS path; an answer we serve locally is silence to it. The nasty +// case is an entry cached while ZPA was disconnected, which would otherwise +// outlive InitializeOsResolver() on reconnect and keep Private Access +// disabled for the entry's whole TTL. +// 2. It must apply to every record type, not just A, matching the Control D +// bypass folder the customer confirmed as a workaround. +// +// It runs against three listener shapes, because the profile bypass list the +// customer is using today arrives as a listener domain rule with empty targets +// (see cfg.Listener rules built from resolverConfig.Exclude). Such a rule +// routes to the OS path on its own, so it would silently keep the cache — a +// machine upgraded while still carrying the workaround must get the fix too. +// +// A pre-existing upstream.os entry with a distinguishable rcode is seeded +// before every query, so a locally served answer is detectable. +func TestZPADNSEchoBypassReachesOSResolverUncached(t *testing.T) { + oldDNS64 := dns64NetworkClassFn + dns64NetworkClassFn = func() (bool, bool, error) { return true, false, nil } + t.Cleanup(func() { dns64NetworkClassFn = oldDNS64 }) + + listeners := []struct { + name string + rules []ctrld.Rule + }{ + {"no-rule", nil}, + {"exact-bypass-rule", []ctrld.Rule{{zpaDNSEchoDomain: []string{}}}}, + {"wildcard-bypass-rule", []ctrld.Rule{{"*.zscaler.com": []string{}}}}, + } + qtypes := []uint16{dns.TypeA, dns.TypeAAAA, dns.TypeTXT, dns.TypeHTTPS, dns.TypeMX} + for _, lcCase := range listeners { + for _, qtype := range qtypes { + for _, bypass := range []bool{true, false} { + name := lcCase.name + "/" + dns.TypeToString[qtype] + if bypass { + name += "/bypass" + } else { + name += "/no-bypass" + } + t.Run(name, func(t *testing.T) { + withZPADNSEchoBypass(t, bypass, true, false) + stub := startOSResolverStub(t) + + cfg := testhelper.SampleConfig(t) + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg)} + for _, nc := range p.cfg.Network { + for _, cidr := range nc.Cidrs { + _, ipNet, err := net.ParseCIDR(cidr) + require.NoError(t, err) + nc.IPNets = append(nc.IPNets, ipNet) + } + } + cache, err := dnscache.NewLRUCache(64) + require.NoError(t, err) + p.cache = cache + + lc := p.cfg.Listener["0"] + if lcCase.rules != nil { + lc = listenerWithRules(lcCase.rules...) + } + + msg := newDnsMsgWithHostname(zpaDNSEchoDomain+".", qtype) + // Every candidate upstream has a fresh REFUSED answer + // waiting. The stub answers NOERROR, so the rcode says + // whether we went to the wire or were served from cache. + seedCache := func() { + for _, upstream := range []string{upstreamOS, "upstream.0", "upstream.1", "upstream.2"} { + cached := new(dns.Msg) + cached.SetRcode(msg, dns.RcodeRefused) + p.cache.Add(dnscache.NewKey(msg, upstream), dnscache.NewValue(cached, time.Now().Add(time.Hour))) + } + } + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + require.NoError(t, err) + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + + // Query twice. The second pass is what a Client Connector + // poll after a ZPA reconnect looks like: if the first answer + // had been cached, this one would never reach the resolver. + for pass := 1; pass <= 2; pass++ { + seedCache() + ufr := p.upstreamFor(ctx, "0", lc, addr, "", zpaDNSEchoDomain) + assert.Equal(t, bypass, ufr.zpaDNSEcho) + res := p.proxy(ctx, &proxyRequest{msg: msg, ufr: ufr}) + + require.NotNil(t, res) + require.NotNil(t, res.answer) + if bypass { + assert.False(t, res.cached, "pass %d: the echo query must not be answered from cache", pass) + assert.Equal(t, dns.RcodeSuccess, res.answer.Rcode, + "pass %d: %s echo query must be answered by the OS resolver", pass, dns.TypeToString[qtype]) + } else { + assert.True(t, res.cached, "pass %d: an ordinary query still uses the cache", pass) + assert.Equal(t, dns.RcodeRefused, res.answer.Rcode, + "pass %d: %s echo query must not reach the OS resolver when the bypass is off", pass, dns.TypeToString[qtype]) + } + } + + got := stub.received() + want := dns.TypeToString[qtype] + " " + zpaDNSEchoDomain + "." + if bypass { + assert.Equal(t, []string{want, want}, got, + "the OS resolver must see the echo query on every poll") + // Write side: nothing may be stored under the OS + // upstream either, or the next poll would be local. + cached := p.cache.Get(dnscache.NewKey(msg, upstreamOS)) + require.NotNil(t, cached, "seeded entry disappeared") + assert.Equal(t, dns.RcodeRefused, cached.Msg.Rcode, + "the echo answer must not be written to the cache") + } else { + assert.Empty(t, got, "the OS resolver must not be queried when the bypass is off") + } + }) + } + } + } +} + func TestCache(t *testing.T) { cfg := testhelper.SampleConfig(t) prog := &prog{cfg: cfg} diff --git a/docs/dns-intercept-mode.md b/docs/dns-intercept-mode.md index 0b92dea8..508952ec 100644 --- a/docs/dns-intercept-mode.md +++ b/docs/dns-intercept-mode.md @@ -483,6 +483,7 @@ exemption). | No VPN | None | ✅ All traffic | N/A | | Split DNS (Tailscale non-exit) | ✅ VPN interface | ✅ Non-VPN domains | ✅ Via MagicDNS | | Exit mode (Tailscale exit node) | ❌ None | ✅ All traffic | ✅ Via ctrld split routing | +| Zscaler Private Access | None | ✅ All traffic except one health-check name | ✅ Via Client Connector | | Windscribe | None (different flow) | ✅ All traffic | N/A | | Hard intercept | None | ✅ All traffic | ❌ Not forwarded | @@ -524,6 +525,71 @@ F5 BIG-IP APM VPN is a known source of DNS conflicts with ctrld (a known support - F5's relay proxy (`F5FltSrv`) performs similar functions to ctrld — they are in direct conflict when both active - The seemingly random failure pattern is caused by timing-dependent race conditions between ctrld's watchdog, F5's DNS enforcement, and (optionally) endpoint security inspection +### Zscaler Private Access (macOS) + +Zscaler Client Connector will not synthesize addresses for Private Access +applications until it has confirmed that DNS answers reach it on its own DNS +path. It confirms this by resolving one fixed public name, +`dnsechotest.zscaler.com`, and observing the answer. + +**How the conflict manifests:** + +1. `--intercept-mode dns` is active, so pf redirects port 53 to ctrld +2. Client Connector's query for `dnsechotest.zscaler.com` is redirected into ctrld +3. ctrld answers it from the Control D upstream, so Client Connector never sees the + answer arrive on its own DNS path +4. Client Connector treats domain validation as failed and keeps Private Access + disabled — while ordinary Control D DNS stays perfectly healthy, which is why the + symptom looks like "ZPA is broken" rather than "DNS is broken" + +**How ctrld handles it:** + +On macOS in `dns` mode, ctrld routes that one exact name the same way a Control D +"bypass" rule does: with an empty upstream list, which resolves through +`upstream.os` — the system resolver set, which under Intercept Mode still includes +Client Connector's own DNS servers. Because the decision is made on the name alone, +it covers every record type. + +The answer is **not cached**. Client Connector enables Private Access only after it +observes the echo answer arrive on its own DNS path, so an answer ctrld served from +its cache reads as silence to Client Connector — the query has to reach the OS +resolver on every poll. This matters most on reconnect: an answer cached while ZPA +was disconnected would otherwise outlive `InitializeOsResolver()` and keep Private +Access disabled for the rest of that entry's TTL (or `cache_ttl_override`). + +The scope is deliberately narrow — a single fixed public health-check name: + +- Private Access **application** domains are not special-cased. Once Client Connector + has validated its DNS path it answers them itself, so they never need a ctrld + routing exception; anything it does not claim stays on Control D. +- No interface, resolver range, or pf rule is exempted. There is no ZPA interface + detection, no CGNAT probing, and no passthrough rule. +- Everything else, including every other `zscaler.com` name, stays intercepted and + filtered by Control D. +- `--intercept-mode hard` opts out, the same as it opts out of VPN DNS split + routing: all DNS goes through ctrld and ZPA is not accommodated. +- The route does **not** authorize the query. A `Restricted` listener still answers + only sources that match its policy, so the built-in route is not a hole in + source authorization. +- An explicit listener **domain rule** that routes this name to a specific upstream + outranks the built-in route, and is the supported way to opt out without leaving + Intercept Mode. Network- and MAC-policy targets do not: they route every name + from a source and so state nothing about this one. +- A rule that itself selects the OS path — empty targets, which is how a Control D + profile's bypass list reaches ctrld — keeps its own routing and labels but is still + treated as the health check, so it gets the no-cache behavior too. A machine + upgraded while still carrying the bypass-folder workaround therefore gets the full + fix rather than just the routing half. + +Adding `dnsechotest.zscaler.com` to a Control D bypass folder produces the same +routing and remains a valid workaround on releases without this fix. Disabling +Intercept Mode entirely (`ctrld start --intercept-mode off`) also restores ZPA, at +the cost of interface-based DNS handling. + +**Status:** the routing decision is confirmed by a customer running the bypass-folder +equivalent. It has not yet been verified against a real ZPA tenant with a compiled +build; see the implementation tracker for the outstanding smoke test. + ### Cisco AnyConnect Cisco AnyConnect exhibits similar DNS override behavior. `--intercept-mode dns` mode prevents the conflict by operating at the packet filter level rather than competing for interface DNS settings. From 7da2507d19ba3f49be280230b27b9f9fc47391b6 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Tue, 15 Sep 2026 18:35:17 +0700 Subject: [PATCH 06/29] cli: route organization Internal Domains from managed config Consume resolver.split_dns and generate a policy rule per domain, plus its wildcard, so a suffix and its subdomains route to the OS resolver when no resolvers are given, or to the configured resolvers when they are. Explicit resolvers are exclusive: when all of them fail the query returns SERVFAIL rather than taking the OS catch-all or starting recovery, and it skips VPN split routing, which is auto-detected. Excludes and custom config keep precedence; a refresh regenerates the config, so removals leave no stale rule or upstream. --- cmd/cli/cli.go | 25 + cmd/cli/dns_proxy.go | 50 +- cmd/cli/internal_domains.go | 456 +++++++++++++++++ cmd/cli/internal_domains_review_test.go | 619 +++++++++++++++++++++++ cmd/cli/internal_domains_test.go | 632 ++++++++++++++++++++++++ cmd/cli/loop.go | 9 +- cmd/cli/prog.go | 47 +- docs/internal-domains.md | 170 +++++++ internal/controld/config.go | 33 +- internal/controld/split_dns_test.go | 154 ++++++ 10 files changed, 2175 insertions(+), 20 deletions(-) create mode 100644 cmd/cli/internal_domains.go create mode 100644 cmd/cli/internal_domains_review_test.go create mode 100644 cmd/cli/internal_domains_test.go create mode 100644 docs/internal-domains.md create mode 100644 internal/controld/split_dns_test.go diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 2c4a6a36..870d5be6 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -1021,6 +1021,14 @@ func processCDFlags(ctx context.Context, cfg *ctrld.Config) (*controld.ResolverC } cfg.Listener["0"] = lc + // Organization Internal Domains. Applied after the Magic Folder excludes + // above, so an excluded domain keeps its exclusion, and before the + // platform AD auto-detection in addExtraSplitDnsRule, which skips any + // domain that already has a rule. + if summary := applyInternalDomains(cfg, resolverConfig.SplitDNS); !summary.empty() { + logInternalDomainsSummary(logger, summary) + } + // Set default value. setListenerDefaultValue(cfg) setNetworkDefaultValue(cfg) @@ -1028,6 +1036,23 @@ func processCDFlags(ctx context.Context, cfg *ctrld.Config) (*controld.ResolverC return resolverConfig, nil } +// logInternalDomainsSummary reports what the Internal Domains list produced, +// as counts only. Domain names and resolver addresses are private to the +// organization and stay at debug level. +func logInternalDomainsSummary(logger zerolog.Logger, summary internalDomainsSummary) { + event := logger.Info() + if summary.skipped > 0 || summary.preempted > 0 { + event = logger.Warn() + } + event.Int("domains", summary.domains). + Int("os_resolver", summary.osMode). + Int("explicit_resolver", summary.explicit). + Int("resolvers", summary.resolvers). + Int("skipped", summary.skipped). + Int("preempted", summary.preempted). + Msg("applied organization Internal Domains") +} + // setListenerDefaultValue sets the default value for cfg.Listener if none existed. func setListenerDefaultValue(cfg *ctrld.Config) { if len(cfg.Listener) == 0 { diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index 29fd4408..f1318d17 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -561,7 +561,14 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { // DNS intercept recovery bypass: forward all queries to OS/DHCP resolver. // This runs when upstreams are unreachable (e.g., captive portal network) // and allows the network's DNS to handle authentication pages. - if dnsIntercept && p.recoveryBypass.Load() { + // + // An Internal Domain with explicit resolvers does not take part. The bypass + // exists because general DNS is broken, which says nothing about the + // administrator's own resolvers: they may well be reachable, and sending a + // private name to the network's DNS is the disclosure the explicit + // selection exists to prevent. Such a query continues to the normal flow, + // where it reaches only its configured resolvers and fails if none answer. + if dnsIntercept && p.recoveryBypass.Load() && !internalDomainExplicitUpstreams(req.ufr.upstreams) { ctrld.Log(ctx, mainLog.Load().Debug(), "Recovery bypass active: forwarding to OS resolver") resolver, err := ctrld.NewResolver(osUpstreamConfig) if err == nil { @@ -688,8 +695,17 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { } } - // VPN DNS split routing (only in dns-intercept mode) - if dnsIntercept && p.vpnDNS != nil && len(req.msg.Question) > 0 { + // VPN DNS split routing (only in dns-intercept mode). + // + // An Internal Domain with explicit resolvers is skipped here: the + // administrator named the resolvers for that suffix, and VPN suffixes + // are auto-detected, so handing the query to a VPN DNS server would + // override an explicit selection with a discovered one. + // + // Internal Domains in OS-resolver mode are not skipped, they ask for the + // endpoint's default resolution, which under intercept mode includes + // the VPN's own resolver. + if dnsIntercept && p.vpnDNS != nil && len(req.msg.Question) > 0 && !internalDomainExplicitUpstreams(upstreams) { domain := req.msg.Question[0].Name if vpnServers := p.vpnDNS.UpstreamForDomain(domain); len(vpnServers) > 0 { ctrld.Log(ctx, mainLog.Load().Debug(), "VPN DNS route matched for domain %s, using servers: %v", domain, vpnServers) @@ -836,7 +852,17 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { return answer } - ctrld.Log(ctx, mainLog.Load().Error().Err(err), "failed to resolve query") + // A transport error names the endpoint it failed to reach, which for a + // generated Internal Domain upstream is the organization's private + // resolver address. Report the classification at error level and keep + // the address-bearing error at debug. + if isInternalDomainUpstream(upstream) { + ctrld.Log(ctx, mainLog.Load().Debug().Err(err), "failed to resolve query") + ctrld.Log(ctx, mainLog.Load().Error().Str("failure", internalDomainFailureReason(err)), + "failed to resolve query using an Internal Domain resolver") + } else { + ctrld.Log(ctx, mainLog.Load().Error().Err(err), "failed to resolve query") + } // increase failure count when there is no answer // rehardless of what kind of error we get @@ -949,8 +975,14 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { } ctrld.Log(ctx, mainLog.Load().Error(), "all %v endpoints failed", upstreams) - // if we have no healthy upstreams, trigger recovery flow - if p.leakOnUpstreamFailure() { + // An Internal Domain with explicit resolvers is served only by the + // configured resolvers. When all of them are unreachable the query fails + // here: the OS-resolver catch-all below would send a private name to a resolver + // that was not selected for it, and the recovery flow exists for the loss of + // general DNS, not for one unavailable internal server. + if internalDomainExplicitUpstreams(upstreams) { + ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain resolvers unreachable; not falling back") + } else if p.leakOnUpstreamFailure() { if p.um.countHealthy(upstreams) == 0 { p.recoveryCancelMu.Lock() if p.recoveryCancel == nil { @@ -1943,7 +1975,8 @@ func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig) erro resolver, err := ctrld.NewResolver(uc) if err != nil { - mainLog.Load().Error().Err(err).Msgf("Failed to create resolver for upstream %s", upstream) + logUpstreamProbeFailure(isGeneratedInternalDomainUpstreamRef(upstream, uc), uc, err, + mainLog.Load().Error, "Failed to create resolver for upstream %s", upstream) return err } @@ -1982,7 +2015,8 @@ func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig) erro mainLog.Load().Debug().Err(err).Msgf("Upstream %s check failed after %v (network unreachable)", upstream, duration) return err } - mainLog.Load().Error().Err(err).Msgf("Upstream %s check failed after %v", upstream, duration) + logUpstreamProbeFailure(isGeneratedInternalDomainUpstreamRef(upstream, uc), uc, err, + mainLog.Load().Error, "Upstream %s check failed after %v", upstream, duration) return err } mainLog.Load().Debug().Msgf("Upstream %s responded successfully in %v", upstream, duration) diff --git a/cmd/cli/internal_domains.go b/cmd/cli/internal_domains.go new file mode 100644 index 00000000..3461c063 --- /dev/null +++ b/cmd/cli/internal_domains.go @@ -0,0 +1,456 @@ +package cli + +import ( + "context" + "errors" + "net" + "net/netip" + "slices" + "sort" + "strconv" + "strings" + "syscall" + + "github.com/rs/zerolog" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" + ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" +) + +// internalDomainUpstreamPrefix names the upstreams generated for organization +// Internal Domains that select explicit resolvers. The prefix is what tells +// proxy() that a rule is served only by configured resolvers, so it +// must not collide with the numeric keys used for the Control D upstream. +const internalDomainUpstreamPrefix = "internal_" + +// internalDomainUpstreamName is the Name given to every generated Internal +// Domain upstream. Together with the key prefix and the legacy type it is what +// distinguishes an upstream ctrld generated from one an endpoint custom +// configuration happens to name the same way. +const internalDomainUpstreamName = "Internal Domain resolver" + +// internalDomainsSummary counts the outcome of applying Internal Domains. It +// carries no domain names and no resolver addresses: info and warning logs +// report these counts, and the values themselves stay at debug level. +type internalDomainsSummary struct { + domains int // domains that produced rules + osMode int // domains routed to the OS/default resolver + explicit int // domains routed to configured resolvers + resolvers int // explicit resolver upstreams generated + skipped int // entries dropped: unusable domain, no usable resolver, or duplicate + preempted int // domains already covered by a higher precedence rule +} + +// empty reports whether nothing was applied and nothing was dropped, which is +// the shape of a config whose split_dns list was absent or empty. +func (s internalDomainsSummary) empty() bool { + return s.domains == 0 && s.skipped == 0 && s.preempted == 0 +} + +// internalDomainLabelOK reports whether label satisfies the ASCII hostname +// contract the API validates against: 1-63 characters, letters, digits and +// hyphens only, and no leading or trailing hyphen. Punycode labels are plain +// LDH and pass unchanged. +func internalDomainLabelOK(label string) bool { + if len(label) == 0 || len(label) > 63 { + return false + } + if label[0] == '-' || label[len(label)-1] == '-' { + return false + } + for i := 0; i < len(label); i++ { + c := label[i] + switch { + case c >= 'a' && c <= 'z', c >= '0' && c <= '9', c == '-': + default: + return false + } + } + return true +} + +// normalizeInternalDomain canonicalizes a domain to the form policy rules are +// matched in, or returns "" if it is not a domain ctrld will route. +// +// Canonicalization is limited to what cannot change which names match: +// lowercasing, surrounding whitespace, the root dot, and a leading "*." (the +// caller adds the wildcard rule itself). What remains is validated as an ASCII +// hostname suffix rather than screened against a list of bad characters, +// because a denylist admits whatever it forgot: an embedded newline, quote, +// colon or control character would otherwise reach a generated rule. +// +// An invalid entry is rejected rather than repaired. Deleting the offending +// characters would produce a different suffix from the one that was +// configured, and routing a private domain to the wrong place is worse than +// not routing it. Single-label names stay supported. +// +// The rule matches the API's own validation, so a value the dashboard accepted +// passes here too. +func normalizeInternalDomain(domain string) string { + d := strings.ToLower(strings.TrimSpace(domain)) + d = strings.TrimPrefix(d, "*.") + d = strings.TrimSuffix(d, ".") + if d == "" || len(d) > 253 { + return "" + } + for _, label := range strings.Split(d, ".") { + if !internalDomainLabelOK(label) { + return "" + } + } + return d +} + +// internalDomainEndpoint converts a resolver to an endpoint ctrld can dial. +// A bare IPv4 or IPv6 address gets the default DNS port; and address that +// already carries a port keeps it. Anything else is rejected, so a malformed +// entry cannot silently become a hostname lookup. +func internalDomainEndpoint(resolver string) (string, bool) { + s := strings.TrimSpace(resolver) + if s == "" { + return "", false + } + if addr, err := netip.ParseAddr(s); err == nil { + return net.JoinHostPort(addr.String(), "53"), true + } + host, port, err := net.SplitHostPort(s) + if err != nil { + return "", false + } + addr, err := netip.ParseAddr(host) + if err != nil { + return "", false + } + p, err := strconv.Atoi(port) + if err != nil || p < 1 || p > 65535 { + return "", false + } + return net.JoinHostPort(addr.String(), port), true +} + +// internalDomainMode reports whether entry selects explicit resolvers, and +// whether ctrld understands its mode at all. +// +// Mode is authoritative when the API sends it. "os" ignores any resolver +// addresses the entry still carries, so switching a domain back to the OS +// resolver cannot be undone by addresses a previous selection left behind; +// "resolvers" stays explicit even when the list it points at turns out to be +// unusable, rather than silently becoming OS resolution. An empty mode is a +// deployment that predates the field, where the list is the only signal. +// +// An unrecognized mode is not guessed at: routing a private domain by guess is +// the disclosure this feature exists to prevent. +func internalDomainMode(entry controld.SplitDNS) (explicit, known bool) { + switch strings.ToLower(strings.TrimSpace(entry.Mode)) { + case controld.SplitDNSModeOS: + return false, true + case controld.SplitDNSModeResolvers: + return true, true + case "": + return len(entry.Resolvers) > 0, true + default: + return false, false + } +} + +// internalDomainRouting returns the endpoints an entry routes to, or a reason +// it cannot be used. No endpoints and no reason means OS resolution. +// +// Generation and refresh comparison both go through here, so the two can never +// disagree about what an entry means. +func internalDomainRouting(entry controld.SplitDNS) (endpoints []string, reason string) { + explicit, known := internalDomainMode(entry) + if !known { + return nil, "unknown_mode" + } + if !explicit { + return nil, "" + } + endpoints = make([]string, 0, len(entry.Resolvers)) + for _, resolver := range entry.Resolvers { + if endpoint, ok := internalDomainEndpoint(resolver); ok { + endpoints = append(endpoints, endpoint) + } + } + // The administrator selected explicit resolvers and none of them parsed. + // Falling back to the OS resolver would turn that selection into a + // different one, so the entry is dropped and the domain keeps its normal + // path instead. + if len(endpoints) == 0 { + return nil, "no_usable_resolver" + } + return endpoints, "" +} + +// internalDomainRuleExists reports whether the listener policy already routes +// source. Internal Domains never overwrite a rule that is already there, which +// is what gives Magic Folder excludes and endpoint custom configuration +// precedence over them. +func internalDomainRuleExists(lc *ctrld.ListenerConfig, source string) bool { + if lc.Policy == nil { + return false + } + for _, rule := range lc.Policy.Rules { + if _, ok := rule[source]; ok { + return true + } + } + return false +} + +// internalDomainSpecificity ranks a normalized domain by label count, which is +// how much of a name it pins down. "z.example.com" ranks above "example.com". +func internalDomainSpecificity(domain string) int { + if domain == "" { + return 0 + } + return strings.Count(domain, ".") + 1 +} + +// orderInternalDomainsBySpecificity returns entries most-specific first. +// +// upstreamFor matches policy rules in slice order and takes the first hit, so +// generation order is routing precedence. The API sorts Internal Domains +// bytewise by domain, which puts a parent suffix ahead of its children: +// appending in that order would let "*.example.com" swallow every query a +// configured "z.example.com" rule was meant to answer, and the administrator +// never chose that priority. Overlapping suffixes are permitted, so the more +// specific one has to win. +// +// The sort is stable, so entries that are equally specific keep API order and +// the first of two identical domains still wins the duplicate check. +func orderInternalDomainsBySpecificity(entries []controld.SplitDNS) []controld.SplitDNS { + ordered := make([]controld.SplitDNS, len(entries)) + copy(ordered, entries) + sort.SliceStable(ordered, func(i, j int) bool { + a, b := normalizeInternalDomain(ordered[i].Domain), normalizeInternalDomain(ordered[j].Domain) + if sa, sb := internalDomainSpecificity(a), internalDomainSpecificity(b); sa != sb { + return sa > sb + } + return a < b + }) + return ordered +} + +// applyInternalDomains adds a policy rule per organization Internal Domain to +// every listener in cfg, and the upstreams those rules need. +// +// Each domain contributes two rules, the domain itself and "*." + domain, which +// is how a suffix and all of its subdomains are matched. An entry with no +// resolvers routes to the OS/default resolver by way of an empty upstream list, +// the same representation Magic Folder excludes use. An entry with resolvers +// gets one generated upstream per address, listed in configured order so the +// later ones act as failover for the first. +// +// cfg is rebuilt from the API response on every fetch, so this function is the +// only writer of internal_* upstreams: a removed domain leaves nothing behind. +func applyInternalDomains(cfg *ctrld.Config, entries []controld.SplitDNS) internalDomainsSummary { + var summary internalDomainsSummary + if len(entries) == 0 || len(cfg.Listener) == 0 { + return summary + } + if cfg.Upstream == nil { + cfg.Upstream = make(map[string]*ctrld.UpstreamConfig) + } + seen := make(map[string]struct{}, len(entries)) + for _, entry := range orderInternalDomainsBySpecificity(entries) { + domain := normalizeInternalDomain(entry.Domain) + if domain == "" { + mainLog.Load().Debug().Msgf("internal domains: skipping unusable domain %q", entry.Domain) + summary.skipped++ + continue + } + if _, dup := seen[domain]; dup { + mainLog.Load().Debug().Msgf("internal domains: skipping duplicate domain %q", domain) + summary.skipped++ + continue + } + + // Routing is resolved, and preemption decided, before anything is + // written: an entry that cannot be used, or a domain already covered by + // a higher precedence rule, must not leave a generated upstream behind + // with no rule referring to it. + endpoints, reason := internalDomainRouting(entry) + if reason != "" { + mainLog.Load().Debug().Msgf("internal domains: skipping %q: %s", domain, reason) + summary.skipped++ + continue + } + seen[domain] = struct{}{} + + sources := []string{domain, "*." + domain} + free := make(map[*ctrld.ListenerConfig][]string, len(cfg.Listener)) + anyFree := false + for _, lc := range cfg.Listener { + for _, source := range sources { + if internalDomainRuleExists(lc, source) { + mainLog.Load().Debug().Msgf("internal domains: %q already routed by a higher precedence rule", source) + continue + } + free[lc] = append(free[lc], source) + anyFree = true + } + } + if !anyFree { + summary.preempted++ + continue + } + + targets := make([]string, 0, len(endpoints)) + for _, endpoint := range endpoints { + key := internalDomainUpstreamPrefix + strconv.Itoa(summary.resolvers) + cfg.Upstream[key] = &ctrld.UpstreamConfig{ + Name: internalDomainUpstreamName, + Type: ctrld.ResolverTypeLegacy, + Endpoint: endpoint, + Timeout: 5000, + } + targets = append(targets, upstreamPrefix+key) + summary.resolvers++ + } + for lc, sources := range free { + if lc.Policy == nil { + lc.Policy = &ctrld.ListenerPolicyConfig{} + } + for _, source := range sources { + lc.Policy.Rules = append(lc.Policy.Rules, ctrld.Rule{source: targets}) + } + } + summary.domains++ + if len(targets) == 0 { + summary.osMode++ + mainLog.Load().Debug().Msgf("internal domains: %q routed to the OS resolver", domain) + } else { + summary.explicit++ + mainLog.Load().Debug().Msgf("internal domains: %q routed to %v", domain, targets) + } + } + return summary +} + +// isGeneratedInternalDomainUpstream reports whether a cfg.Upstream entry is one +// applyInternalDomains created. +// +// Generated Internal Domain resolvers are part of the managed Control D +// configuration, not upstreams the endpoint operator chose, so eligibility +// checks that ask "is this a plain single-upstream Control D install" must not +// count them. The key prefix alone is not proof of that, because an endpoint +// custom configuration replaces the generated config outright and could name an +// upstream anything; the shape this function checks is only produced here. +func isGeneratedInternalDomainUpstream(name string, uc *ctrld.UpstreamConfig) bool { + return uc != nil && + strings.HasPrefix(name, internalDomainUpstreamPrefix) && + uc.Type == ctrld.ResolverTypeLegacy && + uc.Name == internalDomainUpstreamName +} + +// isInternalDomainUpstream reports whether an upstream reference was generated +// for an Internal Domain with explicit resolvers. +func isInternalDomainUpstream(upstream string) bool { + return strings.HasPrefix(upstream, upstreamPrefix+internalDomainUpstreamPrefix) +} + +// internalDomainExplicitUpstreams reports whether every upstream serving a +// query is an configured Internal Domain resolver. +// +// proxy() uses this to keep such a query off the OS-resolver catch-all when the +// configured resolvers are unreachable: the administrator selected those +// resolvers and no other, so a private name must fail rather than be sent +// somewhere it was not meant to go. It also keeps an unreachable internal +// resolver from triggering the endpoint-wide recovery flow, which exists for +// the loss of general DNS, not for one unavailable internal server. +func internalDomainExplicitUpstreams(upstreams []string) bool { + if len(upstreams) == 0 { + return false + } + for _, upstream := range upstreams { + if !isInternalDomainUpstream(upstream) { + return false + } + } + return true +} + +// internalDomainsEqual reports whether two Internal Domains lists would produce +// the same routing. Domains are compared in canonical form and independently of +// the order the API returned them, so an unchanged list does not trigger a +// reload on every refresh. Resolver order is significant: it is the order the +// administrator chose, and it decides which resolver is tried first. +func internalDomainsEqual(a, b []controld.SplitDNS) bool { + canonical := func(entries []controld.SplitDNS) []controld.SplitDNS { + out := make([]controld.SplitDNS, 0, len(entries)) + for _, entry := range entries { + domain := normalizeInternalDomain(entry.Domain) + if domain == "" { + continue + } + // Entries that cannot be used route nothing, so they cannot make + // an otherwise unchanged list look changed. + endpoints, reason := internalDomainRouting(entry) + if reason != "" { + continue + } + out = append(out, controld.SplitDNS{Domain: domain, Resolvers: endpoints}) + } + sort.Slice(out, func(i, j int) bool { return out[i].Domain < out[j].Domain }) + return out + } + x, y := canonical(a), canonical(b) + if len(x) != len(y) { + return false + } + for i := range x { + if x[i].Domain != y[i].Domain || !slices.Equal(x[i].Resolvers, y[i].Resolvers) { + return false + } + } + return true +} + +// internalDomainFailureReason classifies a resolver failure for logging. The +// error itself names the endpoint that could not be reached, so only this +// classification may appear above debug level. +func internalDomainFailureReason(err error) string { + switch { + case err == nil: + return "no_answer" + case errors.Is(err, syscall.ECONNREFUSED): + return "refused" + case errors.Is(err, syscall.EADDRNOTAVAIL): + return "source_unavailable" + case ctrldnet.IsUnreachable(err): + return "unreachable" + } + var netErr net.Error + if errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &netErr) && netErr.Timeout()) { + return "timeout" + } + return "error" +} + +// isGeneratedInternalDomainUpstreamRef is isGeneratedInternalDomainUpstream for +// call sites that hold an "upstream." reference rather than the bare key. +func isGeneratedInternalDomainUpstreamRef(upstream string, uc *ctrld.UpstreamConfig) bool { + return isGeneratedInternalDomainUpstream(strings.TrimPrefix(upstream, upstreamPrefix), uc) +} + +// logUpstreamProbeFailure reports a background probe failure against an +// upstream. +// +// Loop checking and recovery checking run on timers, so for a generated +// Internal Domain resolver they would publish the organization's private +// address without any user ever querying the domain. They get the same +// treatment as a query failure: endpoint and the address-bearing error at +// debug, a classification above it, so the outcome stays visible while the +// address does not. +func logUpstreamProbeFailure(generated bool, uc *ctrld.UpstreamConfig, err error, level func() *zerolog.Event, format string, args ...any) { + detail := append([]any{}, args...) + if !generated { + level().Err(err).Msgf(format+" for upstream: %q, endpoint: %q", append(detail, uc.Name, uc.Endpoint)...) + return + } + mainLog.Load().Debug().Err(err).Msgf(format+" for upstream: %q, endpoint: %q", append(detail, uc.Name, uc.Endpoint)...) + level().Str("failure", internalDomainFailureReason(err)). + Msgf(format+" for an Internal Domain resolver", detail...) +} diff --git a/cmd/cli/internal_domains_review_test.go b/cmd/cli/internal_domains_review_test.go new file mode 100644 index 00000000..f26e9cb9 --- /dev/null +++ b/cmd/cli/internal_domains_review_test.go @@ -0,0 +1,619 @@ +package cli + +import ( + "context" + "net" + "strings" + "sync" + "testing" + "time" + + "github.com/miekg/dns" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// dnsFixture is a loopback DNS server that records the questions it is asked, +// so a test can prove where a query actually went on the wire rather than +// inferring it from an rcode. +type dnsFixture struct { + addr string + mu sync.Mutex + qs []string +} + +func (f *dnsFixture) questions() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.qs...) +} + +func startDNSFixture(t *testing.T) *dnsFixture { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + f := &dnsFixture{addr: pc.LocalAddr().String()} + started := make(chan struct{}) + server := &dns.Server{ + PacketConn: pc, + NotifyStartedFunc: func() { close(started) }, + Handler: dns.HandlerFunc(func(w dns.ResponseWriter, msg *dns.Msg) { + f.mu.Lock() + if len(msg.Question) > 0 { + f.qs = append(f.qs, msg.Question[0].Name) + } + f.mu.Unlock() + reply := new(dns.Msg) + reply.SetRcode(msg, dns.RcodeSuccess) + _ = w.WriteMsg(reply) + }), + } + go func() { _ = server.ActivateAndServe() }() + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("DNS fixture did not start") + } + t.Cleanup(func() { _ = server.Shutdown() }) + return f +} + +// useOSResolverFixture points the OS-resolver boundary at a loopback server for +// the duration of the test, so "went to the OS resolver" is observable. +func useOSResolverFixture(t *testing.T, f *dnsFixture) { + t.Helper() + prev := osUpstreamConfig + osUpstreamConfig = &ctrld.UpstreamConfig{ + Name: "OS resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: f.addr, + Timeout: 2000, + } + t.Cleanup(func() { osUpstreamConfig = prev }) +} + +func useDNSIntercept(t *testing.T) { + t.Helper() + prev := dnsIntercept + dnsIntercept = true + t.Cleanup(func() { dnsIntercept = prev }) +} + +func newInternalDomainsProg(t *testing.T, cfg *ctrld.Config) *prog { + t.Helper() + p := &prog{cfg: cfg} + p.um = newUpstreamMonitor(cfg) + p.lanLoopGuard = newLoopGuard() + p.ptrLoopGuard = newLoopGuard() + return p +} + +// The DNS-intercept recovery bypass forwards everything to the OS/DHCP resolver +// while general DNS is broken. An Internal Domain with explicit resolvers must +// not take part: its resolvers may be perfectly healthy, and the private name +// must not reach the network's DNS. +func TestInternalDomainsExplicitResolverSurvivesRecoveryBypass(t *testing.T) { + osFixture := startDNSFixture(t) + internalFixture := startDNSFixture(t) + useOSResolverFixture(t, osFixture) + useDNSIntercept(t) + + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "corp.example", Resolvers: []string{internalFixture.addr}}, + }) + p := newInternalDomainsProg(t, cfg) + p.recoveryBypass.Store(true) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ask := func(name string) *proxyResponse { + t.Helper() + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", strings.TrimSuffix(name, ".")) + return p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname(name, dns.TypeA), ufr: ufr}) + } + + res := ask("host.corp.example.") + if got := internalFixture.questions(); len(got) != 1 || got[0] != "host.corp.example." { + t.Errorf("internal resolver questions = %v, want the query to reach it", got) + } + if got := osFixture.questions(); len(got) != 0 { + t.Errorf("recovery bypass leaked an Internal Domain query to the OS resolver: %v", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Errorf("answer = %v, want the internal resolver's NOERROR", res) + } + + // Control: an ordinary query still takes the bypass. + if res := ask("www.example.net."); res == nil || res.answer == nil { + t.Errorf("ordinary query during recovery bypass = %v", res) + } + if got := osFixture.questions(); len(got) != 1 || got[0] != "www.example.net." { + t.Errorf("OS resolver questions = %v, want the ordinary query to take the bypass", got) + } +} + +// When the configured resolvers are unreachable the query must fail rather than +// reach the OS resolver, and the failure must be observable on the wire as an +// attempt against the configured address. +func TestInternalDomainsUnreachableResolverDoesNotReachOSResolver(t *testing.T) { + osFixture := startDNSFixture(t) + useOSResolverFixture(t, osFixture) + useDNSIntercept(t) + + // A loopback port nothing listens on: the dial fails fast and locally. + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + dead := pc.LocalAddr().String() + _ = pc.Close() + + cfg := internalDomainsTestConfig() + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true) + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{dead}}}) + p := newInternalDomainsProg(t, cfg) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + + if got := osFixture.questions(); len(got) != 0 { + t.Errorf("unreachable internal resolver leaked to the OS resolver: %v", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeServerFailure { + t.Errorf("answer = %v, want SERVFAIL", res) + } +} + +// The API returns Internal Domains sorted bytewise by domain, which puts a +// parent suffix ahead of its children. Generation order is routing precedence, +// so the more specific suffix has to win regardless of API order. +func TestInternalDomainsOverlappingSuffixPrecedence(t *testing.T) { + for _, tc := range []struct { + name string + entries []controld.SplitDNS + want map[string][]string + }{ + { + name: "parent os, child explicit", + entries: []controld.SplitDNS{ + {Domain: "example.com"}, + {Domain: "z.example.com", Resolvers: []string{"10.0.0.2"}}, + }, + want: map[string][]string{ + "z.example.com": {"upstream.internal_0"}, + "host.z.example.com": {"upstream.internal_0"}, + "a.b.z.example.com": {"upstream.internal_0"}, + "example.com": {}, + "other.example.com": {}, + "notz.example.com": {}, + "z.example.com.evil.gg": {"upstream.0"}, + "example.org": {"upstream.0"}, + }, + }, + { + name: "parent explicit, child os", + entries: []controld.SplitDNS{ + {Domain: "example.com", Resolvers: []string{"10.0.0.2"}}, + {Domain: "z.example.com"}, + }, + want: map[string][]string{ + "z.example.com": {}, + "host.z.example.com": {}, + "example.com": {"upstream.internal_0"}, + "other.example.com": {"upstream.internal_0"}, + "example.org": {"upstream.0"}, + }, + }, + { + name: "three levels, API order", + entries: []controld.SplitDNS{ + {Domain: "example.com", Resolvers: []string{"10.0.0.1"}}, + {Domain: "y.example.com"}, + {Domain: "x.y.example.com", Resolvers: []string{"10.0.0.3"}}, + }, + want: map[string][]string{ + "host.x.y.example.com": {"upstream.internal_0"}, + "x.y.example.com": {"upstream.internal_0"}, + "other.y.example.com": {}, + "y.example.com": {}, + "other.example.com": {"upstream.internal_1"}, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, tc.entries) + p := &prog{cfg: cfg} + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + for domain, want := range tc.want { + ufr := p.upstreamFor(context.Background(), "0", cfg.Listener["0"], addr, "", domain) + if strings.Join(ufr.upstreams, ",") != strings.Join(want, ",") { + t.Errorf("upstreamFor(%q) = %v, want %v", domain, ufr.upstreams, want) + } + } + }) + } +} + +// Ordering is a generation concern; it must not make an unchanged list look +// changed at refresh time. +func TestInternalDomainsOrderingIsRefreshStable(t *testing.T) { + apiOrder := []controld.SplitDNS{ + {Domain: "example.com"}, + {Domain: "z.example.com", Resolvers: []string{"10.0.0.2"}}, + } + if !internalDomainsEqual(apiOrder, orderInternalDomainsBySpecificity(apiOrder)) { + t.Error("reordering for specificity must not read as a configuration change") + } + first, second := internalDomainsTestConfig(), internalDomainsTestConfig() + applyInternalDomains(first, apiOrder) + applyInternalDomains(second, orderInternalDomainsBySpecificity(apiOrder)) + if len(first.Listener["0"].Policy.Rules) != len(second.Listener["0"].Policy.Rules) { + t.Error("generation must not depend on the order the API returned") + } +} + +// Generated Internal Domain upstreams carry the organization's private resolver +// addresses. The production setup path must not publish them above debug. +func TestInternalDomainsSetupKeepsResolverAddressesOutOfInfoLogs(t *testing.T) { + const secret = "10.77.88.99" + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "corp.example", Resolvers: []string{secret}}, + }) + // Drop the Control D upstream: setupUpstream would try to bootstrap it. + delete(cfg.Upstream, "0") + + p := newInternalDomainsProg(t, cfg) + before := len(logOutput.String()) + p.setupUpstream(cfg) + out := logOutput.String()[before:] + + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + if line == "" || strings.Contains(line, `"level":"debug"`) { + continue + } + if strings.Contains(line, secret) { + t.Errorf("resolver address disclosed above debug level: %s", line) + } + } + if !strings.Contains(out, secret) { + t.Errorf("expected the address at debug level for diagnosis, got: %s", out) + } +} + +// A transport error names the endpoint it could not reach. For an Internal +// Domain upstream that endpoint is private, so only a classification may appear +// at error level. +func TestInternalDomainsResolverFailureLogsNoAddress(t *testing.T) { + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + dead := pc.LocalAddr().String() + _ = pc.Close() + + cfg := internalDomainsTestConfig() + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{dead}}}) + p := newInternalDomainsProg(t, cfg) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + before := len(logOutput.String()) + p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + out := logOutput.String()[before:] + + host, _, _ := net.SplitHostPort(dead) + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + if line == "" || strings.Contains(line, `"level":"debug"`) { + continue + } + if strings.Contains(line, dead) || strings.Contains(line, host+":") { + t.Errorf("resolver address disclosed above debug level: %s", line) + } + } + if !strings.Contains(out, `"failure":`) { + t.Errorf("expected a classified failure at error level, got: %s", out) + } +} + +// selfUninstallEligibility runs the production setup path and reports whether +// the REFUSED-triggered deletion check stays available. +func selfUninstallEligibility(t *testing.T, p *prog, cfg *ctrld.Config) bool { + t.Helper() + p.setupUpstream(cfg) + return p.canSelfUninstall.Load() +} + +// Generated Internal Domain resolvers are part of the managed configuration. +// Counting them as independent upstreams made an ordinary managed install look +// custom, which silently disabled the REFUSED-triggered deletion check. +func TestInternalDomainsKeepSelfUninstallEligibility(t *testing.T) { + for _, tc := range []struct { + name string + entries []controld.SplitDNS + extra map[string]*ctrld.UpstreamConfig + want bool + }{ + {name: "no internal domains", want: true}, + { + name: "os mode", + entries: []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeOS}}, + want: true, + }, + { + name: "explicit mode", + entries: []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}}, + want: true, + }, + { + name: "several explicit domains", + entries: []controld.SplitDNS{ + {Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53", "10.0.0.54"}}, + {Domain: "lab.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.55"}}, + }, + want: true, + }, + { + // A genuinely custom multi-upstream install keeps its safeguard. + name: "a real second upstream", + extra: map[string]*ctrld.UpstreamConfig{"1": {Endpoint: "https://dns.example.net/x", Type: ctrld.ResolverTypeDOH, BootstrapIP: "192.0.2.1"}}, + want: false, + }, + { + // Only the shape applyInternalDomains produces is discounted, so a + // custom config cannot buy eligibility by naming an upstream + // "internal_0". + name: "an upstream merely named like a generated one", + extra: map[string]*ctrld.UpstreamConfig{"internal_0": {Name: "mine", Endpoint: "https://dns.example.net/x", Type: ctrld.ResolverTypeDOH, BootstrapIP: "192.0.2.1"}}, + want: false, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, tc.entries) + for k, uc := range tc.extra { + cfg.Upstream[k] = uc + } + p := newInternalDomainsProg(t, cfg) + if got := selfUninstallEligibility(t, p, cfg); got != tc.want { + t.Errorf("canSelfUninstall = %v, want %v (upstreams: %d)", got, tc.want, len(cfg.Upstream)) + } + }) + } +} + +// Eligibility is recomputed on every setup, so a reload that adds a real second +// upstream withdraws it and one that removes Internal Domains keeps it. The old +// code only ever raised the flag, which made startup and refresh disagree. +func TestSelfUninstallEligibilityRecomputedAcrossReloads(t *testing.T) { + explicit := []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}} + + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, explicit) + p := newInternalDomainsProg(t, cfg) + if !selfUninstallEligibility(t, p, cfg) { + t.Fatal("fresh start with explicit Internal Domains must stay eligible") + } + + // Reload: Internal Domains removed. Still a plain managed install. + removed := internalDomainsTestConfig() + if !selfUninstallEligibility(t, p, removed) { + t.Error("removing Internal Domains must keep eligibility") + } + + // Reload: a genuinely custom second upstream appears. + custom := internalDomainsTestConfig() + custom.Upstream["1"] = &ctrld.UpstreamConfig{Endpoint: "https://dns.example.net/x", Type: ctrld.ResolverTypeDOH, BootstrapIP: "192.0.2.1"} + if selfUninstallEligibility(t, p, custom) { + t.Error("a real second upstream must withdraw eligibility, not stay latched on") + } + + // Reload: back to the managed config with Internal Domains. + again := internalDomainsTestConfig() + applyInternalDomains(again, explicit) + if !selfUninstallEligibility(t, p, again) { + t.Error("eligibility must come back with the managed config") + } +} + +// The visible consequence: with explicit Internal Domains configured, a REFUSED +// answer is still counted toward the deletion check instead of being ignored. +// Counting alone performs no uninstall; that stays gated on the API confirming +// the device is gone. +func TestInternalDomainsKeepRefusedQueryCounting(t *testing.T) { + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}, + }) + p := newInternalDomainsProg(t, cfg) + p.setupUpstream(cfg) + + refused := new(dns.Msg) + refused.SetRcode(newDnsMsgWithHostname("host.corp.example.", dns.TypeA), dns.RcodeRefused) + p.doSelfUninstall(refused) + + p.selfUninstallMu.Lock() + count, checking := p.refusedQueryCount, p.checkingSelfUninstall + p.selfUninstallMu.Unlock() + if count != 1 { + t.Errorf("refusedQueryCount = %d, want 1", count) + } + if checking { + t.Error("a single REFUSED answer must not start a device-status check") + } +} + +// logLinesAboveDebug returns the log lines emitted since mark that a support +// bundle reader would see at info level or above. +func logLinesAboveDebug(mark int) []string { + var out []string + for _, line := range strings.Split(strings.TrimSpace(logOutput.String()[mark:]), "\n") { + if line == "" || strings.Contains(line, `"level":"debug"`) { + continue + } + out = append(out, line) + } + return out +} + +// deadUDPAddr returns a loopback address with nothing listening, so a dial +// fails fast and locally with connection refused. +func deadUDPAddr(t *testing.T) string { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := pc.LocalAddr().String() + _ = pc.Close() + return addr +} + +// blackholeUDPAddr returns a loopback address that accepts packets and never +// answers, so an exchange fails with a timeout rather than a refusal. +func blackholeUDPAddr(t *testing.T) string { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = pc.Close() }) + return pc.LocalAddr().String() +} + +// internalDomainsProbeConfig generates an explicit Internal Domain pointing at +// addr, with a short timeout so a blackholed probe fails quickly. +func internalDomainsProbeConfig(t *testing.T, addr string) *ctrld.Config { + t.Helper() + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{addr}}, + }) + uc := cfg.Upstream["internal_0"] + if uc == nil { + t.Fatal("no internal upstream generated") + } + uc.Timeout = 300 + uc.Init() + return cfg +} + +// The loop checker runs on a one-minute ticker and probes every upstream whose +// address is local, which every generated Internal Domain resolver is. No user +// query is involved, so without the privacy treatment it publishes the +// organization's resolver addresses on its own schedule. +func TestInternalDomainsLoopCheckHidesResolverAddress(t *testing.T) { + for _, tc := range []struct { + name string + addr func(*testing.T) string + }{ + {"connection refused", deadUDPAddr}, + {"timeout", blackholeUDPAddr}, + } { + t.Run(tc.name, func(t *testing.T) { + addr := tc.addr(t) + host, _, _ := net.SplitHostPort(addr) + cfg := internalDomainsProbeConfig(t, addr) + p := newInternalDomainsProg(t, cfg) + p.loop = make(map[string]bool) + // Only the generated upstream is local, so it is the only one probed. + delete(cfg.Upstream, "0") + + mark := len(logOutput.String()) + p.checkDnsLoop() + above := logLinesAboveDebug(mark) + + for _, line := range above { + if strings.Contains(line, addr) || strings.Contains(line, host+":") { + t.Errorf("loop check disclosed the resolver address: %s", line) + } + } + if len(above) == 0 { + t.Error("the failure must stay visible above debug") + } + var classified bool + for _, line := range above { + if strings.Contains(line, `"failure":`) { + classified = true + } + } + if !classified { + t.Errorf("expected a classified failure, got: %v", above) + } + if !strings.Contains(logOutput.String()[mark:], addr) { + t.Error("the address must remain at debug level for diagnosis") + } + }) + } +} + +// Preventing an Internal Domain query from starting recovery does not keep its +// resolver out of recovery started for another reason: buildRecoveryUpstreams +// takes every non-OS upstream. +func TestInternalDomainsRecoveryCheckHidesResolverAddress(t *testing.T) { + for _, tc := range []struct { + name string + addr func(*testing.T) string + }{ + {"connection refused", deadUDPAddr}, + {"timeout", blackholeUDPAddr}, + } { + t.Run(tc.name, func(t *testing.T) { + addr := tc.addr(t) + host, _, _ := net.SplitHostPort(addr) + cfg := internalDomainsProbeConfig(t, addr) + p := newInternalDomainsProg(t, cfg) + + upstreams := p.buildRecoveryUpstreams(RecoveryReasonNetworkChange) + uc, selected := upstreams["upstream.internal_0"] + if !selected { + t.Fatalf("recovery upstreams = %v, want the Internal Domain resolver included", upstreams) + } + + mark := len(logOutput.String()) + if err := p.checkUpstreamOnce("upstream.internal_0", uc); err == nil { + t.Fatal("expected the probe to fail") + } + above := logLinesAboveDebug(mark) + + for _, line := range above { + if strings.Contains(line, addr) || strings.Contains(line, host+":") { + t.Errorf("recovery check disclosed the resolver address: %s", line) + } + } + var classified bool + for _, line := range above { + if strings.Contains(line, `"failure":`) { + classified = true + } + } + if !classified { + t.Errorf("expected a classified failure above debug, got: %v", above) + } + if !strings.Contains(logOutput.String()[mark:], addr) { + t.Error("the address must remain at debug level for diagnosis") + } + }) + } +} diff --git a/cmd/cli/internal_domains_test.go b/cmd/cli/internal_domains_test.go new file mode 100644 index 00000000..f96323ec --- /dev/null +++ b/cmd/cli/internal_domains_test.go @@ -0,0 +1,632 @@ +package cli + +import ( + "context" + "net" + "strings" + "testing" + + "github.com/miekg/dns" + "github.com/rs/zerolog" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +func internalDomainsTestConfig() *ctrld.Config { + return &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{"0": {Policy: &ctrld.ListenerPolicyConfig{Name: "My Policy"}}}, + Upstream: map[string]*ctrld.UpstreamConfig{"0": {Endpoint: "https://dns.controld.com/test", Type: ctrld.ResolverTypeDOH}}, + Network: map[string]*ctrld.NetworkConfig{"0": {Name: "Network 0", Cidrs: []string{"0.0.0.0/0"}}}, + } +} + +// ruleTargets returns the targets of the rule for source, and whether it exists. +func ruleTargets(t *testing.T, cfg *ctrld.Config, source string) ([]string, bool) { + t.Helper() + for _, rule := range cfg.Listener["0"].Policy.Rules { + if targets, ok := rule[source]; ok { + return targets, true + } + } + return nil, false +} + +func TestNormalizeInternalDomain(t *testing.T) { + long63 := strings.Repeat("a", 63) + for _, tc := range []struct{ name, in, want string }{ + // Canonicalization that cannot change which names match. + {"plain", "aws.example.com", "aws.example.com"}, + {"case and root dot", " AWS.Example.COM. ", "aws.example.com"}, + {"leading wildcard", "*.aws.example.com", "aws.example.com"}, + {"surrounding control whitespace", "\r\nCORP.EXAMPLE.\r\n", "corp.example"}, + {"single label", "intranet", "intranet"}, + {"punycode", "xn--80ak6aa92e.com", "xn--80ak6aa92e.com"}, + {"digits and hyphens", "dc-01.site2.example.com", "dc-01.site2.example.com"}, + {"max label", long63 + ".example.com", long63 + ".example.com"}, + + // Rejected: an empty or absent name. + {"empty", "", ""}, + {"whitespace only", " ", ""}, + {"root only", ".", ""}, + {"wildcard only", "*.", ""}, + + // Rejected: empty labels. A leading dot is not repaired, because the + // repaired value is a different suffix from the configured one. + {"leading dot", ".aws.example.com.", ""}, + {"double dot", "aws..example.com", ""}, + {"trailing double dot", "aws.example.com..", ""}, + + // Rejected: characters outside the ASCII hostname contract. A denylist + // would have to name every one of these; the positive rule does not. + {"space", "has space.com", ""}, + {"embedded wildcard", "we*rd.com", ""}, + {"slash", "a/b.com", ""}, + {"newline", "aws\nevil.example.com", ""}, + {"carriage return", "aws\revil.example.com", ""}, + {"nul", "aws\x00evil.example.com", ""}, + {"escape", "aws\x1bevil.example.com", ""}, + {"delete", "aws\x7fevil.example.com", ""}, + {"quote", "aws\"evil.example.com", ""}, + {"colon", "aws:53.example.com", ""}, + {"underscore", "_msdcs.example.com", ""}, + {"non-ascii", "exämple.com", ""}, + + // Rejected: hyphen placement and length. + {"leading hyphen", "-aws.example.com", ""}, + {"trailing hyphen", "aws-.example.com", ""}, + {"label too long", strings.Repeat("a", 64) + ".example.com", ""}, + {"name too long", strings.Repeat("a.", 127) + "example.com", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := normalizeInternalDomain(tc.in); got != tc.want { + t.Errorf("normalizeInternalDomain(%q) = %q, want %q", tc.in, got, tc.want) + } + }) + } +} + +// A rejected domain is counted and produces no rule, rather than being repaired +// into a suffix nobody configured. +func TestApplyInternalDomainsRejectsMalformedDomains(t *testing.T) { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws\nevil.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}, + {Domain: "ok.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.54"}}, + }) + if summary.domains != 1 || summary.skipped != 1 { + t.Fatalf("summary = %+v", summary) + } + for _, rule := range cfg.Listener["0"].Policy.Rules { + for source := range rule { + if strings.ContainsAny(source, "\n\r\x00") { + t.Errorf("control character reached a policy rule: %q", source) + } + if source != "ok.example.com" && source != "*.ok.example.com" { + t.Errorf("unexpected rule source %q", source) + } + } + } +} + +func TestInternalDomainEndpoint(t *testing.T) { + for _, tc := range []struct { + in string + want string + ok bool + }{ + {"10.0.0.10", "10.0.0.10:53", true}, + {" 10.0.0.11 ", "10.0.0.11:53", true}, + {"2001:db8::1", "[2001:db8::1]:53", true}, + {"10.0.0.10:5353", "10.0.0.10:5353", true}, + {"[2001:db8::1]:5353", "[2001:db8::1]:5353", true}, + {"dns.internal.example.com", "", false}, + {"10.0.0.256", "", false}, + {"", "", false}, + {"10.0.0.10:0", "", false}, + {"10.0.0.10:70000", "", false}, + {"10.0.0.10:dns", "", false}, + } { + got, ok := internalDomainEndpoint(tc.in) + if got != tc.want || ok != tc.ok { + t.Errorf("internalDomainEndpoint(%q) = %q, %v; want %q, %v", tc.in, got, ok, tc.want, tc.ok) + } + } +} + +func TestApplyInternalDomainsOSResolverMode(t *testing.T) { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example.com"}}) + + if summary.domains != 1 || summary.osMode != 1 || summary.explicit != 0 || summary.resolvers != 0 { + t.Fatalf("summary = %+v", summary) + } + for _, source := range []string{"corp.example.com", "*.corp.example.com"} { + targets, ok := ruleTargets(t, cfg, source) + if !ok { + t.Fatalf("no rule for %q", source) + } + if len(targets) != 0 { + t.Errorf("rule %q targets = %v, want empty (OS resolver)", source, targets) + } + } + if len(cfg.Upstream) != 1 { + t.Errorf("upstreams = %v, want only the Control D upstream", cfg.Upstream) + } +} + +func TestApplyInternalDomainsExplicitResolverMode(t *testing.T) { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10", "2001:db8::11"}}, + }) + + if summary.domains != 1 || summary.explicit != 1 || summary.osMode != 0 || summary.resolvers != 2 { + t.Fatalf("summary = %+v", summary) + } + want := []string{"upstream.internal_0", "upstream.internal_1"} + for _, source := range []string{"aws.example.com", "*.aws.example.com"} { + targets, ok := ruleTargets(t, cfg, source) + if !ok { + t.Fatalf("no rule for %q", source) + } + if strings.Join(targets, ",") != strings.Join(want, ",") { + t.Errorf("rule %q targets = %v, want %v (administrator order)", source, targets, want) + } + } + if got := cfg.Upstream["internal_0"]; got == nil || got.Endpoint != "10.0.0.10:53" || got.Type != ctrld.ResolverTypeLegacy { + t.Errorf("internal_0 = %+v", got) + } + if got := cfg.Upstream["internal_1"]; got == nil || got.Endpoint != "[2001:db8::11]:53" { + t.Errorf("internal_1 = %+v", got) + } +} + +func TestApplyInternalDomainsMalformedResolvers(t *testing.T) { + t.Run("drops only the malformed address", func(t *testing.T) { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"not-an-ip", "10.0.0.10"}}, + }) + if summary.domains != 1 || summary.resolvers != 1 || summary.skipped != 0 { + t.Fatalf("summary = %+v", summary) + } + targets, _ := ruleTargets(t, cfg, "aws.example.com") + if len(targets) != 1 || targets[0] != "upstream.internal_0" { + t.Errorf("targets = %v", targets) + } + }) + + t.Run("drops the entry when no address is usable", func(t *testing.T) { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"not-an-ip", ""}}, + }) + if summary.domains != 0 || summary.skipped != 1 { + t.Fatalf("summary = %+v", summary) + } + // Falling back to the OS resolver here would silently replace the + // administrator's explicit selection with a different one. + if _, ok := ruleTargets(t, cfg, "aws.example.com"); ok { + t.Error("entry with no usable resolver must not produce a rule") + } + if len(cfg.Upstream) != 1 { + t.Errorf("upstreams = %v, want no generated upstream", cfg.Upstream) + } + }) + + t.Run("drops an unusable domain", func(t *testing.T) { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, []controld.SplitDNS{{Domain: " ", Resolvers: []string{"10.0.0.10"}}}) + if summary.domains != 0 || summary.skipped != 1 || len(cfg.Upstream) != 1 { + t.Fatalf("summary = %+v, upstreams = %v", summary, cfg.Upstream) + } + }) + + t.Run("drops a duplicate domain", func(t *testing.T) { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10"}}, + {Domain: "AWS.Example.com.", Resolvers: []string{"10.0.0.99"}}, + }) + if summary.domains != 1 || summary.skipped != 1 { + t.Fatalf("summary = %+v", summary) + } + targets, _ := ruleTargets(t, cfg, "aws.example.com") + if len(targets) != 1 || cfg.Upstream["internal_0"].Endpoint != "10.0.0.10:53" { + t.Errorf("first entry must win: targets = %v", targets) + } + }) +} + +// Magic Folder excludes and endpoint custom configuration are written to the +// policy before Internal Domains are applied, and must keep their routing. +func TestApplyInternalDomainsPrecedence(t *testing.T) { + t.Run("an existing rule is never overwritten", func(t *testing.T) { + cfg := internalDomainsTestConfig() + cfg.Listener["0"].Policy.Rules = []ctrld.Rule{{"aws.example.com": []string{}}} + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10"}}, + }) + targets, _ := ruleTargets(t, cfg, "aws.example.com") + if len(targets) != 0 { + t.Errorf("exclude rule was overwritten: targets = %v", targets) + } + // The wildcard was still free, so the domain is only partly preempted. + if wildcard, ok := ruleTargets(t, cfg, "*.aws.example.com"); !ok || len(wildcard) != 1 { + t.Errorf("wildcard rule = %v, %v", wildcard, ok) + } + if summary.domains != 1 { + t.Errorf("summary = %+v", summary) + } + }) + + t.Run("a fully covered domain is reported as preempted", func(t *testing.T) { + cfg := internalDomainsTestConfig() + cfg.Listener["0"].Policy.Rules = []ctrld.Rule{ + {"aws.example.com": []string{"upstream.0"}}, + {"*.aws.example.com": []string{"upstream.0"}}, + } + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10"}}, + }) + if summary.domains != 0 || summary.preempted != 1 { + t.Fatalf("summary = %+v", summary) + } + if len(cfg.Listener["0"].Policy.Rules) != 2 { + t.Errorf("rules = %v, want no addition", cfg.Listener["0"].Policy.Rules) + } + }) +} + +// A configured suffix must route itself and its subdomains, and nothing else. +func TestApplyInternalDomainsSuffixMatching(t *testing.T) { + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10"}}, + {Domain: "corp.example.com"}, + }) + p := &prog{cfg: cfg} + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + + for _, tc := range []struct { + domain string + want []string + }{ + {"aws.example.com", []string{"upstream.internal_0"}}, + {"host.aws.example.com", []string{"upstream.internal_0"}}, + {"a.b.c.aws.example.com", []string{"upstream.internal_0"}}, + {"corp.example.com", []string{}}, + {"host.corp.example.com", []string{}}, + {"example.com", []string{"upstream.0"}}, + {"notaws.example.com", []string{"upstream.0"}}, + {"aws.example.com.evil.test", []string{"upstream.0"}}, + } { + ufr := p.upstreamFor(context.Background(), "0", cfg.Listener["0"], addr, "", tc.domain) + if strings.Join(ufr.upstreams, ",") != strings.Join(tc.want, ",") { + t.Errorf("upstreamFor(%q) = %v, want %v", tc.domain, ufr.upstreams, tc.want) + } + } +} + +// A refresh rebuilds the generated config, so a removal must leave no rule and +// no upstream behind. +func TestApplyInternalDomainsAuthoritativeReplacement(t *testing.T) { + first := internalDomainsTestConfig() + applyInternalDomains(first, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10"}}, + {Domain: "old.example.com", Resolvers: []string{"10.0.0.20"}}, + }) + if len(first.Upstream) != 3 { + t.Fatalf("upstreams = %v", first.Upstream) + } + + // The refresh path regenerates cfg from the API response rather than + // mutating the previous one. + second := internalDomainsTestConfig() + summary := applyInternalDomains(second, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.99"}}, + }) + if summary.domains != 1 || summary.resolvers != 1 { + t.Fatalf("summary = %+v", summary) + } + if _, ok := ruleTargets(t, second, "old.example.com"); ok { + t.Error("removed domain left a stale rule") + } + if len(second.Upstream) != 2 { + t.Errorf("upstreams = %v, want the Control D upstream plus one internal", second.Upstream) + } + if second.Upstream["internal_0"].Endpoint != "10.0.0.99:53" { + t.Errorf("resolver change not applied: %+v", second.Upstream["internal_0"]) + } +} + +func TestApplyInternalDomainsEmpty(t *testing.T) { + for _, entries := range [][]controld.SplitDNS{nil, {}} { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, entries) + if !summary.empty() { + t.Errorf("summary = %+v, want empty", summary) + } + if len(cfg.Listener["0"].Policy.Rules) != 0 || len(cfg.Upstream) != 1 { + t.Errorf("absent split_dns changed the config: rules=%v upstreams=%v", + cfg.Listener["0"].Policy.Rules, cfg.Upstream) + } + } +} + +func TestInternalDomainsEqual(t *testing.T) { + base := []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10", "10.0.0.11"}}, + {Domain: "corp.example.com"}, + } + for _, tc := range []struct { + name string + next []controld.SplitDNS + want bool + }{ + {"unchanged", base, true}, + {"reordered domains", []controld.SplitDNS{ + {Domain: "corp.example.com"}, + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10", "10.0.0.11"}}, + }, true}, + {"equivalent spelling", []controld.SplitDNS{ + {Domain: "AWS.Example.com.", Resolvers: []string{" 10.0.0.10 ", "10.0.0.11:53"}}, + {Domain: "*.corp.example.com"}, + }, true}, + {"resolver order changed", []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.11", "10.0.0.10"}}, + {Domain: "corp.example.com"}, + }, false}, + {"resolver changed", []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10", "10.0.0.99"}}, + {Domain: "corp.example.com"}, + }, false}, + {"domain removed", base[:1], false}, + {"domain added", append(append([]controld.SplitDNS{}, base...), controld.SplitDNS{Domain: "new.example.com"}), false}, + {"mode changed to explicit", []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10", "10.0.0.11"}}, + {Domain: "corp.example.com", Resolvers: []string{"10.0.0.30"}}, + }, false}, + {"emptied", nil, false}, + } { + if got := internalDomainsEqual(base, tc.next); got != tc.want { + t.Errorf("%s: internalDomainsEqual = %v, want %v", tc.name, got, tc.want) + } + } + if !internalDomainsEqual(nil, []controld.SplitDNS{}) { + t.Error("nil and empty lists must compare equal") + } +} + +func TestInternalDomainExplicitUpstreams(t *testing.T) { + for _, tc := range []struct { + in []string + want bool + }{ + {[]string{"upstream.internal_0"}, true}, + {[]string{"upstream.internal_0", "upstream.internal_1"}, true}, + {[]string{"upstream.internal_0", "upstream.0"}, false}, + {[]string{"upstream.0"}, false}, + {[]string{upstreamOS}, false}, + {nil, false}, + {[]string{}, false}, + } { + if got := internalDomainExplicitUpstreams(tc.in); got != tc.want { + t.Errorf("internalDomainExplicitUpstreams(%v) = %v, want %v", tc.in, got, tc.want) + } + } +} + +// Info and warning logs must not carry organization domain names or resolver +// addresses; those stay at debug level. +func TestInternalDomainsSummaryLogsCountsOnly(t *testing.T) { + var buf strings.Builder + logger := zerolog.New(&buf) + logInternalDomainsSummary(logger, internalDomainsSummary{ + domains: 2, osMode: 1, explicit: 1, resolvers: 3, skipped: 1, + }) + out := buf.String() + for _, want := range []string{`"domains":2`, `"os_resolver":1`, `"explicit_resolver":1`, `"resolvers":3`, `"skipped":1`} { + if !strings.Contains(out, want) { + t.Errorf("log %q missing %q", out, want) + } + } + if !strings.Contains(out, `"level":"warn"`) { + t.Errorf("a summary with skipped entries must warn: %q", out) + } + + buf.Reset() + logInternalDomainsSummary(logger, internalDomainsSummary{domains: 1, osMode: 1}) + if !strings.Contains(buf.String(), `"level":"info"`) { + t.Errorf("a clean summary must be info: %q", buf.String()) + } +} + +// An explicit-resolver Internal Domain must not be handed to VPN DNS split +// routing: the administrator named those resolvers, and VPN suffixes are +// auto-detected. When the named resolvers are unreachable the query fails +// instead of reaching the OS resolver. +func TestInternalDomainsBeatVPNSplitRoutingAndDoNotLeak(t *testing.T) { + cfg := internalDomainsTestConfig() + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true) + applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"127.0.0.1:1"}}, + }) + + p := &prog{cfg: cfg} + p.um = newUpstreamMonitor(p.cfg) + p.lanLoopGuard = newLoopGuard() + p.ptrLoopGuard = newLoopGuard() + p.vpnDNS = &vpnDNSManager{routes: map[string][]string{"example.com": {"127.0.0.1:1"}}} + + prevIntercept := dnsIntercept + dnsIntercept = true + t.Cleanup(func() { dnsIntercept = prevIntercept }) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.aws.example.com") + if !internalDomainExplicitUpstreams(ufr.upstreams) { + t.Fatalf("upstreams = %v, want only internal resolvers", ufr.upstreams) + } + + // logOutput accumulates for the whole test binary; only this call's tail + // describes what proxy() did here. + before := len(logOutput.String()) + // A trailing dot: without it the name is not a valid wire question, and the + // failure would prove nothing about whether a resolver was contacted. + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.aws.example.com.", dns.TypeA), ufr: ufr}) + out := logOutput.String()[before:] + + if strings.Contains(out, "VPN DNS route matched") { + t.Errorf("VPN split routing overrode an explicit Internal Domain: %s", out) + } + if strings.Contains(out, "attempting query to OS resolver as a retry catch all") { + t.Errorf("unreachable internal resolvers leaked to the OS resolver: %s", out) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeServerFailure { + t.Errorf("answer = %v, want SERVFAIL", res) + } +} + +// A domain that produces no rule must produce no upstream either, or the +// generated config would carry an internal_* upstream nothing refers to. +func TestApplyInternalDomainsLeaveNoOrphanUpstreams(t *testing.T) { + cfg := internalDomainsTestConfig() + cfg.Listener["0"].Policy.Rules = []ctrld.Rule{ + {"aws.example.com": []string{"upstream.0"}}, + {"*.aws.example.com": []string{"upstream.0"}}, + } + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "aws.example.com", Resolvers: []string{"10.0.0.10", "10.0.0.11"}}, + {Domain: "ok.example.com", Resolvers: []string{"10.0.0.20"}}, + }) + if summary.preempted != 1 || summary.domains != 1 || summary.resolvers != 1 { + t.Fatalf("summary = %+v", summary) + } + for key, uc := range cfg.Upstream { + if !strings.HasPrefix(key, internalDomainUpstreamPrefix) { + continue + } + if uc.Endpoint != "10.0.0.20:53" { + t.Errorf("orphan upstream %s = %+v", key, uc) + } + } + if len(cfg.Upstream) != 2 { + t.Errorf("upstreams = %v, want the Control D upstream plus one internal", cfg.Upstream) + } +} + +// Mode is what the administrator selected, so it decides the routing. The list +// of resolvers is only read when the mode calls for it. +func TestApplyInternalDomainsModeIsAuthoritative(t *testing.T) { + for _, tc := range []struct { + name string + entry controld.SplitDNS + wantTargets []string + wantRule bool + wantUpstreams int + }{ + { + // The administrator switched back to the OS resolver; addresses the + // previous selection left behind must not resurrect themselves. + name: "os mode ignores leftover resolvers", + entry: controld.SplitDNS{Domain: "a.example.com", Mode: controld.SplitDNSModeOS, Resolvers: []string{"10.0.0.53"}}, + wantTargets: []string{}, + wantRule: true, + wantUpstreams: 1, + }, + { + // Explicit with nothing to be explicit about is not OS resolution. + name: "resolvers mode with an empty list is dropped", + entry: controld.SplitDNS{Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers}, + wantRule: false, + wantUpstreams: 1, + }, + { + name: "resolvers mode with only malformed addresses is dropped", + entry: controld.SplitDNS{Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"nope"}}, + wantRule: false, + wantUpstreams: 1, + }, + { + name: "resolvers mode routes explicitly", + entry: controld.SplitDNS{Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}, + wantTargets: []string{"upstream.internal_0"}, + wantRule: true, + wantUpstreams: 2, + }, + { + name: "mode is matched case and space insensitively", + entry: controld.SplitDNS{Domain: "a.example.com", Mode: " OS ", Resolvers: []string{"10.0.0.53"}}, + wantTargets: []string{}, + wantRule: true, + wantUpstreams: 1, + }, + { + // A mode ctrld does not know is not guessed at. + name: "unknown mode is dropped", + entry: controld.SplitDNS{Domain: "a.example.com", Mode: "forward-someday", Resolvers: []string{"10.0.0.53"}}, + wantRule: false, + wantUpstreams: 1, + }, + { + // A deployment that predates the field: the list is the only signal. + name: "absent mode infers explicit", + entry: controld.SplitDNS{Domain: "a.example.com", Resolvers: []string{"10.0.0.53"}}, + wantTargets: []string{"upstream.internal_0"}, + wantRule: true, + wantUpstreams: 2, + }, + { + name: "absent mode infers os", + entry: controld.SplitDNS{Domain: "a.example.com"}, + wantTargets: []string{}, + wantRule: true, + wantUpstreams: 1, + }, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{tc.entry}) + targets, ok := ruleTargets(t, cfg, "a.example.com") + if ok != tc.wantRule { + t.Fatalf("rule present = %v, want %v", ok, tc.wantRule) + } + if ok && strings.Join(targets, ",") != strings.Join(tc.wantTargets, ",") { + t.Errorf("targets = %v, want %v", targets, tc.wantTargets) + } + if len(cfg.Upstream) != tc.wantUpstreams { + t.Errorf("upstreams = %v, want %d", cfg.Upstream, tc.wantUpstreams) + } + }) + } +} + +// A mode change is a routing change even when the resolver list is untouched, +// so it has to reach the endpoint on the next refresh. +func TestInternalDomainsEqualTracksMode(t *testing.T) { + explicit := []controld.SplitDNS{{Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}} + os := []controld.SplitDNS{{Domain: "a.example.com", Mode: controld.SplitDNSModeOS, Resolvers: []string{"10.0.0.53"}}} + if internalDomainsEqual(explicit, os) { + t.Error("switching to the OS resolver must be detected as a change") + } + if internalDomainsEqual(os, explicit) { + t.Error("switching to explicit resolvers must be detected as a change") + } + // An unusable entry routes nothing, so it must not look like a change. + unusable := []controld.SplitDNS{ + {Domain: "a.example.com", Mode: controld.SplitDNSModeOS, Resolvers: []string{"10.0.0.53"}}, + {Domain: "b.example.com", Mode: "forward-someday"}, + } + if !internalDomainsEqual(os, unusable) { + t.Error("an entry ctrld cannot use must not read as a configuration change") + } +} diff --git a/cmd/cli/loop.go b/cmd/cli/loop.go index 3504bc34..e088001b 100644 --- a/cmd/cli/loop.go +++ b/cmd/cli/loop.go @@ -86,6 +86,10 @@ func (p *prog) detectLoop(msg *dns.Msg) { func (p *prog) checkDnsLoop() { mainLog.Load().Debug().Msg("start checking DNS loop") upstream := make(map[string]*ctrld.UpstreamConfig) + // Generated Internal Domain resolvers are local addresses, so they pass + // canBeLocalUpstream and are probed every minute. Record which ones they + // are while the config key is still in hand. + generated := make(map[string]bool) p.loopMu.Lock() for n, uc := range p.cfg.Upstream { if p.um.isDown("upstream." + n) { @@ -99,6 +103,7 @@ func (p *prog) checkDnsLoop() { uid := uc.UID() p.loop[uid] = false upstream[uid] = uc + generated[uid] = isGeneratedInternalDomainUpstream(n, uc) } p.loopMu.Unlock() @@ -111,11 +116,11 @@ func (p *prog) checkDnsLoop() { } resolver, err := ctrld.NewResolver(uc) if err != nil { - mainLog.Load().Warn().Err(err).Msgf("could not perform loop check for upstream: %q, endpoint: %q", uc.Name, uc.Endpoint) + logUpstreamProbeFailure(generated[uid], uc, err, mainLog.Load().Warn, "could not perform loop check") continue } if _, err := resolver.Resolve(context.Background(), msg); err != nil { - mainLog.Load().Warn().Err(err).Msgf("could not send DNS loop check query for upstream: %q, endpoint: %q", uc.Name, uc.Endpoint) + logUpstreamProbeFailure(generated[uid], uc, err, mainLog.Load().Warn, "could not send DNS loop check query") } } mainLog.Load().Debug().Msg("end checking DNS loop") diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index a8382210..73f48199 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -552,17 +552,27 @@ func (p *prog) apiConfigReload() { p.mu.Unlock() noCustomConfig := resolverConfig.Ctrld.CustomConfig == "" noExcludeListChanged := true + // Internal Domains are regenerated from the API response, so an add, a + // resolver change, a domain change and a removal all reach the endpoint + // through the same reload as an exclude-list change. + noInternalDomainsChanged := true if rc != nil { slices.Sort(rc.Exclude) slices.Sort(resolverConfig.Exclude) noExcludeListChanged = slices.Equal(rc.Exclude, resolverConfig.Exclude) + noInternalDomainsChanged = internalDomainsEqual(rc.SplitDNS, resolverConfig.SplitDNS) } - if noCustomConfig && noExcludeListChanged { + if noCustomConfig && noExcludeListChanged && noInternalDomainsChanged { return } - if noCustomConfig && !noExcludeListChanged { - logger.Debug().Msg("exclude list changes detected, reloading...") + if noCustomConfig && (!noExcludeListChanged || !noInternalDomainsChanged) { + if !noExcludeListChanged { + logger.Debug().Msg("exclude list changes detected, reloading...") + } + if !noInternalDomainsChanged { + logger.Info().Msg("internal domain changes detected, reloading...") + } p.apiReloadCh <- nil return } @@ -613,11 +623,21 @@ func (p *prog) setupUpstream(cfg *ctrld.Config) { mainLog.Load().Debug().Msgf("initialized DNS Stamps with endpoint: %s, type: %s", uc.Endpoint, uc.Type) } isControlDUpstream = isControlDUpstream || uc.IsControlD() + // uc.Init copies an IP endpoint straight into BootstrapIP, so for a + // generated Internal Domain upstream these lines would publish the + // organization's private resolver address at info level. Detail for + // those upstreams stays at debug; the summary reports counts instead. + bootstrapEvent := func() *zerolog.Event { + if strings.HasPrefix(n, internalDomainUpstreamPrefix) { + return mainLog.Load().Debug() + } + return mainLog.Load().Info() + } if uc.BootstrapIP == "" { uc.SetupBootstrapIP() - mainLog.Load().Info().Msgf("bootstrap IPs for upstream.%s: %q", n, uc.BootstrapIPs()) + bootstrapEvent().Msgf("bootstrap IPs for upstream.%s: %q", n, uc.BootstrapIPs()) } else { - mainLog.Load().Info().Str("bootstrap_ip", uc.BootstrapIP).Msgf("using bootstrap IP for upstream.%s", n) + bootstrapEvent().Str("bootstrap_ip", uc.BootstrapIP).Msgf("using bootstrap IP for upstream.%s", n) } uc.SetCertPool(rootCertPool) go uc.Ping() @@ -630,9 +650,22 @@ func (p *prog) setupUpstream(cfg *ctrld.Config) { } } // Self-uninstallation is ok If there is only 1 ControlD upstream, and no remote config. - if len(cfg.Upstream) == 1 && isControlDUpstream { - p.canSelfUninstall.Store(true) + // + // Generated Internal Domain resolvers do not count: they come from the + // managed configuration itself, so counting them would read an ordinary + // managed install as a custom multi-upstream one and cost the endpoint the + // REFUSED-triggered deletion check. The value is recomputed rather than + // only raised, so removing the last Internal Domain - or gaining a real + // second upstream on reload - is reflected too. The uninstall itself stays + // gated on the API confirming the device is gone. + managedUpstreams := 0 + for n := range cfg.Upstream { + if isGeneratedInternalDomainUpstream(n, cfg.Upstream[n]) { + continue + } + managedUpstreams++ } + p.canSelfUninstall.Store(managedUpstreams == 1 && isControlDUpstream) p.localUpstreams = localUpstreams p.ptrNameservers = ptrNameservers } diff --git a/docs/internal-domains.md b/docs/internal-domains.md new file mode 100644 index 00000000..44caad62 --- /dev/null +++ b/docs/internal-domains.md @@ -0,0 +1,170 @@ +# Organization Internal Domains + +Internal Domains let an organization administrator route a domain suffix, and +every subdomain under it, to a chosen resolver. The selection is made in the +Control D dashboard and reaches the endpoint through the managed resolver +configuration, so no hand-edited `ctrld.toml` is involved and changes apply on +the next configuration refresh. + +## API contract + +The managed resolver configuration carries the list in `resolver.split_dns`: + +```json +{ + "resolver": { + "split_dns": [ + { "domain": "corp.example.com", "mode": "resolvers", "resolvers": ["10.0.0.53", "10.0.0.54"] }, + { "domain": "office.example.com", "mode": "os", "resolvers": [] } + ] + } +} +``` + +`mode` is what the administrator selected, and it decides the routing: + +| `mode` | Behavior | +|---|---| +| `os` | The query goes to `upstream.os`, the endpoint's current system resolver, and follows DHCP and VPN resolver changes. Any addresses still present in `resolvers` are **not** read, so switching a domain back to the OS resolver cannot be undone by leftovers from a previous selection. | +| `resolvers` | The query goes only to the addresses in `resolvers`, tried in the order given. An empty or entirely unusable list does not become OS resolution: the entry is dropped instead. | +| absent | A deployment that predates the field. The mode is inferred from `resolvers`, which is what the field encodes. | +| anything else | Dropped. Routing a private domain by guess is the disclosure this feature exists to prevent. | + +A dropped entry leaves its domain on its normal path and is reported in the +`skipped` count. + +Addresses may be IPv4 or IPv6, bare or with a port; a bare address uses port 53. +An address that is not a valid IP is dropped from the list. + +A domain is lowercased and stripped of surrounding whitespace, a leading `*.` +and the root dot, then validated as an ASCII hostname suffix: 1-63 character +labels, letters, digits and hyphens only, no leading or trailing hyphen, 253 +characters overall. Single-label names and punycode are supported. This is the +same rule the API validates against, so anything the dashboard accepted passes. +A value that fails is rejected and counted, never repaired — deleting the +offending characters would produce a different suffix from the configured one, +and routing a private domain to the wrong place is worse than not routing it. + +## Generated configuration + +Each accepted domain contributes two policy rules, `` and `*.`, +which is how ctrld matches a suffix and all of its subdomains. Explicit mode +also generates one upstream per resolver address, named `internal_`: + +```toml +[upstream.internal_0] + type = "legacy" + endpoint = "10.0.0.10:53" + +[listener.0.policy] + rules = [ + { "aws.example.com" = ["upstream.internal_0", "upstream.internal_1"] }, + { "*.aws.example.com" = ["upstream.internal_0", "upstream.internal_1"] }, + { "corp.internal" = [] }, + { "*.corp.internal" = [] }, + ] +``` + +An empty target list is the same representation Magic Folder excludes use, and +`proxy()` resolves it through `upstream.os`. + +## Precedence + +The generated configuration is built in a fixed order, and a rule that already +exists for a source is never overwritten. From highest precedence to lowest: + +1. **Endpoint custom configuration.** A valid `ctrld.custom_config` replaces the + generated configuration outright. Internal Domains are not merged into it, + the same as Magic Folder excludes — if an organization uses a custom config, + its rules are the whole policy. +2. **Magic Folder excludes** (`resolver.exclude`). These are written first, so an + excluded domain keeps its exclusion even if it also appears as an Internal + Domain. Only the exact source collides: an exclude for `aws.example.com` does + not stop the Internal Domain from claiming `*.aws.example.com`. +3. **Internal Domains** (`resolver.split_dns`), most specific suffix first. + The API permits overlapping suffixes and returns the list sorted bytewise by + domain, which puts a parent ahead of its children. Policy rules are matched + in order and the first hit wins, so appending in API order would let + `*.example.com` swallow every query a configured `z.example.com` was meant to + answer. Entries are therefore ordered by label count before generation, and + the ordering is stable, so equally specific entries keep API order and the + first of two identical domains still wins. +4. **Active Directory auto-detection** (Windows). `addExtraSplitDnsRule` runs + after the generated configuration and skips any domain that already has a + rule, so an Internal Domain for the AD domain takes precedence over the + auto-detected one. + +**VPN DNS auto-detection** is evaluated inside `proxy()` in DNS-intercept mode, +after policy matching and independently of it, so it needs its own rule: + +- An **explicit-resolver** Internal Domain skips VPN routing. The administrator + named the resolvers for that suffix; VPN suffixes are discovered from the OS, + and letting a discovered route override a named one is the leak this feature + exists to prevent. +- An **OS-resolver** Internal Domain does not skip it, and behaves exactly like + a Magic Folder exclude. It asks for the endpoint's default resolution, which + under intercept mode includes the VPN's own resolver. + +Every domain that is not an Internal Domain keeps its current VPN routing. + +## Unreachable explicit resolvers + +When every resolver for an explicit-mode domain fails, the query returns +SERVFAIL. It is deliberately not retried against the OS resolver, and it does +not trigger the endpoint recovery flow: + +- Sending a private name to a resolver the administrator did not select is the + leak this feature exists to prevent. +- The recovery flow exists for the loss of general DNS. One unavailable internal + server is not that, and letting it start recovery would reset endpoint DNS + settings for every other domain. + +OS-resolver mode has no such restriction: it is the system resolver, so it +follows DHCP and VPN changes and shares their failure handling. + +## Self-uninstall eligibility + +ctrld treats a plain managed install — one Control D upstream and no endpoint +custom configuration — as eligible for the REFUSED-triggered deletion check, +which is the fast path that notices a device has been deleted server-side. +Generated `internal_*` upstreams do not count toward that, because they come +from the managed configuration itself rather than from an operator choosing a +second upstream. Eligibility is recomputed on every setup and reload, so adding +or removing Internal Domains, or gaining a genuinely custom second upstream, +is reflected instead of latching on the first value seen. + +Discounting is by upstream shape, not by key name alone: an endpoint custom +configuration replaces the generated config outright and could name an upstream +`internal_0`, which must not buy eligibility. The uninstall itself remains +gated on the API confirming the device is gone. + +## Refresh + +`apiConfigReload` compares the stored `split_dns` against each fetch. Domains +are compared in canonical form and independently of API ordering, so an +unchanged list does not reload; resolver order is significant, because it is the +administrator's failover order. Any add, removal, domain change or resolver +change triggers a reload, which regenerates the configuration from the API +response. Because the generated configuration is rebuilt rather than patched, a +removed domain leaves no rule and no `internal_*` upstream behind. + +## Logging + +Generated `internal_*` upstreams carry the organization's private resolver +addresses, so anything naming one stays at debug, and the level above it carries +only a classification (`timeout`, `refused`, `unreachable`, `source_unavailable`, +`error`). That covers every consumer, not just query handling: + +- the bootstrap-IP line in `setupUpstream`; +- the transport error from a failed query, which names the endpoint; +- the **loop checker**, which probes every local upstream on a one-minute + ticker — a generated resolver is always a local address, so this path runs + with no user query at all; +- the **recovery checker**, which takes every non-OS upstream. Keeping an + Internal Domain query from *starting* recovery does not keep its resolver out + of recovery started for another reason. + +Info and warning logs report counts only — domains applied, OS-resolver and +explicit-resolver totals, generated resolvers, skipped and preempted entries. +Domain names and resolver addresses are organization-private and appear only at +debug level. diff --git a/internal/controld/config.go b/internal/controld/config.go index 5efbcc44..aad670ab 100644 --- a/internal/controld/config.go +++ b/internal/controld/config.go @@ -56,9 +56,36 @@ type ResolverConfig struct { CustomLastUpdate int64 `json:"custom_last_update"` VersionTarget string `json:"version_target"` } `json:"ctrld"` - Exclude []string `json:"exclude"` - UID string `json:"uid"` - DeactivationPin *int64 `json:"deactivation_pin,omitempty"` + Exclude []string `json:"exclude"` + SplitDNS []SplitDNS `json:"split_dns"` + UID string `json:"uid"` + DeactivationPin *int64 `json:"deactivation_pin,omitempty"` +} + +// Internal Domain resolution modes. Mode is what the administrator selected; +// Resolvers is only meaningful under SplitDNSModeResolvers. +const ( + // SplitDNSModeOS resolves the domain through the endpoint's own OS/default + // resolver, so it follows DHCP and VPN resolver changes. + SplitDNSModeOS = "os" + // SplitDNSModeResolvers resolves the domain through the addresses in + // Resolvers and nothing else. + SplitDNSModeResolvers = "resolvers" +) + +// SplitDNS is one organization Internal Domain: a domain suffix, the resolution +// mode the organization administrator selected for it, and the resolver +// addresses that mode may need. +// +// Mode decides the routing. Resolvers is read only under +// SplitDNSModeResolvers, so addresses left behind by an earlier selection +// cannot resurrect themselves once the administrator switches back to the OS +// resolver. An empty Mode is a deployment that predates the field; the caller +// infers the mode from Resolvers in that case. +type SplitDNS struct { + Domain string `json:"domain"` + Mode string `json:"mode"` + Resolvers []string `json:"resolvers"` } type utilityResponse struct { diff --git a/internal/controld/split_dns_test.go b/internal/controld/split_dns_test.go new file mode 100644 index 00000000..626287a5 --- /dev/null +++ b/internal/controld/split_dns_test.go @@ -0,0 +1,154 @@ +package controld + +import ( + "encoding/json" + "strings" + "testing" +) + +// The API's utility response carries organization Internal Domains in +// resolver.split_dns. Parsing it must not change anything when the field is +// absent or empty, which is what every resolver without the feature returns. +func TestResolverConfigParsesSplitDNS(t *testing.T) { + for _, tc := range []struct { + name string + body string + want []SplitDNS + }{ + { + name: "absent", + body: `{"success":true,"body":{"resolver":{"doh":"https://dns.controld.com/abc","uid":"abc"}}}`, + }, + { + name: "null", + body: `{"success":true,"body":{"resolver":{"doh":"https://d","uid":"a","split_dns":null}}}`, + }, + { + name: "empty", + body: `{"success":true,"body":{"resolver":{"doh":"https://d","uid":"a","split_dns":[]}}}`, + want: []SplitDNS{}, + }, + { + name: "explicit resolvers", + body: `{"success":true,"body":{"resolver":{"doh":"https://d","uid":"a","split_dns":[ + {"domain":"aws.example.com","resolvers":["10.0.0.10","10.0.0.11"]}]}}}`, + want: []SplitDNS{{Domain: "aws.example.com", Resolvers: []string{"10.0.0.10", "10.0.0.11"}}}, + }, + { + name: "os resolver selection", + body: `{"success":true,"body":{"resolver":{"doh":"https://d","uid":"a","split_dns":[ + {"domain":"corp.example.com","resolvers":[]}, + {"domain":"lab.example.com"}]}}}`, + want: []SplitDNS{ + {Domain: "corp.example.com", Resolvers: []string{}}, + {Domain: "lab.example.com"}, + }, + }, + { + name: "coexists with exclude", + body: `{"success":true,"body":{"resolver":{"doh":"https://d","uid":"a", + "exclude":["corp.example.com"], + "split_dns":[{"domain":"aws.example.com","resolvers":["10.0.0.10"]}]}}}`, + want: []SplitDNS{{Domain: "aws.example.com", Resolvers: []string{"10.0.0.10"}}}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + var resp utilityResponse + if err := json.NewDecoder(strings.NewReader(tc.body)).Decode(&resp); err != nil { + t.Fatal(err) + } + got := resp.Body.Resolver.SplitDNS + if len(got) != len(tc.want) { + t.Fatalf("split_dns = %+v, want %+v", got, tc.want) + } + for i := range got { + if got[i].Domain != tc.want[i].Domain { + t.Errorf("[%d].Domain = %q, want %q", i, got[i].Domain, tc.want[i].Domain) + } + if len(got[i].Resolvers) != len(tc.want[i].Resolvers) { + t.Fatalf("[%d].Resolvers = %v, want %v", i, got[i].Resolvers, tc.want[i].Resolvers) + } + for j := range got[i].Resolvers { + if got[i].Resolvers[j] != tc.want[i].Resolvers[j] { + t.Errorf("[%d].Resolvers[%d] = %q, want %q", i, j, got[i].Resolvers[j], tc.want[i].Resolvers[j]) + } + } + } + // The rest of the model must be unaffected. + if resp.Body.Resolver.DOH == "" || resp.Body.Resolver.UID == "" { + t.Errorf("resolver = %+v", resp.Body.Resolver) + } + }) + } +} + +// An unknown field in an entry must not fail the whole response: the API can +// add one before ctrld knows about it. +func TestResolverConfigToleratesUnknownSplitDNSFields(t *testing.T) { + body := `{"success":true,"body":{"resolver":{"doh":"https://d","uid":"a","split_dns":[ + {"domain":"aws.example.com","resolvers":["10.0.0.10"],"future_field":{"a":1}}]}}}` + var resp utilityResponse + if err := json.NewDecoder(strings.NewReader(body)).Decode(&resp); err != nil { + t.Fatal(err) + } + if len(resp.Body.Resolver.SplitDNS) != 1 || resp.Body.Resolver.SplitDNS[0].Domain != "aws.example.com" { + t.Fatalf("split_dns = %+v", resp.Body.Resolver.SplitDNS) + } +} + +// The shape the API actually returns, mode field included. +func TestResolverConfigParsesSplitDNSMode(t *testing.T) { + const body = `{ + "success": true, + "body": { + "resolver": { + "split_dns": [ + { + "domain": "corp.example.com", + "mode": "resolvers", + "resolvers": ["10.0.0.53", "10.0.0.54"] + }, + { + "domain": "office.example.com", + "mode": "os", + "resolvers": [] + } + ] + } + } +}` + var resp utilityResponse + if err := json.NewDecoder(strings.NewReader(body)).Decode(&resp); err != nil { + t.Fatal(err) + } + got := resp.Body.Resolver.SplitDNS + if len(got) != 2 { + t.Fatalf("split_dns = %+v", got) + } + if got[0].Domain != "corp.example.com" || got[0].Mode != SplitDNSModeResolvers { + t.Errorf("[0] = %+v", got[0]) + } + if len(got[0].Resolvers) != 2 || got[0].Resolvers[0] != "10.0.0.53" || got[0].Resolvers[1] != "10.0.0.54" { + t.Errorf("[0].Resolvers = %v", got[0].Resolvers) + } + if got[1].Domain != "office.example.com" || got[1].Mode != SplitDNSModeOS { + t.Errorf("[1] = %+v", got[1]) + } + if len(got[1].Resolvers) != 0 { + t.Errorf("[1].Resolvers = %v, want empty", got[1].Resolvers) + } +} + +// A response without the field must still decode, so a deployment that predates +// mode keeps working. +func TestResolverConfigToleratesAbsentSplitDNSMode(t *testing.T) { + const body = `{"success":true,"body":{"resolver":{"split_dns":[ + {"domain":"corp.example.com","resolvers":["10.0.0.53"]}]}}}` + var resp utilityResponse + if err := json.NewDecoder(strings.NewReader(body)).Decode(&resp); err != nil { + t.Fatal(err) + } + if got := resp.Body.Resolver.SplitDNS; len(got) != 1 || got[0].Mode != "" { + t.Fatalf("split_dns = %+v", got) + } +} From 799f14e63975f8ea782772e920c5d5d593990351 Mon Sep 17 00:00:00 2001 From: Dev Scribe Date: Wed, 16 Sep 2026 08:41:28 +0000 Subject: [PATCH 07/29] macOS: repair network recovery and improve diagnostics --- cmd/cli/dns_intercept_darwin.go | 250 ++++++---- cmd/cli/dns_intercept_darwin_test.go | 7 +- cmd/cli/dns_proxy.go | 468 +++++++++++------- cmd/cli/main.go | 14 +- cmd/cli/network_monitor_test.go | 52 ++ cmd/cli/network_source_error_test.go | 32 ++ cmd/cli/network_transition.go | 200 ++++++++ .../network_transition_concurrency_test.go | 87 ++++ cmd/cli/network_transition_reordered_test.go | 94 ++++ cmd/cli/network_transition_review_test.go | 128 +++++ cmd/cli/network_transition_test.go | 308 ++++++++++++ cmd/cli/pf_probe.go | 233 +++++++++ cmd/cli/pf_probe_darwin_test.go | 213 ++++++++ cmd/cli/pf_probe_test.go | 292 +++++++++++ cmd/cli/prog.go | 8 + docs/dns-intercept-mode.md | 8 +- docs/network-recovery-diagnostics.md | 88 ++++ docs/pf-dns-intercept.md | 11 +- 18 files changed, 2200 insertions(+), 293 deletions(-) create mode 100644 cmd/cli/network_monitor_test.go create mode 100644 cmd/cli/network_source_error_test.go create mode 100644 cmd/cli/network_transition.go create mode 100644 cmd/cli/network_transition_concurrency_test.go create mode 100644 cmd/cli/network_transition_reordered_test.go create mode 100644 cmd/cli/network_transition_review_test.go create mode 100644 cmd/cli/network_transition_test.go create mode 100644 cmd/cli/pf_probe.go create mode 100644 cmd/cli/pf_probe_darwin_test.go create mode 100644 cmd/cli/pf_probe_test.go create mode 100644 docs/network-recovery-diagnostics.md diff --git a/cmd/cli/dns_intercept_darwin.go b/cmd/cli/dns_intercept_darwin.go index eb9eafc9..ab724921 100644 --- a/cmd/cli/dns_intercept_darwin.go +++ b/cmd/cli/dns_intercept_darwin.go @@ -7,6 +7,7 @@ import ( "crypto/sha256" "fmt" "net" + "net/netip" "os" "os/exec" "path/filepath" @@ -77,7 +78,8 @@ const ( // The full probe domain is "_pf-probe-.". // These queries are sent by a subprocess WITHOUT the _ctrld group GID, // so pf should intercept them and redirect to ctrld. If ctrld receives - // the query, pf interception is working. If not (timeout), rdr is broken. + // the query, interception is working. A sent query without receipt permits + // repair. An unsent or unavailable probe is indeterminate. // No trailing dot — canonicalName() in the DNS handler strips trailing dots. pfProbeDomain = "pf-probe.ctrld.test" @@ -1477,10 +1479,19 @@ func (p *prog) finishPFStabilization(stableRequired time.Duration) { } // probePFInterceptFn and forceReloadPFInterceptFn are the functional verification seams -// shared by both probers. +// shared by all four probe callers. var ( probePFInterceptFn = (*prog).probePFIntercept forceReloadPFInterceptFn = (*prog).forceReloadPFMainRuleset + pfProbeNameservers = ctrld.OsResolverNameservers + newPFProbeCommand = func(ctx context.Context, host, packet string) *exec.Cmd { + cmd := exec.CommandContext(ctx, os.Args[0], "pf-probe-send", host, packet) + // wheel, not _ctrld: the probe must traverse interception. + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: 0, Gid: 0}, + } + return cmd + } ) // pfFunctionalProbeOwnerWait bounds how long the post-stabilization verifier waits for @@ -1551,17 +1562,29 @@ func (p *prog) verifyInterceptAfterStabilization() { return } - if probePFInterceptFn(p) { + result := probePFInterceptFn(p) + if result.result == pfProbeIndeterminate { + mainLog.Load().Debug().Msg("DNS intercept: post-stabilization probe indeterminate; repair deferred") + return + } + if result.result == pfProbeIntercepted { mainLog.Load().Debug().Msg("DNS intercept: post-stabilization probe passed — interception is translating") return } mainLog.Load().Warn().Msg("DNS intercept: post-stabilization rules are intact but the probe FAILED — forcing one reload") - if !forceReloadPFInterceptFn(p) { + if !p.repairPFIntercept(result, "post_stabilization") { mainLog.Load().Error().Msg("DNS intercept: post-stabilization forced reload did not run — leaving recovery to the watchdog") return } - if probePFInterceptFn(p) { + cause := result + result = probePFInterceptFn(p) + logPFRepair(cause, "post_stabilization", result.result.String(), true, result) + if result.result == pfProbeIndeterminate { + mainLog.Load().Warn().Msg("DNS intercept: post-stabilization reload completed; verification indeterminate") + return + } + if result.result == pfProbeIntercepted { mainLog.Load().Info().Msg("DNS intercept: interception restored by the post-stabilization reload") return } @@ -1600,8 +1623,8 @@ var pfWakeProbeOwnerWait = 250 * time.Millisecond // drive the post-stabilization verifier; without that reconnect nothing would have probed. // // The OS resolver list is refreshed before every probe. probePFIntercept aims at the -// first OS nameserver and reports success when it has no target, so a list still holding -// a pre-sleep VPN resolver turns this into a meaningless pass or a false failure. +// first usable IPv4 LAN target, or the original usable first public target. +// No target is indeterminate. Refreshing avoids probing a stale VPN resolver. func (p *prog) verifyInterceptAfterWake(gap time.Duration) { if p.dnsInterceptState == nil { return @@ -1653,7 +1676,7 @@ func (p *prog) verifyInterceptAfterWake(gap time.Duration) { mainLog.Load().Warn().Msg("DNS intercept: post-wake verification never probed - every attempt was deferred or skipped - the watchdog will retry") return } - mainLog.Load().Error().Msgf("DNS intercept: interception still not translating after resume (%d probe attempts, %d forced reloads) - the watchdog will retry", + mainLog.Load().Error().Msgf("DNS intercept: interception not verified after resume (%d probe attempts, %d forced reloads) - the watchdog will retry", probed, repairs) } @@ -1680,7 +1703,12 @@ func (p *prog) waitBeforeWakeProbe(delay time.Duration) bool { // The caller holds functional-probe ownership for the call. func (p *prog) runWakeProbeAttempt(attempt, repairs int) (restored, repaired bool) { initializeOsResolver(true) - if probePFInterceptFn(p) { + result := probePFInterceptFn(p) + if result.result == pfProbeIndeterminate { + mainLog.Load().Debug().Msg("DNS intercept: post-wake probe indeterminate; repair deferred") + return false, false + } + if result.result == pfProbeIntercepted { if attempt > 0 || repairs > 0 { mainLog.Load().Info().Msgf("DNS intercept: post-wake probe %d/%d passed - interception is translating", attempt+1, len(pfWakeProbeDelays)) } else { @@ -1692,16 +1720,19 @@ func (p *prog) runWakeProbeAttempt(attempt, repairs int) (restored, repaired boo if repairs >= pfWakeMaxRepairs { mainLog.Load().Warn().Msgf("DNS intercept: post-wake probe %d/%d FAILED - repair budget of %d forced reloads is spent, not reloading again", attempt+1, len(pfWakeProbeDelays), pfWakeMaxRepairs) + logPFRepair(result, "post_wake", "budget_exhausted", false, pfProbeObservation{}) return false, false } mainLog.Load().Warn().Msgf("DNS intercept: post-wake probe %d/%d FAILED - rules survived the suspend but pf is not translating, forcing a reload", attempt+1, len(pfWakeProbeDelays)) - if !forceReloadPFInterceptFn(p) { + if !p.repairPFIntercept(result, "post_wake") { mainLog.Load().Error().Msg("DNS intercept: post-wake forced reload did not run") return false, false } - if probePFInterceptFn(p) { + confirmation := probePFInterceptFn(p) + logPFRepair(result, "post_wake", confirmation.result.String(), true, confirmation) + if confirmation.result == pfProbeIntercepted { mainLog.Load().Info().Msg("DNS intercept: interception restored by the post-wake reload") return true, true } @@ -2011,18 +2042,7 @@ func (p *prog) pfWatchdog() { ensureInterceptDNSTargetFn(p, []string{}) result := p.ensurePFAnchorActive() - if result == pfAnchorCheckIntact { - // Only an authoritative intact result may trigger the functional probe. - // Skipped/backoff/failed checks must not be treated as healthy text state. - if !p.pfMonitorRunning.Load() && !p.probePFIntercept() { - mainLog.Load().Warn().Msg("DNS intercept watchdog: rules intact but probe FAILED — forcing full reload") - if p.forceReloadPFMainRuleset() { - result = pfAnchorCheckRestored - } else { - result = pfAnchorCheckFailed - } - } - } + result = p.checkPFWatchdogProbe(result) if result == pfAnchorCheckIntact { // Check if backoff should be reset only after an authoritative healthy check. @@ -2070,6 +2090,27 @@ func (p *prog) pfWatchdog() { } } +// A successful forced reload following a probe miss is not evidence that an +// anchor was missing. Keep that repair out of the missing-anchor counter. +func (p *prog) checkPFWatchdogProbe(result pfAnchorCheckResult) pfAnchorCheckResult { + if result != pfAnchorCheckIntact || p.pfMonitorRunning.Load() { + return result + } + probe := probePFInterceptFn(p) + switch probe.result { + case pfProbeIntercepted: + return result + case pfProbeNotIntercepted: + mainLog.Load().Warn().Msg("DNS intercept watchdog: sent probe was not intercepted; forcing full reload") + if !p.repairPFIntercept(probe, "watchdog") { + mainLog.Load().Warn().Msg("DNS intercept watchdog: probe repair did not complete; will retry") + } + case pfProbeIndeterminate: + mainLog.Load().Debug().Msg("DNS intercept watchdog: probe indeterminate; repair deferred") + } + return pfAnchorCheckDeferred +} + // exemptVPNDNSServers rebuilds the pf anchor rules to exclude VPN DNS interfaces // and server IPs from interception. VPN DNS handlers using Network Extensions // (e.g., Tailscale MagicDNS) need DNS traffic to flow without any pf interference. @@ -2144,67 +2185,91 @@ func (p *prog) exemptVPNDNSServers(exemptions []vpnDNSExemption) error { return nil } -// probePFIntercept tests whether pf's rdr translation is actually working by -// sending a DNS query through the interception path from a subprocess that does -// NOT have the _ctrld group GID. If pf interception is working, the query gets -// redirected to 127.0.0.1:53 (ctrld), and the DNS handler signals us. If broken -// (rdr rules present but not evaluating), the query goes to the real DNS server -// and we time out. -// -// Returns true if interception is working, false if broken or indeterminate. -func (p *prog) probePFIntercept() bool { - if p.dnsInterceptState == nil { - return true - } +// Keep warning coalescing available throughout intercept startup and recovery. +var pfProbeLogs pfProbeLogState - nsIPs := ctrld.OsResolverNameservers() - if len(nsIPs) == 0 { - mainLog.Load().Debug().Msg("DNS intercept probe: no OS resolver nameservers available") - return true // can't probe without a target - } - host, _, _ := net.SplitHostPort(nsIPs[0]) - if host == "" || host == "127.0.0.1" || host == "::1" { - mainLog.Load().Debug().Msg("DNS intercept probe: OS resolver is localhost, skipping probe") - return true // can't probe through localhost +// probePFIntercept distinguishes a meaningful interception failure from a probe +// that could not send. Only pfProbeNotIntercepted permits destructive repair. +func (p *prog) probePFIntercept() pfProbeObservation { + observed := pfProbeObservation{ + probeID: fmt.Sprintf("_pf-probe-%x.%s", time.Now().UnixNano(), pfProbeDomain), + recoveryGeneration: p.recoveryGen.Load(), + stage: "target", code: "unavailable", } - - // Generate unique probe domain - probeID := fmt.Sprintf("_pf-probe-%x.%s", time.Now().UnixNano()&0xFFFFFFFF, pfProbeDomain) - - // Register this attempt's own domain: overlapping probes must not cancel each other. - probeCh, deregister := p.registerInterceptProbe(probeID) - defer deregister() - - // Build a minimal DNS query packet for the probe domain. - // We use exec.Command to send from a subprocess with GID=0 (wheel), - // so pf's _ctrld group exemption does NOT apply and the query gets intercepted. - dnsPacket := buildDNSQueryPacket(probeID) - - // Send via a helper subprocess that drops the _ctrld group - cmd := exec.Command(os.Args[0], "pf-probe-send", host, fmt.Sprintf("%x", dnsPacket)) - cmd.SysProcAttr = &syscall.SysProcAttr{ - Credential: &syscall.Credential{ - Uid: 0, - Gid: 0, // wheel group — NOT _ctrld, so pf intercepts it - }, + if p.dnsInterceptState == nil { + return observed + } + started := time.Now() + servers := pfProbeNameservers() + host := pfProbeTarget(servers) + family := "none" + if len(servers) > 0 { + if first, _, err := net.SplitHostPort(servers[0]); err == nil { + if addr, err := netip.ParseAddr(first); err == nil { + observed.target = net.JoinHostPort(addr.String(), "53") + family = "ipv6" + if addr.Is4() { + family = "ipv4" + } + } + } } + if host != "" { + observed.target = net.JoinHostPort(host, "53") + family = "ipv4" + probeCh, deregister := p.registerInterceptProbe(observed.probeID) + defer deregister() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + go func() { + select { + case <-p.stopCh: + cancel() + case <-ctx.Done(): + } + }() + cmd := newPFProbeCommand(ctx, host, fmt.Sprintf("%x", buildDNSQueryPacket(observed.probeID))) + result := runPFProbe(ctx, cmd, probeCh, pfProbeTimeout) + observed.result, observed.stage, observed.code = result.result, result.stage, result.code + } + // Cancellation is a shutdown diagnostic, not a change in PF health. + retain, repeats := false, uint64(0) + if observed.code != "canceled" { + retain, repeats = pfProbeLogs.change(observed) + } + event := mainLog.Load().Debug() + if retain { + event = mainLog.Load().Warn() + } + event.Str("probe_id", observed.probeID).Uint64("recovery_generation", observed.recoveryGeneration). + Str("resolver_target", observed.target).Str("resolver_family", family). + Str("stage", observed.stage).Str("error_code", observed.code). + Str("outcome", observed.result.String()).Uint64("repeated_results", repeats). + Bool("repair_eligible", observed.result == pfProbeNotIntercepted). + Dur("elapsed_ms", time.Since(started)).Msg("DNS intercept probe result") + return observed +} - if err := cmd.Start(); err != nil { - mainLog.Load().Debug().Err(err).Msg("DNS intercept probe: failed to start probe subprocess") - return true // can't probe, assume OK +// Carry the cause by value. A concurrent probe cannot replace its correlation fields. +func (p *prog) repairPFIntercept(cause pfProbeObservation, caller string) bool { + logPFRepair(cause, caller, "started", false, pfProbeObservation{}) + performed := forceReloadPFInterceptFn(p) + outcome := "not_run" + if performed { + outcome = "reload_completed" } + logPFRepair(cause, caller, outcome, performed, pfProbeObservation{}) + return performed +} - // Don't leak the subprocess - go func() { - _ = cmd.Wait() - }() - - select { - case <-probeCh: - return true - case <-time.After(pfProbeTimeout): - return false - } +func logPFRepair(cause pfProbeObservation, caller, outcome string, performed bool, confirmation pfProbeObservation) { + mainLog.Load().Warn().Str("probe_id", cause.probeID). + Uint64("recovery_generation", cause.recoveryGeneration). + Str("caller", caller).Str("cause", cause.result.String()). + Str("stage", cause.stage).Str("error_code", cause.code). + Str("resolver_target", cause.target).Str("outcome", outcome). + Bool("reload_performed", performed).Str("confirmation_probe_id", confirmation.probeID). + Msg("PF repair") } // buildDNSQueryPacket constructs a minimal DNS query packet (wire format) for the given domain. @@ -2235,6 +2300,8 @@ func buildDNSQueryPacket(domain string) []byte { return append(header, question...) } +var pfInterceptMonitorDelays = []time.Duration{0, 500 * time.Millisecond, time.Second, 2 * time.Second, 4 * time.Second} + // pfInterceptMonitor runs asynchronously after interface changes are detected. // It probes pf interception with exponential backoff and forces a full pf reload // if the probe fails. Only one instance runs at a time (singleton via atomic.Bool). @@ -2259,7 +2326,7 @@ func (p *prog) pfInterceptMonitor() { // Backoff schedule: probe quickly first, then space out. // Total monitoring window: ~0 + 0.5 + 1 + 2 + 4 = ~7.5s - delays := []time.Duration{0, 500 * time.Millisecond, time.Second, 2 * time.Second, 4 * time.Second} + delays := pfInterceptMonitorDelays for i, delay := range delays { if delay > 0 { @@ -2270,22 +2337,33 @@ func (p *prog) pfInterceptMonitor() { return } - if probePFInterceptFn(p) { + result := probePFInterceptFn(p) + if result.result == pfProbeIndeterminate { + mainLog.Load().Debug().Msg("DNS intercept monitor: probe indeterminate; repair deferred") + continue + } + if result.result == pfProbeIntercepted { mainLog.Load().Debug().Msgf("DNS intercept monitor: probe %d/%d passed", i+1, len(delays)) continue // working now — keep monitoring in case it breaks later in the window } // Probe failed — pf translation is broken. Force full reload. mainLog.Load().Warn().Msgf("DNS intercept monitor: probe %d/%d FAILED — pf translation broken, forcing full ruleset reload", i+1, len(delays)) - forceReloadPFInterceptFn(p) + if !p.repairPFIntercept(result, "monitor") { + continue + } // Verify the reload fixed it time.Sleep(200 * time.Millisecond) - if probePFInterceptFn(p) { - mainLog.Load().Info().Msg("DNS intercept monitor: probe passed after reload — interception restored") - // Continue monitoring in case the hypervisor reloads pf again - } else { - mainLog.Load().Error().Msg("DNS intercept monitor: probe still failing after reload — pf may need manual intervention") + confirmation := probePFInterceptFn(p) + logPFRepair(result, "monitor", confirmation.result.String(), true, confirmation) + switch confirmation.result { + case pfProbeIntercepted: + mainLog.Load().Info().Msg("DNS intercept monitor: probe passed after reload - interception restored") + case pfProbeNotIntercepted: + mainLog.Load().Warn().Msg("DNS intercept monitor: probe still failing after reload - pf may need manual intervention") + case pfProbeIndeterminate: + mainLog.Load().Warn().Msg("DNS intercept monitor: reload completed; verification indeterminate") } } diff --git a/cmd/cli/dns_intercept_darwin_test.go b/cmd/cli/dns_intercept_darwin_test.go index aad97a9d..6c8b24e0 100644 --- a/cmd/cli/dns_intercept_darwin_test.go +++ b/cmd/cli/dns_intercept_darwin_test.go @@ -838,13 +838,16 @@ func stubStabilizationProbe(t *testing.T, probeResults []bool, reloadOK bool) (p probePFInterceptFn, forceReloadPFInterceptFn = originalProbe, originalReload }) probeCalls, reloadCalls := 0, 0 - probePFInterceptFn = func(*prog) bool { + probePFInterceptFn = func(*prog) pfProbeObservation { result := false if probeCalls < len(probeResults) { result = probeResults[probeCalls] } probeCalls++ - return result + if result { + return pfProbeObservation{result: pfProbeIntercepted} + } + return pfProbeObservation{result: pfProbeNotIntercepted} } forceReloadPFInterceptFn = func(*prog) bool { reloadCalls++ diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index f1318d17..dc8b844c 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -1655,7 +1655,7 @@ func FlushDNSCache() error { // monitorNetworkChanges starts monitoring for network interface changes func (p *prog) monitorNetworkChanges() error { - mon, err := netmon.New(func(format string, args ...any) { + mon, err := newNetworkChangeMonitorFn(func(format string, args ...any) { // Always fetch the latest logger (and inject the prefix) mainLog.Load().Printf("netmon: "+format, args...) }) @@ -1664,188 +1664,275 @@ func (p *prog) monitorNetworkChanges() error { } mon.RegisterChangeCallback(func(delta *netmon.ChangeDelta) { - isMajorChange := mon.IsMajorChangeFrom(delta.Old, delta.New) - p.handleDNS64NetworkChange(delta, isMajorChange) - - // Get map of valid interfaces - validIfaces := validInterfacesMap() - - mainLog.Load().Debug(). - Interface("old_state", delta.Old). - Interface("new_state", delta.New). - Bool("is_major_change", isMajorChange). - Msg("Network change detected") - - changed := false - activeInterfaceExists := false - var changeIPs []netip.Prefix - // Check each valid interface for changes - for ifaceName := range validIfaces { - oldIface, oldExists := delta.Old.Interface[ifaceName] - newIface, newExists := delta.New.Interface[ifaceName] - if !newExists { - continue - } - - oldIPs := delta.Old.InterfaceIPs[ifaceName] - newIPs := delta.New.InterfaceIPs[ifaceName] - - // if a valid interface did not exist in old - // check that its up and has usable IPs - if !oldExists { - // The interface is new (was not present in the old state). - usableNewIPs := filterUsableIPs(newIPs) - if newIface.IsUp() && len(usableNewIPs) > 0 { - changed = true - changeIPs = usableNewIPs - mainLog.Load().Debug(). - Str("interface", ifaceName). - Interface("new_ips", usableNewIPs). - Msg("Interface newly appeared (was not present in old state)") - break - } - continue - } + p.handleNetworkChange(delta, mon.IsMajorChangeFrom(delta.Old, delta.New)) + }) - // Filter new IPs to only those that are usable. - usableNewIPs := filterUsableIPs(newIPs) + mon.Start() + mainLog.Load().Debug().Msg("Network monitor started") + return nil +} - // Check if interface is up and has usable IPs. - if newIface.IsUp() && len(usableNewIPs) > 0 { - activeInterfaceExists = true +// handleNetworkChange is the network monitor callback, kept separate for synthetic deltas. +func (p *prog) handleNetworkChange(delta *netmon.ChangeDelta, isMajorChange bool) { + // netmon v1.74.0 dispatches callbacks concurrently and caches only major + // snapshots. Minor events refer to that cache through Old. The synthetic + // delta.Major flag after a time jump does not change the cache. + p.networkSourceMu.Lock() + currentState := networkChangeCurrentStateFn(delta) + if !networkSnapshotCurrent(delta, isMajorChange, currentState) { + p.networkSourceMu.Unlock() + mainLog.Load().Debug().Msg("Network transition skipped: snapshot superseded") + return + } + transitionID := p.networkTransitionGen.Add(1) + sourceState := delta.New + // A late major callback and reordered minor callbacks share one cache + // epoch. For these cases, source validity comes from the OS, not ordering. + if delta.Monitor != nil && (!isMajorChange || (p.networkSourceEpoch == currentState && p.networkSourceState != currentState)) { + fresh, err := readNetworkSourceStateFn() + if err != nil || fresh == nil { + if !p.networkSourceReadFailed { + mainLog.Load().Warn().Uint64("transition_id", transitionID). + Msg("Network source snapshot unavailable; source writes deferred") } - - // Compare interface states and IPs (interfaceIPsEqual will itself filter the IPs). - if !interfaceStatesEqual(&oldIface, &newIface) || !interfaceIPsEqual(oldIPs, newIPs) { - if newIface.IsUp() && len(usableNewIPs) > 0 { - changed = true - changeIPs = usableNewIPs - mainLog.Load().Debug(). - Str("interface", ifaceName). - Interface("old_ips", oldIPs). - Interface("new_ips", usableNewIPs). - Msg("Interface state or IPs changed") - break - } + p.networkSourceReadFailed = true + sourceState = nil + } else { + snapshot := *fresh + snapshot.DefaultRouteInterface = currentState.DefaultRouteInterface + sourceState = &snapshot + if p.networkSourceReadFailed { + mainLog.Load().Warn().Uint64("transition_id", transitionID).Msg("Network source snapshot available again") } + p.networkSourceReadFailed = false } + } + p.networkSourceState, p.networkSourceEpoch = sourceState, currentState + sourceSnapshotAvailable := sourceState != nil + before4, before6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + validateDefaultLocalIPsFromDelta(sourceState, transitionID) + after4, after6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + p.networkSourceMu.Unlock() + + p.handleDNS64NetworkChange(delta, isMajorChange) + validIfaces := networkChangeValidInterfacesFn() - // if the default route changed, set changed to true - if delta.New.DefaultRouteInterface != delta.Old.DefaultRouteInterface { - changed = true - mainLog.Load().Debug().Msgf("Default route changed from %s to %s", delta.Old.DefaultRouteInterface, delta.New.DefaultRouteInterface) + changed := false + changedInterface := "" + outcome := "ignored" + defer func() { + mainLog.Load().Debug().Uint64("transition_id", transitionID). + Bool("is_major_change", isMajorChange).Bool("changed", changed). + Str("interface", changedInterface). + Str("default_route_before", delta.Old.DefaultRouteInterface). + Str("default_route_after", delta.New.DefaultRouteInterface). + Str("source_ipv4_before", before4.String()).Str("source_ipv6_before", before6.String()). + Str("source_ipv4_after", after4.String()). + Str("source_ipv6_after", after6.String()). + Bool("source_snapshot_available", sourceSnapshotAvailable).Str("outcome", outcome).Msg("Network transition") + }() + activeInterfaceExists := false + var changeIPs []netip.Prefix + // Check each valid interface for changes + for ifaceName := range validIfaces { + oldIface, oldExists := delta.Old.Interface[ifaceName] + newIface, newExists := delta.New.Interface[ifaceName] + if !newExists { + continue } - if !changed { - mainLog.Load().Debug().Msg("Ignoring interface change - no valid interfaces affected") - // check if the default IPs are still on an interface that is up - ValidateDefaultLocalIPsFromDelta(delta.New) - // Minor interface changes can still accompany pf/WFP or VPN DNS changes. - // On macOS, bound the immediate full reconciliation so link-local-only - // notification storms do not run pfctl/scutil work for every event. - // Windows keeps the existing immediate behavior. Tunnel changes always - // bypass the macOS limit, and delayed checks provide a trailing refresh. - if dnsIntercept && p.dnsInterceptState != nil { - p.handleDNSInterceptIgnoredNetworkChange(delta, time.Now()) + oldIPs := delta.Old.InterfaceIPs[ifaceName] + newIPs := delta.New.InterfaceIPs[ifaceName] + + // if a valid interface did not exist in old + // check that its up and has usable IPs + if !oldExists { + // The interface is new (was not present in the old state). + usableNewIPs := filterUsableIPs(newIPs) + if newIface.IsUp() && len(usableNewIPs) > 0 { + changed = true + changeIPs = usableNewIPs + changedInterface = ifaceName + mainLog.Load().Debug(). + Str("interface", ifaceName). + Interface("new_ips", usableNewIPs). + Msg("Interface newly appeared (was not present in old state)") + break } - return + continue } - if !activeInterfaceExists { - mainLog.Load().Debug().Msg("No active interfaces found, skipping reinitialization") - return + // Filter new IPs to only those that are usable. + usableNewIPs := filterUsableIPs(newIPs) + + // Check if interface is up and has usable IPs. + if newIface.IsUp() && len(usableNewIPs) > 0 { + activeInterfaceExists = true } - mainLog.Load().Debug().Msg("Link state changed, re-bootstrapping") - for _, uc := range p.cfg.Upstream { - uc.ReBootstrap() + // Compare interface states and IPs (interfaceIPsEqual will itself filter the IPs). + if !interfaceStatesEqual(&oldIface, &newIface) || !interfaceIPsEqual(oldIPs, newIPs) { + if newIface.IsUp() && len(usableNewIPs) > 0 { + changed = true + changeIPs = usableNewIPs + changedInterface = ifaceName + mainLog.Load().Debug(). + Str("interface", ifaceName). + Interface("old_ips", oldIPs). + Interface("new_ips", usableNewIPs). + Msg("Interface state or IPs changed") + break + } } + } - // Get IPs from default route interface in new state - selfIP := defaultRouteIP() + // if the default route changed, set changed to true + if delta.New.DefaultRouteInterface != delta.Old.DefaultRouteInterface { + changed = true + mainLog.Load().Debug().Msgf("Default route changed from %s to %s", delta.Old.DefaultRouteInterface, delta.New.DefaultRouteInterface) + } - // Ensure that selfIP is an IPv4 address. - // If defaultRouteIP mistakenly returns an IPv6 (such as a ULA), clear it - if ip := net.ParseIP(selfIP); ip != nil && ip.To4() == nil { - mainLog.Load().Debug().Msgf("defaultRouteIP returned a non-IPv4 address: %s, ignoring it", selfIP) - selfIP = "" + if !changed { + mainLog.Load().Debug().Msg("Ignoring interface change - no valid interfaces affected") + // Minor interface changes can still accompany pf/WFP or VPN DNS changes. + // On macOS, bound the immediate full reconciliation so link-local-only + // notification storms do not run pfctl/scutil work for every event. + // Windows keeps the existing immediate behavior. Tunnel changes always + // bypass the macOS limit, and delayed checks provide a trailing refresh. + if dnsIntercept && p.dnsInterceptState != nil { + networkChangeIgnoredInterceptFn(p, delta, time.Now()) } - var ipv6 string + return + } - if delta.New.DefaultRouteInterface != "" { - mainLog.Load().Debug().Msgf("default route interface: %s, IPs: %v", delta.New.DefaultRouteInterface, delta.New.InterfaceIPs[delta.New.DefaultRouteInterface]) - for _, ip := range delta.New.InterfaceIPs[delta.New.DefaultRouteInterface] { - ipAddr, _ := netip.ParsePrefix(ip.String()) - addr := ipAddr.Addr() - if selfIP == "" && addr.Is4() { - mainLog.Load().Debug().Msgf("checking IP: %s", addr.String()) - if !addr.IsLoopback() && !addr.IsLinkLocalUnicast() { - selfIP = addr.String() - } - } - if addr.Is6() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast() { - ipv6 = addr.String() - } + if !activeInterfaceExists { + outcome = "no_active_interface" + mainLog.Load().Debug().Msg("No active interfaces found, skipping reinitialization") + return + } + + // An ignored event must not cancel this accepted transition. Only a newer + // accepted transition can replace its source commit or debounced recovery. + p.networkSourceMu.Lock() + if p.networkAcceptedGen.Load() > transitionID { + p.networkSourceMu.Unlock() + outcome = "superseded" + return + } + p.networkAcceptedGen.Store(transitionID) + p.networkSourceMu.Unlock() + + mainLog.Load().Debug().Msg("Link state changed, re-bootstrapping") + for _, uc := range p.cfg.Upstream { + uc.ReBootstrap() + } + + // Get IPs from default route interface in new state + selfIP := networkChangeDefaultRouteIPFn() + + // Ensure that selfIP is an IPv4 address. + // If defaultRouteIP mistakenly returns an IPv6 (such as a ULA), clear it + if ip := net.ParseIP(selfIP); ip != nil && ip.To4() == nil { + mainLog.Load().Debug().Msgf("defaultRouteIP returned a non-IPv4 address: %s, ignoring it", selfIP) + selfIP = "" + } + // Route discovery can lag the delta; do not reintroduce an invalid source. + if ip := net.ParseIP(selfIP); ip != nil && sourceInvalidReason(delta.New, ip) != "" { + selfIP = "" + } + var ipv6 string + + if delta.New.DefaultRouteInterface != "" { + mainLog.Load().Debug().Msgf("default route interface: %s, IPs: %v", delta.New.DefaultRouteInterface, delta.New.InterfaceIPs[delta.New.DefaultRouteInterface]) + for _, ip := range delta.New.InterfaceIPs[delta.New.DefaultRouteInterface] { + if sourceInvalidReason(delta.New, net.ParseIP(ip.Addr().String())) != "" { + continue } - } else { - // If no default route interface is set yet, use the changed IPs - mainLog.Load().Debug().Msgf("no default route interface found, using changed IPs: %v", changeIPs) - for _, ip := range changeIPs { - ipAddr, _ := netip.ParsePrefix(ip.String()) - addr := ipAddr.Addr() - if selfIP == "" && addr.Is4() { - mainLog.Load().Debug().Msgf("checking IP: %s", addr.String()) - if !addr.IsLoopback() && !addr.IsLinkLocalUnicast() { - selfIP = addr.String() - } - } - if addr.Is6() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast() { - ipv6 = addr.String() + ipAddr, _ := netip.ParsePrefix(ip.String()) + addr := ipAddr.Addr() + if selfIP == "" && addr.Is4() { + mainLog.Load().Debug().Msgf("checking IP: %s", addr.String()) + if !addr.IsLoopback() && !addr.IsLinkLocalUnicast() { + selfIP = addr.String() } } - } - - // Only set the IPv4 default if selfIP is a valid IPv4 address. - if ip := net.ParseIP(selfIP); ip != nil && ip.To4() != nil { - ctrld.SetDefaultLocalIPv4(ip) - if !isMobile() && p.ciTable != nil { - p.ciTable.SetSelfIP(selfIP) + if addr.Is6() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast() { + ipv6 = addr.String() } } - if ip := net.ParseIP(ipv6); ip != nil { - ctrld.SetDefaultLocalIPv6(ip) + } else { + // If no default route interface is set yet, use the changed IPs + mainLog.Load().Debug().Msgf("no default route interface found, using changed IPs: %v", changeIPs) + for _, ip := range changeIPs { + ipAddr, _ := netip.ParsePrefix(ip.String()) + addr := ipAddr.Addr() + if selfIP == "" && addr.Is4() { + mainLog.Load().Debug().Msgf("checking IP: %s", addr.String()) + if !addr.IsLoopback() && !addr.IsLinkLocalUnicast() { + selfIP = addr.String() + } + } + if addr.Is6() && !addr.IsLoopback() && !addr.IsLinkLocalUnicast() { + ipv6 = addr.String() + } } - mainLog.Load().Debug().Msgf("Set default local IPv4: %s, IPv6: %s", selfIP, ipv6) + } - // we only trigger recovery flow for network changes on non router devices - if router.Name() == "" { - p.debounceRecovery() + // An ignored event can change source validity without replacing recovery. + // Recheck addresses against the newest state after blocking route lookup. + p.networkSourceMu.Lock() + if p.networkAcceptedGen.Load() != transitionID { + p.networkSourceMu.Unlock() + outcome = "superseded" + return + } + latestState := p.sourceCommitState(delta) + sourceSnapshotAvailable = latestState != nil + validateDefaultLocalIPsFromDelta(latestState, transitionID) + // Only keep candidate addresses that still exist on an up interface. + if ip := net.ParseIP(selfIP); ip != nil && ip.To4() != nil && latestState != nil && sourceInvalidReason(latestState, ip) == "" { + ctrld.SetDefaultLocalIPv4(ip) + if !isMobile() && p.ciTable != nil { + p.ciTable.SetSelfIP(selfIP) } + } + if ip := net.ParseIP(ipv6); ip != nil && latestState != nil && sourceInvalidReason(latestState, ip) == "" { + ctrld.SetDefaultLocalIPv6(ip) + } + after4, after6 = ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + p.networkSourceMu.Unlock() + mainLog.Load().Debug().Msgf("Set default local IPv4: %s, IPv6: %s", selfIP, ipv6) - // After network changes, verify our pf anchor is still active and - // refresh VPN DNS state. Order matters: tunnel checks first (may rebuild - // anchor), then VPN DNS refresh (updates exemptions in anchor), then - // delayed re-checks for async VPN teardown. - if dnsIntercept && p.dnsInterceptState != nil { - if !p.pfStabilizing.Load() { - p.ensurePFAnchorActive() - } - // Check tunnel interfaces unconditionally — it decides internally - // whether to enter stabilization or rebuild immediately. - p.checkTunnelInterfaceChanges() - // Refresh VPN DNS routes — runs after tunnel checks so the anchor - // rebuild includes current VPN DNS exemptions. - if p.vpnDNS != nil { - p.vpnDNS.Refresh(true) - } - // Schedule delayed re-checks to catch async VPN teardown changes. - p.scheduleDelayedRechecks() - } - }) + outcome = "accepted" + networkChangeReconcileFn(p, transitionID) +} - mon.Start() - mainLog.Load().Debug().Msg("Network monitor started") - return nil +// reconcileNetworkChange is the first DNS/PF mutation boundary after source updates. +func (p *prog) reconcileNetworkChange(transitionID uint64) { + // we only trigger recovery flow for network changes on non router devices + if router.Name() == "" { + p.debounceRecovery(transitionID) + } + + // After network changes, verify our pf anchor is still active and + // refresh VPN DNS state. Order matters: tunnel checks first (may rebuild + // anchor), then VPN DNS refresh (updates exemptions in anchor), then + // delayed re-checks for async VPN teardown. + if dnsIntercept && p.dnsInterceptState != nil { + if !p.pfStabilizing.Load() { + p.ensurePFAnchorActive() + } + // Check tunnel interfaces unconditionally — it decides internally + // whether to enter stabilization or rebuild immediately. + p.checkTunnelInterfaceChanges() + // Refresh VPN DNS routes — runs after tunnel checks so the anchor + // rebuild includes current VPN DNS exemptions. + if p.vpnDNS != nil { + p.vpnDNS.Refresh(true) + } + // Schedule delayed re-checks to catch async VPN teardown changes. + p.scheduleDelayedRechecks() + } } // handleDNSInterceptIgnoredNetworkChange runs the DNS-intercept work for a @@ -2035,7 +2122,12 @@ const recoveryDebounceWindow = 500 * time.Millisecond // recovery runs once with the final network state. All other state updates (IP, // pf anchor, VPN DNS, tunnel checks) run immediately — only the recovery flow // with its upstream probing and DHCP bypass logic is debounced. -func (p *prog) debounceRecovery() { +func (p *prog) debounceRecovery(transitionID uint64) { + p.networkSourceMu.Lock() + defer p.networkSourceMu.Unlock() + if p.networkAcceptedGen.Load() != transitionID { + return + } p.recoveryDebounceMu.Lock() defer p.recoveryDebounceMu.Unlock() @@ -2044,10 +2136,16 @@ func (p *prog) debounceRecovery() { mainLog.Load().Debug().Msg("Recovery debounce: resetting timer (rapid network change)") } p.recoveryDebounceTimer = time.AfterFunc(recoveryDebounceWindow, func() { + p.networkSourceMu.Lock() + if p.networkAcceptedGen.Load() != transitionID { + p.networkSourceMu.Unlock() + return + } p.recoveryDebounceMu.Lock() p.recoveryDebounceTimer = nil p.recoveryDebounceMu.Unlock() - p.handleRecovery(RecoveryReasonNetworkChange) + p.networkSourceMu.Unlock() + handleRecoveryForTransitionFn(p, RecoveryReasonNetworkChange, transitionID) }) mainLog.Load().Debug().Msg("Recovery debounce: scheduled (500ms window)") } @@ -2057,13 +2155,28 @@ func (p *prog) debounceRecovery() { // upstream failure recoveries, waiting for recovery to complete (using a cancellable context without timeout), // and then re-applying the DNS settings. func (p *prog) handleRecovery(reason RecoveryReason) { - mainLog.Load().Debug().Msg("Starting recovery process: removing DNS settings") + p.handleRecoveryForTransition(reason, 0) +} +func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID uint64) { + p.networkSourceMu.Lock() + if transitionID != 0 && p.networkAcceptedGen.Load() != transitionID { + p.networkSourceMu.Unlock() + mainLog.Load().Debug().Uint64("transition_id", transitionID).Msg("Recovery skipped: transition superseded") + return + } recoveryCtx, gen, interceptRecovery, ok := p.beginRecovery(reason) + p.networkSourceMu.Unlock() if !ok { - mainLog.Load().Debug().Msg("Upstream recovery already in progress; skipping duplicate trigger") + mainLog.Load().Debug().Uint64("transition_id", transitionID). + Uint64("recovery_generation", p.recoveryGen.Load()). + Msg("Upstream recovery already in progress; skipping duplicate trigger") return } + diagnostic := recoveryDiagnostic{transitionID: transitionID, generation: gen, reason: reason, started: time.Now()} + diagnostic.event(mainLog.Load().Debug()).Bool("intercept", interceptRecovery).Msg("Recovery begin") + outcome := "superseded" + defer func() { diagnostic.end(outcome) }() if reason == RecoveryReasonNetworkChange { mainLog.Load().Debug().Msg("Network change recovery now owns shared recovery state") } @@ -2125,7 +2238,7 @@ func (p *prog) handleRecovery(reason RecoveryReason) { } } else { // Traditional flow: remove DNS settings to expose DHCP nameservers - p.resetDNS(false, false) + recoveryResetDNSFn(p, false, false) // For an OS failure, reinitialize OS resolver nameservers immediately. if reason == RecoveryReasonOSFailure { @@ -2143,9 +2256,11 @@ func (p *prog) handleRecovery(reason RecoveryReason) { upstreams := p.buildRecoveryUpstreams(reason) // Wait indefinitely until one of the upstreams recovers. - recovered, err := p.waitForUpstreamRecovery(recoveryCtx, upstreams) + recovered, err := p.waitForUpstreamRecovery(recoveryCtx, upstreams, &diagnostic) if err != nil { - mainLog.Load().Error().Err(err).Msg("Recovery canceled; DNS settings remain removed") + if p.recoveryOwnsState(gen) { + outcome = "canceled" + } p.recoveryCanceledCleanup(gen) return } @@ -2200,6 +2315,7 @@ func (p *prog) handleRecovery(reason RecoveryReason) { mainLog.Load().Debug().Msgf("Recovery generation %d was superseded during completion; preserving successor state", gen) return } + outcome = "completed" if interceptRecovery { mainLog.Load().Info().Msg("DNS intercept recovery complete: disabling DHCP bypass, resuming normal flow") } @@ -2207,7 +2323,7 @@ func (p *prog) handleRecovery(reason RecoveryReason) { // waitForUpstreamRecovery checks the provided upstreams concurrently until one recovers. // It returns the name of the recovered upstream or an error if the check times out. -func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string]*ctrld.UpstreamConfig) (string, error) { +func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string]*ctrld.UpstreamConfig, diagnostic *recoveryDiagnostic) (string, error) { recoveryCtx, cancel := context.WithCancel(ctx) defer cancel() @@ -2243,6 +2359,7 @@ func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string } return } + diagnostic.failure(err) // Back off the retry cadence for an unroutable endpoint so a // host with IPv6 up but no route to the IPv6 DoH endpoint does // not re-bootstrap/re-check every checkUpstreamBackoffSleep and @@ -2330,27 +2447,8 @@ func (p *prog) buildRecoveryUpstreams(reason RecoveryReason) map[string]*ctrld.U // are still present in the new network state (provided by delta.New). // If a stored default IP is no longer active, it resets that default (sets it to nil) // so that it won't be used in subsequent custom dialer contexts. +// Deprecated: retained for callers of the existing exported cli API. +// The network monitor uses transition-aware validation instead. func ValidateDefaultLocalIPsFromDelta(newState *netmon.State) { - currentIPv4 := ctrld.GetDefaultLocalIPv4() - currentIPv6 := ctrld.GetDefaultLocalIPv6() - - // Build a map of active IP addresses from the new state. - activeIPs := make(map[string]bool) - for _, prefixes := range newState.InterfaceIPs { - for _, prefix := range prefixes { - activeIPs[prefix.Addr().String()] = true - } - } - - // Check if the default IPv4 is still active. - if currentIPv4 != nil && !activeIPs[currentIPv4.String()] { - mainLog.Load().Debug().Msgf("DefaultLocalIPv4 %s is no longer active in the new state. Resetting.", currentIPv4) - ctrld.SetDefaultLocalIPv4(nil) - } - - // Check if the default IPv6 is still active. - if currentIPv6 != nil && !activeIPs[currentIPv6.String()] { - mainLog.Load().Debug().Msgf("DefaultLocalIPv6 %s is no longer active in the new state. Resetting.", currentIPv6) - ctrld.SetDefaultLocalIPv6(nil) - } + validateDefaultLocalIPsFromDelta(newState, 0) } diff --git a/cmd/cli/main.go b/cmd/cli/main.go index 7e6dd8dc..d372e8de 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -1,7 +1,6 @@ package cli import ( - "encoding/hex" "io" "net" "os" @@ -214,18 +213,7 @@ func initCache() { // // Usage: ctrld pf-probe-send func pfProbeSend(host, hexPacket string) { - packet, err := hex.DecodeString(hexPacket) - if err != nil { - os.Exit(1) - } - conn, err := net.DialTimeout("udp", net.JoinHostPort(host, "53"), time.Second) - if err != nil { + if err := sendPFProbe(host, hexPacket, net.DialTimeout, os.Stdout); err != nil { os.Exit(1) } - defer conn.Close() - conn.SetDeadline(time.Now().Add(time.Second)) - _, _ = conn.Write(packet) - // Read response (don't care about result, just need the send to happen) - buf := make([]byte, 512) - _, _ = conn.Read(buf) } diff --git a/cmd/cli/network_monitor_test.go b/cmd/cli/network_monitor_test.go new file mode 100644 index 00000000..33dba8dd --- /dev/null +++ b/cmd/cli/network_monitor_test.go @@ -0,0 +1,52 @@ +package cli + +import ( + "net" + "testing" + + ctrld "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +type sourceTestMonitor struct { + callback netmon.ChangeFunc + started bool +} + +func (m *sourceTestMonitor) RegisterChangeCallback(fn netmon.ChangeFunc) func() { + m.callback = fn + return func() {} +} +func (*sourceTestMonitor) IsMajorChangeFrom(*netmon.State, *netmon.State) bool { return true } +func (m *sourceTestMonitor) Start() { m.started = true } + +func TestNetworkMonitorRegistersSourceCleanup(t *testing.T) { + sourceTestGlobals(t) + old := newNetworkChangeMonitorFn + t.Cleanup(func() { newNetworkChangeMonitorFn = old }) + mon := &sourceTestMonitor{} + newNetworkChangeMonitorFn = func(func(string, ...any)) (networkChangeMonitor, error) { return mon, nil } + ctrld.SetDefaultLocalIPv4(net.ParseIP("192.0.2.10")) + ctrld.SetDefaultLocalIPv6(net.ParseIP("2001:db8::10")) + calls := 0 + networkChangeReconcileFn = func(*prog, uint64) { + calls++ + if ctrld.GetDefaultLocalIPv4() != nil { + t.Fatal("departed IPv4 survived to reconciliation") + } + } + p := &prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}} + if err := p.monitorNetworkChanges(); err != nil { + t.Fatal(err) + } + if !mon.started || mon.callback == nil { + t.Fatal("monitor did not register/start") + } + mon.callback(&netmon.ChangeDelta{ + Old: sourceTestState("en0", true, "192.0.2.10/24", "2001:db8::10/64"), + New: sourceTestState("en0", true, "2001:db8::10/64"), + }) + if calls != 1 { + t.Fatalf("registered callback never reached source reconciliation: %d", calls) + } +} diff --git a/cmd/cli/network_source_error_test.go b/cmd/cli/network_source_error_test.go new file mode 100644 index 00000000..54450fc3 --- /dev/null +++ b/cmd/cli/network_source_error_test.go @@ -0,0 +1,32 @@ +package cli + +import ( + "errors" + "net" + "testing" + + ctrld "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +func TestSourceReadFailureDefersAcceptedWrites(t *testing.T) { + sourceTestGlobals(t) + dnsIntercept = false + oldCurrent, oldRead := networkChangeCurrentStateFn, readNetworkSourceStateFn + t.Cleanup(func() { networkChangeCurrentStateFn, readNetworkSourceStateFn = oldCurrent, oldRead }) + major := sourceTestState("en0", true, "192.0.2.10/24") + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return major } + readNetworkSourceStateFn = func() (*netmon.State, error) { return nil, errors.New("read failed") } + ctrld.SetDefaultLocalIPv4(net.ParseIP("192.0.2.1")) + ctrld.SetDefaultLocalIPv6(nil) + p := &prog{cfg: &ctrld.Config{}, networkSourceEpoch: major} + calls := 0 + networkChangeReconcileFn = func(*prog, uint64) { calls++ } + p.handleNetworkChange(&netmon.ChangeDelta{ + Monitor: netmon.NewStatic(), + Old: sourceTestState("en0", true, "192.0.2.1/24"), New: major, + }, true) + if calls != 1 || !ctrld.GetDefaultLocalIPv4().Equal(net.ParseIP("192.0.2.1")) { + t.Fatal("unknown source validity changed a binding or skipped accepted recovery") + } +} diff --git a/cmd/cli/network_transition.go b/cmd/cli/network_transition.go new file mode 100644 index 00000000..5638e2ce --- /dev/null +++ b/cmd/cli/network_transition.go @@ -0,0 +1,200 @@ +package cli + +import ( + "context" + "errors" + "net" + "net/netip" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/rs/zerolog" + "github.com/Control-D-Inc/ctrld" + ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" + "tailscale.com/net/netmon" +) + +type networkChangeMonitor interface { + RegisterChangeCallback(netmon.ChangeFunc) func() + IsMajorChangeFrom(*netmon.State, *netmon.State) bool + Start() +} + +// Tests replace these functions to avoid changes to host DNS and PF state. +var ( + newNetworkChangeMonitorFn = func(logf func(string, ...any)) (networkChangeMonitor, error) { + return netmon.New(logf) + } + networkChangeCurrentStateFn = func(delta *netmon.ChangeDelta) *netmon.State { + if delta.Monitor != nil { + return delta.Monitor.InterfaceState() + } + return delta.New + } + readNetworkSourceStateFn = readNetworkSourceState + networkChangeValidInterfacesFn = validInterfacesMap + networkChangeDefaultRouteIPFn = defaultRouteIP + networkChangeReconcileFn = (*prog).reconcileNetworkChange + networkChangeIgnoredInterceptFn = (*prog).handleDNSInterceptIgnoredNetworkChange + handleRecoveryForTransitionFn = (*prog).handleRecoveryForTransition + recoveryResetDNSFn = (*prog).resetDNS +) + +// Minor callbacks have no observation sequence in netmon v1.74.0. Read only +// current interface addresses and flags rather than trusting their arrival order. +func readNetworkSourceState() (*netmon.State, error) { + interfaces, err := net.Interfaces() + if err != nil { + return nil, err + } + state := &netmon.State{Interface: map[string]netmon.Interface{}, InterfaceIPs: map[string][]netip.Prefix{}} + for _, iface := range interfaces { + addresses, err := iface.Addrs() + if err != nil { + return nil, err + } + state.Interface[iface.Name] = netmon.Interface{Interface: &iface} + for _, address := range addresses { + var ip net.IP + switch address := address.(type) { + case *net.IPNet: + ip = address.IP + case *net.IPAddr: + ip = address.IP + default: + return nil, errors.New("unsupported interface address type") + } + addr, ok := netip.AddrFromSlice(ip) + if !ok { + return nil, errors.New("invalid interface address") + } + addr = addr.Unmap() + state.InterfaceIPs[iface.Name] = append(state.InterfaceIPs[iface.Name], netip.PrefixFrom(addr, addr.BitLen())) + } + } + return state, nil +} + +// netmon v1.74.0 caches only major snapshots. Minor and time-jump events +// belong to their Old snapshot, not their distinct New snapshot. +func networkSnapshotCurrent(delta *netmon.ChangeDelta, major bool, current *netmon.State) bool { + if major || delta.Monitor == nil { + return current == delta.New + } + return current == delta.Old +} + +// Callers hold networkSourceMu. Preserve minor changes within the current +// major epoch, but use a new major snapshot even before its callback runs. +func (p *prog) sourceCommitState(delta *netmon.ChangeDelta) *netmon.State { + if delta.Monitor != nil { + if current := networkChangeCurrentStateFn(delta); current != p.networkSourceEpoch { + return current + } + } + return p.networkSourceState +} + +// sourceInvalidReason considers all up interfaces, not just hardware ports: a +// preferred source remains valid if it moved to another active interface. +func sourceInvalidReason(state *netmon.State, ip net.IP) string { + if ip == nil || state == nil { + return "" + } + reason := "address_removed" + for name, prefixes := range state.InterfaceIPs { + for _, prefix := range prefixes { + if !ip.Equal(net.ParseIP(prefix.Addr().String())) { + continue + } + iface, exists := state.Interface[name] + if exists && iface.IsUp() { + return "" + } + if exists { + reason = "interface_down" + } else if reason != "interface_down" { + reason = "interface_removed" + } + } + } + return reason +} + +func validateDefaultLocalIPsFromDelta(state *netmon.State, transitionID uint64) { + before4, before6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + reason4, reason6 := sourceInvalidReason(state, before4), sourceInvalidReason(state, before6) + if reason4 != "" { + ctrld.SetDefaultLocalIPv4(nil) + } + if reason6 != "" { + ctrld.SetDefaultLocalIPv6(nil) + } + if reason4 == "" && reason6 == "" { + return + } + // One warning per actual invalidation, not per notification or query. The + // normal warning buffer retains this summary when debug traffic rotates. + mainLog.Load().Warn().Uint64("transition_id", transitionID). + Str("default_route", state.DefaultRouteInterface). + Str("source_ipv4_before", before4.String()).Str("source_ipv6_before", before6.String()). + Str("source_ipv4_after", ctrld.GetDefaultLocalIPv4().String()). + Str("source_ipv6_after", ctrld.GetDefaultLocalIPv6().String()). + Str("ipv4_clear_reason", reason4).Str("ipv6_clear_reason", reason6). + Msg("Removed stale resolver source") +} + +type recoveryDiagnostic struct { + transitionID uint64 + generation uint64 + reason RecoveryReason + started time.Time + firstFailure sync.Once + failed atomic.Bool +} + +func recoveryReasonName(reason RecoveryReason) string { + switch reason { + case RecoveryReasonNetworkChange: + return "network_change" + case RecoveryReasonRegularFailure: + return "upstream_failure" + case RecoveryReasonOSFailure: + return "os_resolver_failure" + default: + return "unknown" + } +} + +func (d *recoveryDiagnostic) event(e *zerolog.Event) *zerolog.Event { + return e.Uint64("transition_id", d.transitionID).Uint64("recovery_generation", d.generation). + Str("recovery_reason", recoveryReasonName(d.reason)) +} + +func (d *recoveryDiagnostic) failure(err error) { + d.firstFailure.Do(func() { + d.failed.Store(true) + // Classify only; resolver errors can contain queried names or endpoints. + failure := "upstream_error" + var networkError net.Error + if errors.Is(err, syscall.EADDRNOTAVAIL) { + failure = "source_unavailable" + } else if ctrldnet.IsUnreachable(err) { + failure = "network_unreachable" + } else if errors.Is(err, context.DeadlineExceeded) || (errors.As(err, &networkError) && networkError.Timeout()) { + failure = "timeout" + } + d.event(mainLog.Load().Warn()).Str("failure", failure). + Msg("Recovery waiting for upstream; first failure") + }) +} + +func (d *recoveryDiagnostic) end(outcome string) { + e := mainLog.Load().Debug() + if outcome == "canceled" || d.failed.Load() { + e = mainLog.Load().Warn() + } + d.event(e).Str("outcome", outcome).Bool("had_failure", d.failed.Load()).Dur("duration_ms", time.Since(d.started)).Msg("Recovery end") +} diff --git a/cmd/cli/network_transition_concurrency_test.go b/cmd/cli/network_transition_concurrency_test.go new file mode 100644 index 00000000..1b9e1bf4 --- /dev/null +++ b/cmd/cli/network_transition_concurrency_test.go @@ -0,0 +1,87 @@ +package cli + +import ( + "net" + "strings" + "testing" + "time" + + ctrld "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +func TestNetworkChangeOverlappingCallbackCannotRestoreRemovedSource(t *testing.T) { + sourceTestGlobals(t) + logs := captureTransitionLogs(t) + ctrld.SetDefaultLocalIPv4(net.ParseIP("192.0.2.10")) + ctrld.SetDefaultLocalIPv6(nil) + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + networkChangeDefaultRouteIPFn = func() string { + close(entered) + <-release + return "192.0.2.10" + } + reconciliations := 0 + networkChangeReconcileFn = func(*prog, uint64) { reconciliations++ } + networkChangeIgnoredInterceptFn = func(*prog, *netmon.ChangeDelta, time.Time) {} + p := &prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}} + stateA := sourceTestState("en0", true, "192.0.2.10/24") + t.Cleanup(func() { + select { + case <-release: + default: + close(release) + } + <-done + }) + go func() { + defer close(done) + p.handleNetworkChange(&netmon.ChangeDelta{Old: sourceTestState("en0", true, "192.0.2.20/24"), New: stateA}, true) + }() + select { + case <-entered: + case <-time.After(3 * time.Second): + t.Fatal("first callback did not reach route discovery") + } + // The newer callback completes while A is paused. Its ignored/address-loss + // path clears the source and deliberately schedules no replacement recovery. + p.handleNetworkChange(&netmon.ChangeDelta{Old: stateA, New: sourceTestState("en0", true)}, true) + close(release) + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("superseded callback did not finish") + } + if ctrld.GetDefaultLocalIPv4() != nil { + t.Fatal("older callback restored the removed source") + } + if reconciliations != 1 || !strings.Contains(logs.String(), `"outcome":"accepted"`) { + t.Fatal("an ignored address-loss delta must preserve the accepted recovery") + } +} + +func TestNetworkChangeLateOldSnapshotIsRejected(t *testing.T) { + sourceTestGlobals(t) + old := networkChangeCurrentStateFn + t.Cleanup(func() { networkChangeCurrentStateFn = old }) + latest := sourceTestState("en0", true) + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return latest } + ctrld.SetDefaultLocalIPv4(nil) + ctrld.SetDefaultLocalIPv6(nil) + networkChangeReconcileFn = func(*prog, uint64) { t.Fatal("old snapshot reconciled") } + networkChangeIgnoredInterceptFn = func(*prog, *netmon.ChangeDelta, time.Time) { t.Fatal("old snapshot reconciled") } + p := &prog{cfg: &ctrld.Config{}} + p.handleNetworkChange(&netmon.ChangeDelta{Old: latest, New: sourceTestState("en0", true, "192.0.2.10/24")}, true) + if p.networkTransitionGen.Load() != 0 || ctrld.GetDefaultLocalIPv4() != nil { + t.Fatal("old snapshot became a new transition/source") + } +} + +func TestRecoveryRejectsSupersededTransition(t *testing.T) { + p := &prog{cfg: &ctrld.Config{}} + p.networkAcceptedGen.Store(2) + p.handleRecoveryForTransition(RecoveryReasonNetworkChange, 1) + if p.recoveryGen.Load() != 0 || p.recoveryRunning.Load() { + t.Fatal("superseded transition started recovery") + } +} diff --git a/cmd/cli/network_transition_reordered_test.go b/cmd/cli/network_transition_reordered_test.go new file mode 100644 index 00000000..2946af25 --- /dev/null +++ b/cmd/cli/network_transition_reordered_test.go @@ -0,0 +1,94 @@ +package cli + +import ( + "errors" + "net" + "net/netip" + "testing" + "time" + + ctrld "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +func TestLateSameEpochCallbacksCannotRestoreRemovedSource(t *testing.T) { + for _, initial := range []string{"192.0.2.1", "192.0.2.10"} { + t.Run(initial, func(t *testing.T) { + sourceTestGlobals(t) + dnsIntercept = false + oldCurrent, oldRead := networkChangeCurrentStateFn, readNetworkSourceStateFn + t.Cleanup(func() { networkChangeCurrentStateFn, readNetworkSourceStateFn = oldCurrent, oldRead }) + mon := netmon.NewStatic() + before := sourceTestState("en0", true, "192.0.2.1/24") + major := sourceTestState("ipsec0", true, "192.0.2.1/24") + major.Interface["ipsec0"] = netmon.Interface{Interface: &net.Interface{Name: "ipsec0", Flags: net.FlagUp | net.FlagPointToPoint}} + major.InterfaceIPs["ipsec0"] = []netip.Prefix{netip.MustParsePrefix("192.0.2.10/24")} + latest := sourceTestState("ipsec0", true, "192.0.2.1/24") + latest.Interface["ipsec0"] = major.Interface["ipsec0"] + // netmon ignores ipsec address changes on Darwin. Its major cache + // stays at A while B removes the source. These are OS-read seams. + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return major } + reads := 0 + readNetworkSourceStateFn = func() (*netmon.State, error) { reads++; return latest, nil } + ctrld.SetDefaultLocalIPv4(net.ParseIP(initial)) + ctrld.SetDefaultLocalIPv6(nil) + p := &prog{cfg: &ctrld.Config{}} + reconciled := 0 + networkChangeReconcileFn = func(*prog, uint64) { reconciled++ } + // B finishes before A's major callback even enters. A second, older + // minor callback also arrives late in that same cache epoch. + p.handleNetworkChange(&netmon.ChangeDelta{Monitor: mon, Old: major, New: latest}, false) + olderMinor := *major + p.handleNetworkChange(&netmon.ChangeDelta{Monitor: mon, Old: major, New: &olderMinor}, false) + if len(p.networkSourceState.InterfaceIPs["ipsec0"]) != 0 { + t.Fatal("late minor callback replaced current source validity") + } + p.handleNetworkChange(&netmon.ChangeDelta{Monitor: mon, Old: before, New: major}, true) + want := net.IP(nil) + if initial == "192.0.2.1" { + want = net.ParseIP(initial) + } + if !ctrld.GetDefaultLocalIPv4().Equal(want) { + t.Fatalf("late major restored an obsolete source: got %v want %v", ctrld.GetDefaultLocalIPv4(), want) + } + if reads != 3 || reconciled != 1 { + t.Fatalf("fresh reads=%d reconciliation=%d", reads, reconciled) + } + }) + } +} + +func TestMinorSourceReadFailureDoesNotSkipReconciliation(t *testing.T) { + sourceTestGlobals(t) + oldRead := readNetworkSourceStateFn + t.Cleanup(func() { readNetworkSourceStateFn = oldRead }) + readNetworkSourceStateFn = func() (*netmon.State, error) { return nil, errors.New("read failed") } + mon := netmon.NewStatic() + old := mon.InterfaceState() + next := *old + networkChangeValidInterfacesFn = func() map[string]struct{} { return map[string]struct{}{} } + calls := 0 + networkChangeIgnoredInterceptFn = func(*prog, *netmon.ChangeDelta, time.Time) { calls++ } + p := &prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}} + p.handleNetworkChange(&netmon.ChangeDelta{Monitor: mon, Old: old, New: &next}, false) + if calls != 1 || p.networkSourceState != nil || !p.networkSourceReadFailed { + t.Fatal("source-read failure skipped reconciliation or treated unknown data as current") + } +} + +func TestReadNetworkSourceState(t *testing.T) { + state, err := readNetworkSourceState() + if err != nil { + t.Fatal(err) + } + for name, iface := range state.Interface { + if iface.Interface == nil || iface.Name != name { + t.Fatal("interface snapshots share the wrong object") + } + for _, prefix := range state.InterfaceIPs[name] { + if !prefix.IsValid() { + t.Fatalf("invalid source prefix on %s", name) + } + } + } +} diff --git a/cmd/cli/network_transition_review_test.go b/cmd/cli/network_transition_review_test.go new file mode 100644 index 00000000..5d914883 --- /dev/null +++ b/cmd/cli/network_transition_review_test.go @@ -0,0 +1,128 @@ +package cli + +import ( + "net" + "sync/atomic" + "testing" + "time" + + ctrld "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +func TestNetmonMinorAndWakeSnapshotReconciliation(t *testing.T) { + for _, timeJump := range []bool{false, true} { + t.Run(map[bool]string{false: "minor", true: "time_jump"}[timeJump], func(t *testing.T) { + sourceTestGlobals(t) + mon := netmon.NewStatic() + // NewStatic has no monitoring resources to close. + old := mon.InterfaceState() + next := *old + // An identical snapshot is non-major; NewStatic has no OS monitor for IsMajorChangeFrom. + major := false + networkChangeValidInterfacesFn = func() map[string]struct{} { return map[string]struct{}{} } + calls := 0 + networkChangeIgnoredInterceptFn = func(*prog, *netmon.ChangeDelta, time.Time) { calls++ } + p := &prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}} + delta := &netmon.ChangeDelta{Monitor: mon, Old: old, New: &next, TimeJumped: timeJump} + //lint:ignore SA1019 This fixture reproduces netmon v1.74.0's legacy wake flag. + delta.Major = timeJump + p.handleNetworkChange(delta, major) + if calls != 1 { + t.Fatalf("real netmon non-major delta discarded: ignored-path calls=%d transition ID=%d", calls, p.networkTransitionGen.Load()) + } + }) + } +} + +func TestIgnoredDeltaPreservesAcceptedRecovery(t *testing.T) { + sourceTestGlobals(t) + dnsIntercept = false + oldRecovery := handleRecoveryForTransitionFn + t.Cleanup(func() { handleRecoveryForTransitionFn = oldRecovery }) + calls := make(chan uint64, 2) + handleRecoveryForTransitionFn = func(_ *prog, _ RecoveryReason, id uint64) { calls <- id } + networkChangeReconcileFn = (*prog).reconcileNetworkChange // Real accepted-path wiring, with recovery stubbed below. + p := &prog{cfg: &ctrld.Config{}} + a := sourceTestState("en0", true, "192.0.2.20/24") + p.handleNetworkChange(&netmon.ChangeDelta{Old: sourceTestState("en0", true, "192.0.2.10/24"), New: a}, true) + b := sourceTestState("en0", true, "192.0.2.20/24") + p.handleNetworkChange(&netmon.ChangeDelta{Old: a, New: b}, false) + select { + case <-calls: + case <-time.After(1200 * time.Millisecond): + t.Fatal("ignored delta canceled accepted recovery; no replacement was scheduled") + } +} + +func TestIgnoredDeltaPreservesValidPendingSource(t *testing.T) { + sourceTestGlobals(t) + dnsIntercept = false + ctrld.SetDefaultLocalIPv4(nil) + ctrld.SetDefaultLocalIPv6(nil) + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + networkChangeDefaultRouteIPFn = func() string { close(entered); <-release; return "192.0.2.20" } + reconciled := make(chan uint64, 2) + networkChangeReconcileFn = func(_ *prog, id uint64) { reconciled <- id } + p := &prog{cfg: &ctrld.Config{}} + a := sourceTestState("en0", true, "192.0.2.20/24") + go func() { + defer close(done) + p.handleNetworkChange(&netmon.ChangeDelta{Old: sourceTestState("en0", true, "192.0.2.10/24"), New: a}, true) + }() + <-entered + p.handleNetworkChange(&netmon.ChangeDelta{Old: a, New: sourceTestState("en0", true, "192.0.2.20/24")}, false) + close(release) + <-done + if !ctrld.GetDefaultLocalIPv4().Equal(net.ParseIP("192.0.2.20")) || len(reconciled) != 1 { + t.Fatalf("valid source discarded after ignored delta: source=%v reconciliations=%d", ctrld.GetDefaultLocalIPv4(), len(reconciled)) + } +} + +func TestNewAcceptedDeltaSupersedesPendingAcceptedSource(t *testing.T) { + sourceTestGlobals(t) + dnsIntercept = false + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var lookups atomic.Int32 + networkChangeDefaultRouteIPFn = func() string { + if lookups.Add(1) == 1 { + close(entered) + <-release + return "192.0.2.20" + } + return "" + } + reconciled := make(chan uint64, 2) + networkChangeReconcileFn = func(_ *prog, id uint64) { reconciled <- id } + p := &prog{cfg: &ctrld.Config{}} + a := sourceTestState("en0", true, "192.0.2.20/24") + go func() { + defer close(done) + p.handleNetworkChange(&netmon.ChangeDelta{Old: sourceTestState("en0", true, "192.0.2.10/24"), New: a}, true) + }() + <-entered + p.handleNetworkChange(&netmon.ChangeDelta{Old: a, New: sourceTestState("en0", true, "192.0.2.30/24")}, true) + close(release) + <-done + if !ctrld.GetDefaultLocalIPv4().Equal(net.ParseIP("192.0.2.30")) || len(reconciled) != 1 || <-reconciled != 2 { + t.Fatal("older accepted delta replaced the newer source or recovery") + } +} + +func TestSourceCommitUsesCurrentMajorEpoch(t *testing.T) { + mon := netmon.NewStatic() + latest := mon.InterfaceState() + minor := *latest + p := &prog{networkSourceState: &minor, networkSourceEpoch: latest} + delta := &netmon.ChangeDelta{Monitor: mon, Old: latest, New: &minor} + if p.sourceCommitState(delta) != &minor { + t.Fatal("discarded the observed minor update within the current epoch") + } + p.networkSourceEpoch = &minor + if p.sourceCommitState(delta) != latest { + t.Fatal("did not refresh from a newer major epoch") + } + if networkSnapshotCurrent(&netmon.ChangeDelta{Monitor: mon, Old: &minor, New: &minor}, false, latest) { + t.Fatal("accepted a minor callback from an obsolete major epoch") + } +} diff --git a/cmd/cli/network_transition_test.go b/cmd/cli/network_transition_test.go new file mode 100644 index 00000000..5db17793 --- /dev/null +++ b/cmd/cli/network_transition_test.go @@ -0,0 +1,308 @@ +package cli + +import ( + "context" + "encoding/json" + "errors" + "net" + "net/netip" + "strings" + "sync" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +func sourceTestState(route string, up bool, ips ...string) *netmon.State { + flags := net.Flags(0) + if up { + flags = net.FlagUp + } + s := &netmon.State{ + DefaultRouteInterface: route, + Interface: map[string]netmon.Interface{"en0": {Interface: &net.Interface{Name: "en0", Flags: flags}}}, + InterfaceIPs: map[string][]netip.Prefix{}, + } + for _, ip := range ips { + s.InterfaceIPs["en0"] = append(s.InterfaceIPs["en0"], netip.MustParsePrefix(ip)) + } + return s +} + +func captureTransitionLogs(t *testing.T) *syncBuffer { + t.Helper() + old := mainLog.Load() + buf := &syncBuffer{} + logger := zerolog.New(buf) + mainLog.Store(&logger) + t.Cleanup(func() { mainLog.Store(old) }) + return buf +} + +func sourceTestGlobals(t *testing.T) { + t.Helper() + v4, v6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + valid, route := networkChangeValidInterfacesFn, networkChangeDefaultRouteIPFn + reconcile, ignored := networkChangeReconcileFn, networkChangeIgnoredInterceptFn + intercept := dnsIntercept + dnsIntercept = true + networkChangeValidInterfacesFn = func() map[string]struct{} { return map[string]struct{}{"en0": {}} } + networkChangeDefaultRouteIPFn = func() string { return "" } + t.Cleanup(func() { + ctrld.SetDefaultLocalIPv4(v4) + ctrld.SetDefaultLocalIPv6(v6) + networkChangeValidInterfacesFn, networkChangeDefaultRouteIPFn = valid, route + networkChangeReconcileFn, networkChangeIgnoredInterceptFn = reconcile, ignored + dnsIntercept = intercept + }) +} + +func TestNetworkChangeSourceLifecycle(t *testing.T) { + const v4, v6 = "192.0.2.10", "2001:db8::10" + for _, tt := range []struct { + name string + newState *netmon.State + preferred string + want4, want6 string + outcome, reason4, reason6 string + }{ + {"dhcp_lease_replacement_keeps_ipv6", sourceTestState("en0", true, "192.0.2.20/24", v6+"/64"), "", "192.0.2.20", v6, "accepted", "address_removed", ""}, + {"dhcp_lease_replacement", sourceTestState("en0", true, "192.0.2.20/24"), "", "192.0.2.20", "", "accepted", "address_removed", "address_removed"}, + {"accepted_ipv4_loss", sourceTestState("en0", true, v6+"/64"), "", "", v6, "accepted", "address_removed", ""}, + {"accepted_ipv6_loss", sourceTestState("en0", true, v4+"/24"), v4, v4, "", "accepted", "", "address_removed"}, + {"stale_route_discovery", sourceTestState("en0", true, v6+"/64"), v4, "", v6, "accepted", "address_removed", ""}, + {"ignored_down_retains_addresses", sourceTestState("en0", false, v4+"/24", v6+"/64"), "", "", "", "ignored", "interface_down", "interface_down"}, + {"ignored_address_loss", sourceTestState("en0", true), "", "", "", "ignored", "address_removed", "address_removed"}, + {"skipped_no_active_interface", sourceTestState("", false, v4+"/24", v6+"/64"), "", "", "", "no_active_interface", "interface_down", "interface_down"}, + {"ignored_valid_sources", sourceTestState("en0", true, v4+"/24", v6+"/64"), "", v4, v6, "ignored", "", ""}, + {"accepted_valid_preferred_source", sourceTestState("en0", true, "192.0.2.20/24", v4+"/24", v6+"/64"), v4, v4, v6, "accepted", "", ""}, + } { + t.Run(tt.name, func(t *testing.T) { + sourceTestGlobals(t) + logs := captureTransitionLogs(t) + ctrld.SetDefaultLocalIPv4(net.ParseIP(v4)) + ctrld.SetDefaultLocalIPv6(net.ParseIP(v6)) + networkChangeDefaultRouteIPFn = func() string { return tt.preferred } + p := &prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}} + accepted, ignored := 0, 0 + assertSources := func() { + t.Helper() + if got := ctrld.GetDefaultLocalIPv4(); !got.Equal(net.ParseIP(tt.want4)) { + t.Errorf("IPv4 = %v, want %s", got, tt.want4) + } + if got := ctrld.GetDefaultLocalIPv6(); !got.Equal(net.ParseIP(tt.want6)) { + t.Errorf("IPv6 = %v, want %s", got, tt.want6) + } + } + // These seams precede all DNS/PF writes and recovery timers, on every OS. + networkChangeReconcileFn = func(_ *prog, id uint64) { + accepted++ + if id != 1 { + t.Errorf("transition ID = %d", id) + } + assertSources() + } + networkChangeIgnoredInterceptFn = func(_ *prog, _ *netmon.ChangeDelta, _ time.Time) { ignored++; assertSources() } + p.handleNetworkChange(&netmon.ChangeDelta{Old: sourceTestState("en0", true, v4+"/24", v6+"/64"), New: tt.newState}, true) + assertSources() + wantAccepted, wantIgnored := 0, 0 + if tt.outcome == "accepted" { + wantAccepted = 1 + } + if tt.outcome == "ignored" { + wantIgnored = 1 + } + if accepted != wantAccepted || ignored != wantIgnored { + t.Fatalf("mutation boundaries: accepted=%d ignored=%d; want %d/%d", accepted, ignored, wantAccepted, wantIgnored) + } + var transition, warning map[string]any + for _, line := range strings.Split(strings.TrimSpace(logs.String()), "\n") { + var event map[string]any + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatal(err) + } + switch event["message"] { + case "Network transition": + transition = event + case "Removed stale resolver source": + warning = event + } + } + if transition["outcome"] != tt.outcome || transition["transition_id"] != float64(1) { + t.Fatalf("transition: %v", transition) + } + wantWarning := tt.reason4 != "" || tt.reason6 != "" + if (warning != nil) != wantWarning { + t.Fatalf("warning = %v, want warning %v", warning, wantWarning) + } + if wantWarning && (warning["ipv4_clear_reason"] != tt.reason4 || warning["ipv6_clear_reason"] != tt.reason6 || warning["level"] != "warn") { + t.Fatalf("warning reasons/level: %v", warning) + } + }) + } +} + +func TestNetworkChangeDoesNotRestoreDownDefaultRouteSources(t *testing.T) { + sourceTestGlobals(t) + const v4, v6 = "192.0.2.10", "2001:db8::10" + ctrld.SetDefaultLocalIPv4(net.ParseIP(v4)) + ctrld.SetDefaultLocalIPv6(net.ParseIP(v6)) + networkChangeDefaultRouteIPFn = func() string { return v4 } + old := sourceTestState("en1", true, "2001:db8:1::10/64") + old.Interface["en1"] = netmon.Interface{Interface: &net.Interface{Name: "en1", Flags: net.FlagUp}} + old.InterfaceIPs["en1"] = []netip.Prefix{netip.MustParsePrefix(v4 + "/24"), netip.MustParsePrefix(v6 + "/64")} + newState := sourceTestState("en1", true, "2001:db8:1::20/64") + newState.Interface["en1"] = netmon.Interface{Interface: &net.Interface{Name: "en1"}} + newState.InterfaceIPs["en1"] = old.InterfaceIPs["en1"] + called := 0 + networkChangeReconcileFn = func(_ *prog, _ uint64) { + called++ + if ctrld.GetDefaultLocalIPv4() != nil || ctrld.GetDefaultLocalIPv6() != nil { + t.Error("down-interface sources reintroduced before recovery") + } + } + networkChangeIgnoredInterceptFn = func(_ *prog, _ *netmon.ChangeDelta, _ time.Time) { t.Fatal("accepted delta was ignored") } + (&prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}}).handleNetworkChange(&netmon.ChangeDelta{Old: old, New: newState}, true) + if called != 1 { + t.Fatalf("reconciliation called %d times", called) + } +} + +func TestNetworkChangeSourceMovedToUpInterface(t *testing.T) { + sourceTestGlobals(t) + ctrld.SetDefaultLocalIPv4(net.ParseIP("192.0.2.10")) + ctrld.SetDefaultLocalIPv6(nil) + state := sourceTestState("en0", false, "192.0.2.10/24") + state.Interface["utun0"] = netmon.Interface{Interface: &net.Interface{Name: "utun0", Flags: net.FlagUp}} + state.InterfaceIPs["utun0"] = state.InterfaceIPs["en0"] + called := 0 + networkChangeReconcileFn = func(_ *prog, _ uint64) { t.Fatal("unexpected recovery") } + networkChangeIgnoredInterceptFn = func(_ *prog, _ *netmon.ChangeDelta, _ time.Time) { called++ } + (&prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}}).handleNetworkChange(&netmon.ChangeDelta{Old: state, New: state}, false) + if called != 1 || !ctrld.GetDefaultLocalIPv4().Equal(net.ParseIP("192.0.2.10")) { + t.Fatal("valid source on another up interface was cleared") + } +} + +func TestRecoveryTransitionDiagnostics(t *testing.T) { + logs := captureTransitionLogs(t) + original, intercept := recoveryResetDNSFn, dnsIntercept + dnsIntercept = false + t.Cleanup(func() { recoveryResetDNSFn = original; dnsIntercept = intercept }) + for _, superseded := range []bool{false, true} { + p := &prog{cfg: &ctrld.Config{}} + calls := 0 + recoveryResetDNSFn = func(p *prog, start, restore bool) { + calls++ + if start || restore { + t.Error("changed reset flags") + } + if superseded { + p.beginRecovery(RecoveryReasonNetworkChange) + } else { + p.recoveryCancel() + } + } + p.networkAcceptedGen.Store(42) + p.handleRecoveryForTransition(RecoveryReasonNetworkChange, 42) + if calls != 1 { + t.Fatalf("resetDNS seam called %d times", calls) + } + if p.recoveryCancel != nil { + p.recoveryCancel() + } + } + for _, expected := range []string{`"transition_id":42`, `"recovery_generation":1`, `"message":"Recovery begin"`, `"outcome":"canceled"`, `"outcome":"superseded"`, `"message":"Recovery end"`} { + if !strings.Contains(logs.String(), expected) { + t.Errorf("missing %s in %s", expected, logs.String()) + } + } +} + +func TestRecoveryDebouncePreservesTransitionID(t *testing.T) { + original := handleRecoveryForTransitionFn + t.Cleanup(func() { handleRecoveryForTransitionFn = original }) + ids := make(chan uint64, 2) + handleRecoveryForTransitionFn = func(_ *prog, reason RecoveryReason, id uint64) { + if reason != RecoveryReasonNetworkChange { + t.Error("wrong recovery reason") + } + ids <- id + } + p := &prog{cfg: &ctrld.Config{}} + p.networkAcceptedGen.Store(11) + p.debounceRecovery(11) + p.networkAcceptedGen.Store(12) + p.debounceRecovery(12) + p.debounceRecovery(11) // a delayed older callback must not replace the new timer + select { + case id := <-ids: + if id != 12 { + t.Fatalf("recovery transition = %d, want 12", id) + } + case <-time.After(3 * time.Second): + t.Fatal("debounced recovery did not run") + } +} + +func TestTransitionWarningsSurviveDebugRotation(t *testing.T) { + sourceTestGlobals(t) + old := mainLog.Load() + debug, warnings := newLogWriterWithSize(2048), newSmallLogWriter() + // Runtime summaries occur after initialization, outside the preserved prefix. + debug.Write([]byte(logWriterInitEndMarker)) + logger := zerolog.New(zerolog.MultiLevelWriter(debug, &zerolog.FilteredLevelWriter{Writer: zerolog.LevelWriterAdapter{Writer: warnings}, Level: zerolog.WarnLevel})) + mainLog.Store(&logger) + t.Cleanup(func() { mainLog.Store(old) }) + ctrld.SetDefaultLocalIPv4(net.ParseIP("192.0.2.10")) + ctrld.SetDefaultLocalIPv6(nil) + state := sourceTestState("en0", false, "192.0.2.10/24") + for range 100 { + validateDefaultLocalIPsFromDelta(state, 7) + } + d := &recoveryDiagnostic{transitionID: 7, generation: 9, started: time.Now()} + var wg sync.WaitGroup + for range 100 { + wg.Add(1) + go func() { defer wg.Done(); d.failure(errors.New("private.customer.example secret payload")) }() + } + wg.Wait() + d.end("completed") + (&recoveryDiagnostic{transitionID: 8, generation: 10, started: time.Now()}).end("completed") + for range 100 { + logger.Debug().Msg(strings.Repeat("noise", 100)) + } + text := warnings.buf.String() + for _, msg := range []string{"Removed stale resolver source", "Recovery waiting for upstream; first failure"} { + if strings.Count(text, msg) != 1 { + t.Fatalf("expected one retained %q: %s", msg, text) + } + if strings.Contains(debug.buf.String(), msg) { + t.Fatalf("debug buffer did not rotate %q", msg) + } + } + if strings.Count(text, "Recovery end") != 1 || !strings.Contains(text, `"outcome":"completed"`) { + t.Fatal("recovery completion must survive rotation beside its retained failure") + } + if strings.Contains(text, "private.customer") || strings.Contains(text, "secret payload") { + t.Fatalf("raw payload or normal debug elevated: %s", text) + } +} + +func TestRecoveryWaitLogsFirstFailure(t *testing.T) { + logs := captureTransitionLogs(t) + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + d := &recoveryDiagnostic{transitionID: 8, generation: 2, started: time.Now()} + // Invalid resolver type fails construction before any network access. + _, err := (&prog{}).waitForUpstreamRecovery(ctx, map[string]*ctrld.UpstreamConfig{"invalid": {Type: "invalid"}}, d) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("wait result = %v", err) + } + if strings.Count(logs.String(), "Recovery waiting for upstream; first failure") != 1 { + t.Fatalf("missing first-failure summary: %s", logs.String()) + } +} diff --git a/cmd/cli/pf_probe.go b/cmd/cli/pf_probe.go new file mode 100644 index 00000000..83d93a75 --- /dev/null +++ b/cmd/cli/pf_probe.go @@ -0,0 +1,233 @@ +package cli + +import ( + "bufio" + "context" + "encoding/hex" + "errors" + "fmt" + "io" + "net" + "net/netip" + "os/exec" + "strings" + "sync" + "syscall" + "time" + + "tailscale.com/net/tsaddr" +) + +type pfProbeResult uint8 + +const ( + pfProbeIndeterminate pfProbeResult = iota + pfProbeIntercepted + pfProbeNotIntercepted +) + +func (r pfProbeResult) String() string { + switch r { + case pfProbeIntercepted: + return "intercepted" + case pfProbeNotIntercepted: + return "not_intercepted" + default: + return "indeterminate" + } +} + +type pfProbeObservation struct { + result pfProbeResult + stage string + code string + probeID string + recoveryGeneration uint64 + target string +} + +// Conditions exclude attempt IDs and generations so repeated failures stay bounded. +type pfProbeCondition struct { + result pfProbeResult + stage, code, target string +} + +type pfProbeLogState struct { + mu sync.Mutex + previous pfProbeCondition + initialized bool + repeats uint64 +} + +// change returns whether to retain a transition and the preceding repeat count. +// A restored receipt is retained too, so an old warning does not imply an outage. +func (s *pfProbeLogState) change(o pfProbeObservation) (retain bool, repeats uint64) { + s.mu.Lock() + defer s.mu.Unlock() + condition := pfProbeCondition{o.result, o.stage, o.code, o.target} + if s.initialized && condition == s.previous { + s.repeats++ + return false, s.repeats + } + retain = o.result != pfProbeIntercepted || (s.initialized && s.previous.result != pfProbeIntercepted) + repeats = s.repeats + s.previous, s.initialized, s.repeats = condition, true, 0 + return retain, repeats +} + +// OsResolverNameservers groups LAN entries before public entries. Preserve a usable +// first target, including public DNS. Otherwise scan only the LAN group. Stop at +// the public boundary, not at a particular IP: the public group can be synthetic. +func pfProbeTarget(servers []string) string { + for i, server := range servers { + host, _, err := net.SplitHostPort(server) + if err != nil { + return "" + } + ip, err := netip.ParseAddr(host) + if err != nil { + return "" + } + lan := ip.IsPrivate() || ip.IsLoopback() || ip.IsLinkLocalUnicast() || tsaddr.CGNATRange().Contains(ip) + usable := ip.Is4() && !ip.IsLoopback() && !ip.IsUnspecified() && !ip.IsMulticast() + if usable && (i == 0 || lan) { + return host + } + if !lan { + return "" + } + } + return "" +} + +// Keep the helper protocol bounded and free of resolver IDs, queries and raw OS errors. +func pfProbeErrorCode(err error) string { + switch { + case errors.Is(err, syscall.ENETUNREACH), errors.Is(err, syscall.EHOSTUNREACH): + return "unreachable" + case errors.Is(err, syscall.EADDRNOTAVAIL): + return "source_unavailable" + case errors.Is(err, syscall.EACCES), errors.Is(err, syscall.EPERM): + return "permission" + default: + var ne net.Error + if errors.As(err, &ne) && ne.Timeout() { + return "timeout" + } + return "io" + } +} + +func sendPFProbe(host, hexPacket string, dial func(string, string, time.Duration) (net.Conn, error), status io.Writer) error { + fail := func(stage string, err error) error { + _, _ = fmt.Fprintf(status, "%s:%s\n", stage, pfProbeErrorCode(err)) + return err + } + packet, err := hex.DecodeString(hexPacket) + if err != nil { + _, _ = io.WriteString(status, "decode:invalid_argument\n") + return err + } + conn, err := dial("udp4", net.JoinHostPort(host, "53"), time.Second) + if err != nil { + return fail("dial", err) + } + defer conn.Close() + if err := conn.SetDeadline(time.Now().Add(time.Second)); err != nil { + return fail("deadline", err) + } + n, err := conn.Write(packet) + if err == nil && n != len(packet) { + err = io.ErrShortWrite + } + if err != nil { + return fail("write", err) + } + if _, err := io.WriteString(status, "sent\n"); err != nil { + return err + } + // Keep the socket alive for the local handler's response. A read failure does + // not undo the successful send acknowledged above. + _, _ = conn.Read(make([]byte, 512)) + return nil +} + +func parsePFProbeStatus(status string) pfProbeObservation { + if status == "sent\n" { + return pfProbeObservation{stage: "sent"} + } + stage, code, ok := strings.Cut(strings.TrimSuffix(status, "\n"), ":") + if ok && (stage == "decode" || stage == "dial" || stage == "deadline" || stage == "write") { + switch code { + case "unreachable", "source_unavailable", "permission", "timeout", "io", "invalid_argument": + return pfProbeObservation{stage: stage, code: code} + } + } + return pfProbeObservation{stage: "helper", code: "invalid_status"} +} + +func pfProbeContextCode(ctx context.Context) string { + if errors.Is(ctx.Err(), context.DeadlineExceeded) { + return "timeout" + } + return "canceled" +} + +// Receipt by the local handler is conclusive. A missing receipt is a failed +// interception test only after the child confirms a successful packet send. +func runPFProbe(ctx context.Context, cmd *exec.Cmd, received <-chan struct{}, deliveryTimeout time.Duration) pfProbeObservation { + stdout, err := cmd.StdoutPipe() + if err != nil { + return pfProbeObservation{stage: "start", code: pfProbeErrorCode(err)} + } + if err := cmd.Start(); err != nil { + _ = stdout.Close() + return pfProbeObservation{stage: "start", code: pfProbeErrorCode(err)} + } + statusCh := make(chan pfProbeObservation, 1) + done := make(chan struct{}) + go func() { + line, _ := bufio.NewReader(io.LimitReader(stdout, 128)).ReadString('\n') + statusCh <- parsePFProbeStatus(line) + _ = cmd.Wait() + close(done) + }() + defer func() { + _ = cmd.Process.Kill() + <-done + }() + + var sent pfProbeObservation + select { + case <-received: + return pfProbeObservation{result: pfProbeIntercepted, stage: "received"} + case <-ctx.Done(): + return pfProbeObservation{stage: "helper", code: pfProbeContextCode(ctx)} + case sent = <-statusCh: + } + if sent.stage != "sent" { + select { + case <-received: + return pfProbeObservation{result: pfProbeIntercepted, stage: "received"} + default: + return sent + } + } + + timer := time.NewTimer(deliveryTimeout) + defer timer.Stop() + select { + case <-received: + return pfProbeObservation{result: pfProbeIntercepted, stage: "received"} + case <-ctx.Done(): + return pfProbeObservation{stage: "delivery", code: pfProbeContextCode(ctx)} + case <-timer.C: + // Prefer receipt when notification and deadline became ready together. + select { + case <-received: + return pfProbeObservation{result: pfProbeIntercepted, stage: "received"} + default: + return pfProbeObservation{result: pfProbeNotIntercepted, stage: "delivery", code: "timeout"} + } + } +} diff --git a/cmd/cli/pf_probe_darwin_test.go b/cmd/cli/pf_probe_darwin_test.go new file mode 100644 index 00000000..87f4d721 --- /dev/null +++ b/cmd/cli/pf_probe_darwin_test.go @@ -0,0 +1,213 @@ +//go:build darwin + +package cli + +import ( + "context" + "encoding/json" + "os" + "os/exec" + "reflect" + "strings" + "testing" + "time" +) + +func TestPFProbeDefaultCommand(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + cmd := newPFProbeCommand(ctx, "192.0.2.1", "0102") + if !reflect.DeepEqual(cmd.Args, []string{os.Args[0], "pf-probe-send", "192.0.2.1", "0102"}) { + t.Fatalf("args=%q", cmd.Args) + } + if cmd.SysProcAttr == nil || cmd.SysProcAttr.Credential == nil { + t.Fatal("missing credentials") + } + cred := cmd.SysProcAttr.Credential + if cred.Uid != 0 || cred.Gid != 0 || cred.NoSetGroups || len(cred.Groups) != 0 { + t.Fatalf("unexpected credentials: %+v", cred) + } + if cmd.Cancel == nil { + t.Fatal("command is not context-bound") + } + // Do not start this command: credential changes require native privileged QA. +} + +func TestPFProbeMonitorIndeterminateContinues(t *testing.T) { + oldProbe, oldReload, oldDelays := probePFInterceptFn, forceReloadPFInterceptFn, pfInterceptMonitorDelays + t.Cleanup(func() { + probePFInterceptFn, forceReloadPFInterceptFn, pfInterceptMonitorDelays = oldProbe, oldReload, oldDelays + }) + pfInterceptMonitorDelays = []time.Duration{0, 0, 0} + probes, reloads := 0, 0 + probePFInterceptFn = func(*prog) pfProbeObservation { + probes++ + if probes == 2 { + return pfProbeObservation{result: pfProbeIntercepted} + } + return pfProbeObservation{stage: "dial", code: "unreachable"} + } + forceReloadPFInterceptFn = func(*prog) bool { reloads++; return true } + p := &prog{dnsInterceptState: &pfState{}} + p.pfInterceptMonitor() + if probes != 3 || reloads != 0 || p.pfMonitorRunning.Load() { + t.Fatalf("probes=%d reloads=%d owner=%v", probes, reloads, p.pfMonitorRunning.Load()) + } +} + +func TestPFProbeRepairCorrelation(t *testing.T) { + for _, caller := range []string{"post_stabilization", "post_wake", "watchdog", "monitor"} { + for _, performed := range []bool{false, true} { + t.Run(caller+"/"+map[bool]string{false: "not_run", true: "completed"}[performed], func(t *testing.T) { + oldProbe, oldReload, oldDelays := probePFInterceptFn, forceReloadPFInterceptFn, pfInterceptMonitorDelays + t.Cleanup(func() { + probePFInterceptFn, forceReloadPFInterceptFn, pfInterceptMonitorDelays = oldProbe, oldReload, oldDelays + }) + stubWakeProbeSchedule(t, 1) + pfInterceptMonitorDelays = []time.Duration{0} + p := &prog{dnsInterceptState: &pfState{}, stopCh: make(chan struct{})} + probes, reloads := 0, 0 + probePFInterceptFn = func(*prog) pfProbeObservation { + probes++ + if probes == 1 { + return pfProbeObservation{result: pfProbeNotIntercepted, probeID: "cause-probe", recoveryGeneration: 7, stage: "delivery", code: "timeout", target: "192.0.2.1:53"} + } + return pfProbeObservation{result: pfProbeIndeterminate, probeID: "confirmation-probe", recoveryGeneration: 8, stage: "dial", code: "unreachable"} + } + forceReloadPFInterceptFn = func(*prog) bool { reloads++; p.recoveryGen.Store(99); return performed } + before := len(logOutput.String()) + switch caller { + case "post_stabilization": + p.verifyInterceptAfterStabilization() + case "post_wake": + p.runWakeProbeAttempt(0, 0) + case "watchdog": + p.checkPFWatchdogProbe(pfAnchorCheckIntact) + case "monitor": + p.pfInterceptMonitor() + } + expected := "not_run" + if performed { + expected = "reload_completed" + } + found, confirmation := false, false + for _, line := range strings.Split(logOutput.String()[before:], "\n") { + var event map[string]interface{} + if json.Unmarshal([]byte(line), &event) != nil || event["message"] != "PF repair" { + continue + } + if event["probe_id"] != "cause-probe" || event["recovery_generation"] != float64(7) || event["caller"] != caller { + t.Fatalf("lost cause: %v", event) + } + if event["outcome"] == expected { + found = true + } + if event["outcome"] == "indeterminate" && event["confirmation_probe_id"] == "confirmation-probe" { + confirmation = true + } + } + if reloads != 1 || !found { + t.Fatalf("reloads=%d outcome found=%v", reloads, found) + } + if performed && caller != "watchdog" && !confirmation { + t.Fatal("missing confirmation outcome") + } + }) + } + } +} + +func TestPFProbeMissingTargetLogTransitions(t *testing.T) { + old := pfProbeNameservers + t.Cleanup(func() { pfProbeNameservers = old; pfProbeLogs = pfProbeLogState{} }) + pfProbeLogs = pfProbeLogState{} + first := "[fe80::1%en0]:53" + pfProbeNameservers = func() []string { return []string{first, "76.76.2.0:53"} } + p := &prog{dnsInterceptState: &pfState{}} + before := len(logOutput.String()) + p.probePFIntercept() + p.probePFIntercept() + first = "127.0.0.1:53" + p.probePFIntercept() + var levels []string + for _, line := range strings.Split(logOutput.String()[before:], "\n") { + var event map[string]interface{} + if json.Unmarshal([]byte(line), &event) != nil || event["message"] != "DNS intercept probe result" { + continue + } + levels = append(levels, event["level"].(string)) + if len(levels) == 3 && event["repeated_results"] != float64(1) { + t.Fatal("missing suppressed count") + } + } + if !reflect.DeepEqual(levels, []string{"warn", "debug", "warn"}) { + t.Fatalf("levels=%v", levels) + } +} + +func TestPFProbeUnsentDoesNotReload(t *testing.T) { + oldServers, oldCommand, oldReload := pfProbeNameservers, newPFProbeCommand, forceReloadPFInterceptFn + t.Cleanup(func() { + pfProbeNameservers, newPFProbeCommand, forceReloadPFInterceptFn = oldServers, oldCommand, oldReload + }) + pfProbeNameservers = func() []string { return []string{"192.0.2.1:53"} } + newPFProbeCommand = func(ctx context.Context, host, packet string) *exec.Cmd { return pfProbeTestCommand(ctx, "dial") } + reloads := 0 + forceReloadPFInterceptFn = func(*prog) bool { reloads++; return true } + p := &prog{dnsInterceptState: &pfState{}, stopCh: make(chan struct{})} + before := len(logOutput.String()) + if got := p.checkPFWatchdogProbe(pfAnchorCheckIntact); got != pfAnchorCheckDeferred { + t.Fatalf("watchdog result=%v", got) + } + p.verifyInterceptAfterStabilization() + stubWakeProbeSchedule(t, 1) + if restored, repaired := p.runWakeProbeAttempt(0, 0); restored || repaired { + t.Fatalf("unsent wake probe: restored=%v repaired=%v", restored, repaired) + } + if reloads != 0 { + t.Fatalf("unsent probes caused %d reloads", reloads) + } + logs := logOutput.String()[before:] + for _, field := range []string{`"probe_id":`, `"recovery_generation":`, `"resolver_target":"192.0.2.1:53"`, `"stage":"dial"`, `"error_code":"unreachable"`, `"outcome":"indeterminate"`, `"repair_eligible":false`} { + if !strings.Contains(logs, field) { + t.Errorf("missing %s in logs", field) + } + } + if strings.Contains(logs, "interception is translating") || strings.Contains(logs, "anchor was missing") { + t.Fatal("unknown probe claimed PF health or missing rules") + } +} + +func TestPFProbeWatchdogMeaningfulFailureRepairsWithoutMissingAnchorClaim(t *testing.T) { + _, reloads := stubStabilizationProbe(t, []bool{false}, true) + p := &prog{dnsInterceptState: &pfState{}} + if got := p.checkPFWatchdogProbe(pfAnchorCheckIntact); got != pfAnchorCheckDeferred { + t.Fatalf("probe repair must not be counted as missing-anchor restoration: %v", got) + } + if *reloads != 1 { + t.Fatalf("reloads=%d", *reloads) + } +} + +func TestPFProbeWatchdogSkipsIndeterminateAnchor(t *testing.T) { + probes, reloads := stubStabilizationProbe(t, nil, true) + p := &prog{dnsInterceptState: &pfState{}} + for _, r := range []pfAnchorCheckResult{pfAnchorCheckDeferred, pfAnchorCheckSkipped, pfAnchorCheckFailed, pfAnchorCheckRestored} { + if got := p.checkPFWatchdogProbe(r); got != r { + t.Fatalf("changed outcome %v -> %v", r, got) + } + } + if *probes != 0 || *reloads != 0 { + t.Fatal("probed without intact rule evidence") + } +} + +func TestPFProbeNoIPv4TargetIsIndeterminate(t *testing.T) { + old := pfProbeNameservers + t.Cleanup(func() { pfProbeNameservers = old }) + pfProbeNameservers = func() []string { return []string{"[fe80::1%en0]:53", "76.76.2.0:53"} } + p := &prog{dnsInterceptState: &pfState{}} + if got := p.probePFIntercept(); got.result != pfProbeIndeterminate { + t.Fatalf("got %v", got) + } +} diff --git a/cmd/cli/pf_probe_test.go b/cmd/cli/pf_probe_test.go new file mode 100644 index 00000000..6afae2ad --- /dev/null +++ b/cmd/cli/pf_probe_test.go @@ -0,0 +1,292 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "io" + "net" + "os" + "os/exec" + "strings" + "sync" + "syscall" + "testing" + "time" +) + +func TestPFProbeProcess(t *testing.T) { + mode := os.Getenv("CTRLD_PF_PROBE_TEST") + if mode == "" { + return + } + switch mode { + case "main_dispatch": + os.Args = []string{os.Args[0], "pf-probe-send", "127.0.0.1", "zz"} + Main() + os.Exit(99) // Production dispatch must exit on invalid input. + case "sent": + _, _ = os.Stdout.WriteString("sent\n") + case "dial": + _, _ = os.Stdout.WriteString("dial:unreachable\n") + os.Exit(1) + case "write": + _, _ = os.Stdout.WriteString("write:permission\n") + os.Exit(1) + case "malformed": + _, _ = os.Stdout.WriteString(strings.Repeat("private-payload", 1000)) + case "hang": + time.Sleep(time.Minute) + case "silent": + os.Exit(1) + } + os.Exit(0) +} + +func pfProbeTestCommand(ctx context.Context, mode string) *exec.Cmd { + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestPFProbeProcess$") + cmd.Env = append(os.Environ(), "CTRLD_PF_PROBE_TEST="+mode) + return cmd +} + +func TestPFProbeProcessClassification(t *testing.T) { + for _, tc := range []struct { + mode string + want pfProbeResult + stage string + }{ + {"dial", pfProbeIndeterminate, "dial"}, + {"write", pfProbeIndeterminate, "write"}, + {"silent", pfProbeIndeterminate, "helper"}, + {"malformed", pfProbeIndeterminate, "helper"}, + {"sent", pfProbeNotIntercepted, "delivery"}, + } { + t.Run(tc.mode, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + got := runPFProbe(ctx, pfProbeTestCommand(ctx, tc.mode), make(chan struct{}), 10*time.Millisecond) + if got.result != tc.want || got.stage != tc.stage { + t.Fatalf("got %+v, want %v/%s", got, tc.want, tc.stage) + } + if strings.Contains(got.code, "private") { + t.Fatal("helper output escaped into diagnostic") + } + }) + } +} + +func TestPFProbeStartFailureAndCancellation(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + got := runPFProbe(ctx, exec.Command(t.TempDir()+"/missing-helper"), make(chan struct{}), time.Second) + if got.result != pfProbeIndeterminate || got.stage != "start" { + t.Fatalf("start failure: %+v", got) + } + ctx, cancel = context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + got = runPFProbe(ctx, pfProbeTestCommand(ctx, "hang"), make(chan struct{}), time.Second) + if got.result != pfProbeIndeterminate { + t.Fatalf("canceled helper must not request repair: %+v", got) + } +} + +func TestPFProbeReceiptWins(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + received := make(chan struct{}) + close(received) + got := runPFProbe(ctx, pfProbeTestCommand(ctx, "sent"), received, time.Millisecond) + if got.result != pfProbeIntercepted { + t.Fatalf("local receipt is conclusive: %+v", got) + } +} + +type pfProbeErrorConn struct { + net.Conn + writeErr error + short bool +} + +func (c pfProbeErrorConn) SetDeadline(time.Time) error { return nil } +func (c pfProbeErrorConn) Close() error { return nil } +func (c pfProbeErrorConn) Write(b []byte) (int, error) { + if c.short { + return len(b) - 1, nil + } + return 0, c.writeErr +} + +func TestPFProbeSendFailures(t *testing.T) { + for _, tc := range []struct { + name, packet, want string + dial func(string, string, time.Duration) (net.Conn, error) + }{ + {"decode", "zz", "decode:invalid_argument\n", nil}, + {"dial", "0102", "dial:unreachable\n", func(string, string, time.Duration) (net.Conn, error) { + return nil, &net.OpError{Op: "dial", Err: syscall.ENETUNREACH} + }}, + {"write", "0102", "write:permission\n", func(string, string, time.Duration) (net.Conn, error) { + return pfProbeErrorConn{writeErr: syscall.EACCES}, nil + }}, + {"short", "0102", "write:io\n", func(string, string, time.Duration) (net.Conn, error) { return pfProbeErrorConn{short: true}, nil }}, + } { + t.Run(tc.name, func(t *testing.T) { + var out bytes.Buffer + err := sendPFProbe("192.0.2.1", tc.packet, tc.dial, &out) + if err == nil || out.String() != tc.want { + t.Fatalf("status=%q err=%v", out.String(), err) + } + if strings.Contains(out.String(), "sent") { + t.Fatal("failed write claimed success") + } + }) + } +} + +func TestPFProbeSendRealUDP(t *testing.T) { + listener, err := net.ListenPacket("udp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + received := make(chan []byte, 1) + go func() { + b := make([]byte, 100) + n, addr, e := listener.ReadFrom(b) + if e != nil { + return + } + received <- append([]byte(nil), b[:n]...) + _, _ = listener.WriteTo([]byte{0}, addr) + }() + var out bytes.Buffer + err = sendPFProbe("127.0.0.1", "010203", func(network, address string, timeout time.Duration) (net.Conn, error) { + if network != "udp4" || address != "127.0.0.1:53" { + t.Errorf("unexpected target %s %s", network, address) + } + return net.DialTimeout(network, listener.LocalAddr().String(), timeout) + }, &out) + if err != nil || out.String() != "sent\n" { + t.Fatalf("status=%q error=%v", out.String(), err) + } + select { + case b := <-received: + if !bytes.Equal(b, []byte{1, 2, 3}) { + t.Fatalf("wrong packet %v", b) + } + case <-time.After(time.Second): + t.Fatal("helper reported sent without UDP receipt") + } +} + +func TestPFProbeTarget(t *testing.T) { + for _, first := range []string{"[fe80::1%en0]:53", "[2001:db8::1]:53", "127.0.0.53:53", "0.0.0.0:53", "224.0.0.1:53", "bad"} { + // Resolver construction can append this public fallback. No later entry + // may become a new probe recipient when the original first target fails. + if got := pfProbeTarget([]string{first, "76.76.2.0:53"}); got != "" { + t.Fatalf("first=%s selected new recipient=%s", first, got) + } + } + for _, first := range []string{"192.0.2.10", "76.76.2.0"} { + if got := pfProbeTarget([]string{first + ":53", "192.0.2.11:53"}); got != first { + t.Fatalf("changed original first target %s to %s", first, got) + } + } + if got := pfProbeTarget(nil); got != "" { + t.Fatalf("empty list selected %s", got) + } +} + +func TestPFProbeMainDispatch(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + out, err := pfProbeTestCommand(ctx, "main_dispatch").Output() + var exit *exec.ExitError + if !errors.As(err, &exit) || exit.ExitCode() != 1 || string(out) != "decode:invalid_argument\n" { + t.Fatalf("production Main dispatch: output=%q error=%v", out, err) + } +} + +func TestPFProbeTargetLANScan(t *testing.T) { + for _, tc := range []struct { + servers []string + want string + }{ + {[]string{"[fe80::1%en0]:53", "192.168.1.1:53", "76.76.2.0:53"}, "192.168.1.1"}, + {[]string{"127.0.0.1:53", "100.64.0.1:53", "76.76.2.0:53"}, "100.64.0.1"}, + {[]string{"[fd00::1]:53", "169.254.1.1:53", "76.76.2.0:53"}, "169.254.1.1"}, + {[]string{"[fe80::1%en0]:53", "76.76.2.0:53"}, ""}, + {[]string{"[fe80::1%en0]:53", "8.8.8.8:53", "192.168.1.1:53"}, ""}, + {[]string{"[2001:db8::1]:53", "192.168.1.1:53"}, ""}, + {[]string{"76.76.2.0:53"}, "76.76.2.0"}, + {[]string{"8.8.8.8:53", "192.168.1.1:53"}, "8.8.8.8"}, + } { + if got := pfProbeTarget(tc.servers); got != tc.want { + t.Errorf("%v: got %q want %q", tc.servers, got, tc.want) + } + } +} + +func TestPFProbeLogConcurrentRepeats(t *testing.T) { + var state pfProbeLogState + observed := pfProbeObservation{stage: "dial", code: "unreachable"} + state.change(observed) + var wg sync.WaitGroup + for range 20 { + wg.Add(1) + go func() { defer wg.Done(); state.change(observed) }() + } + wg.Wait() + observed.result, observed.stage, observed.code = pfProbeIntercepted, "received", "" + if retained, repeats := state.change(observed); !retained || repeats != 20 { + t.Fatalf("retained=%v repeats=%d", retained, repeats) + } +} + +func TestPFProbeLogConditionChanges(t *testing.T) { + var state pfProbeLogState + missing := pfProbeObservation{stage: "target", code: "unavailable", target: "[fe80::1%en0]:53"} + if retain, repeats := state.change(missing); !retain || repeats != 0 { + t.Fatal("first missing target not retained") + } + missing.probeID, missing.recoveryGeneration = "new-attempt", 2 + if retain, repeats := state.change(missing); retain || repeats != 1 { + t.Fatal("attempt IDs defeat dedupe") + } + for _, change := range []func(*pfProbeObservation){ + func(o *pfProbeObservation) { o.target = "127.0.0.1:53" }, + func(o *pfProbeObservation) { o.stage = "dial" }, + func(o *pfProbeObservation) { o.code = "unreachable" }, + func(o *pfProbeObservation) { o.result = pfProbeNotIntercepted }, + } { + change(&missing) + if retain, repeats := state.change(missing); !retain || repeats != 1 { + t.Fatalf("change not retained with prior repeats: %v %d", retain, repeats) + } + if retain, repeats := state.change(missing); retain || repeats != 1 { + t.Fatal("repeat not coalesced") + } + } + healthy := pfProbeObservation{result: pfProbeIntercepted, stage: "received", target: missing.target} + if retain, repeats := state.change(healthy); !retain || repeats != 1 { + t.Fatal("restoration not retained") + } + if retain, _ := state.change(healthy); retain { + t.Fatal("normal receipt retained") + } +} + +func TestPFProbeStatusIsBounded(t *testing.T) { + if got := parsePFProbeStatus("decode:invalid_argument\n"); got.stage != "decode" || got.code != "invalid_argument" || got.result != pfProbeIndeterminate { + t.Fatalf("invalid argument classification: %+v", got) + } + for _, s := range []string{"dial:secret-token\n", "secret:io\n", strings.Repeat("private", 1000)} { + got := parsePFProbeStatus(s) + if got.result != pfProbeIndeterminate || got.stage != "helper" || got.code != "invalid_status" { + t.Fatalf("untrusted status leaked: %+v", got) + } + } + if pfProbeErrorCode(errors.New("secret")) != "io" || pfProbeErrorCode(io.ErrShortWrite) != "io" { + t.Fatal("unbounded diagnostic") + } +} diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 73f48199..60913dc2 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -161,6 +161,14 @@ type prog struct { // only touches shared recovery state if it is still the newest (#597). recoveryGen atomic.Uint64 + // All deltas get an ID. Only accepted deltas replace recovery ownership. + networkTransitionGen atomic.Uint64 + networkAcceptedGen atomic.Uint64 + networkSourceMu sync.Mutex + networkSourceState *netmon.State + networkSourceEpoch *netmon.State + networkSourceReadFailed bool + // recoveryDebounceTimer coalesces rapid NetworkChange recovery triggers // into a single handleRecovery call. Only handleRecovery is debounced — // all other state updates (IP, pf anchor, VPN DNS) run immediately. diff --git a/docs/dns-intercept-mode.md b/docs/dns-intercept-mode.md index 508952ec..e1724a0b 100644 --- a/docs/dns-intercept-mode.md +++ b/docs/dns-intercept-mode.md @@ -312,12 +312,14 @@ Programs like Parallels Desktop reload `/etc/pf.conf` when creating/destroying v ctrld detects interface appearance/disappearance and spawns an async probe monitor: -1. **Probe mechanism:** A subprocess runs with GID=0 (wheel, not `_ctrld`) and sends a DNS query to the OS resolver. If pf interception is working, the query gets redirected to ctrld (127.0.0.1:53) and is detected in the DNS handler. If broken, it times out after 1s. +1. **Probe mechanism:** A subprocess runs with GID=0 (wheel, not `_ctrld`) and sends a DNS query to the OS resolver. If pf interception is working, the query gets redirected to ctrld (127.0.0.1:53) and is detected in the DNS handler. A confirmed send without local receipt within 1s gives `not_intercepted`. 2. **Backoff schedule:** Probes at 0, 0.5, 1, 2, 4 seconds (~8s window) to win the race against async pf reloads by the hypervisor. Only one monitor runs at a time (atomic singleton). -3. **Auto-heal:** On probe failure, `forceReloadPFMainRuleset()` dumps the running ruleset and pipes it back through `pfctl -f -`, resetting pf's translation engine. VPN-safe because it reassembles from the current running state. +3. **Auto-heal:** Only after `not_intercepted`, `forceReloadPFMainRuleset()` dumps the running ruleset and pipes it back through `pfctl -f -`, resetting pf's translation engine. VPN-safe because it reassembles from the current running state. 4. **Watchdog integration:** The 30s watchdog also runs the probe when rule text checks pass, as a safety net for unknown corruption causes. -This approach detects **actual broken DNS** rather than guessing from trigger events, making it robust against future unknown corruption scenarios. +The probe uses IPv4 only. Missing targets and helper failures give `indeterminate`, which does not cause a reload. +See [target selection and diagnostics](network-recovery-diagnostics.md#pf-probes-macos). +A successful probe proves local interception delivery, not remote DNS health. ### 5. Proactive DoH Connection Pool Reset diff --git a/docs/network-recovery-diagnostics.md b/docs/network-recovery-diagnostics.md new file mode 100644 index 00000000..e7c18c68 --- /dev/null +++ b/docs/network-recovery-diagnostics.md @@ -0,0 +1,88 @@ +# Network-recovery diagnostics + +Use continuous `ctrld log tail` capture during a reproduction. Internal logs rotate by size. +A later `log send` can contain recent debug events and older warnings. +Match timestamps and IDs. The last retained error does not necessarily describe the current network. +Issue #604 tracks upload-time device and version headers separately. + +## Transitions and recovery + +`Network transition` debug events include a `transition_id`, affected interface, default route, and stored source addresses before and after the callback. +The outcome identifies an accepted, ignored, superseded, or unusable change. +Newer accepted changes supersede pending recovery. Ignored changes do not cancel accepted recovery. +Source commits validate addresses against the current network state. +Netmon does not order minor callbacks within one major cache epoch. +For minor callbacks and late major callbacks in that epoch, ctrld reads current interface addresses and flags. +This read does not change host DNS. A failed read defers source writes, but reconciliation continues. + +`Removed stale resolver source` is a warning only when ctrld removes an invalid address. +The warning gives the reason and survives debug rotation. +`Recovery begin` and `Recovery end` connect the transition to a `recovery_generation`, reason, outcome, and elapsed time. +Only the first upstream failure in each recovery produces a bounded summary. +If a recovery encounters a failure, its final outcome also enters the warning stream. +Normal successful recovery stays at debug level. + +These summaries exclude upstream URLs and queried names. +Each PF probe records the recovery generation at its start. +A later recovery cannot receive credit for an earlier probe. + +### Cancellation event migration + +`Recovery end` with `outcome=canceled` replaces `Recovery canceled; DNS settings remain removed`. +Update support runbooks and log searches to use the structured event. +The cancellation event alone does not prove that interface DNS remains removed. + +## PF probes (macOS) + +`DNS intercept probe result` includes these fields: + +- `probe_id` and `recovery_generation` +- Numeric `resolver_target` and `resolver_family` +- `stage`, bounded `error_code`, and `outcome` +- `repair_eligible`, `repeated_results`, and elapsed milliseconds + +A changed unsuccessful condition produces a warning. The condition consists of the stage, error code, outcome, and target. +Equal results stay at debug level. The next changed condition includes the preceding repeat count. +The first restored receipt also produces a warning-stream event. This event prevents an older failure from appearing current. +Normal receipts and shutdown cancellation stay at debug level. +The warning buffer is finite. These events do not guarantee indefinite retention. + +The outcomes have these meanings: + +- `intercepted`: the registered query reached the local handler. This proves interception delivery, not remote DNS health. +- `not_intercepted`: the helper confirmed a successful UDP write, but the handler did not receive the query before the deadline. + This result permits bounded repair. It does not identify an external PF writer or explain the delivery failure. +- `indeterminate`: the probe has no suitable target, cannot send, receives an invalid helper acknowledgement, or stops before a conclusive result. + This result alone does not cause a PF reload or transport reset. The existing monitor or watchdog retries later. + Independent evidence of missing rules still permits ordinary repair. + +The literal stages are `target`, `start`, `decode`, `dial`, `deadline`, `write`, `sent`, `helper`, `delivery`, and `received`. +`sent` is the helper acknowledgement before the final delivery result. +The error codes are `unavailable`, `invalid_argument`, `unreachable`, `source_unavailable`, `permission`, `timeout`, `io`, `invalid_status`, and `canceled`. +Successful results have an empty error code. No raw subprocess output enters these fields. + +The sender runs outside the `_ctrld` group exemption. +ctrld blocks outgoing IPv6 DNS with its own PF rule and does not redirect it. +The selector preserves a usable first IPv4 target, including an original public resolver. +Otherwise, it selects the first usable IPv4 address in the leading LAN group. +It stops at the public group, which can contain a synthetic fallback. It does not blacklist a particular public IP. +Loopback, unspecified, and multicast addresses are not usable targets. + +`PF repair` events carry the causal `probe_id` and `recovery_generation` for all four callers. +They record `started`, `not_run`, or `reload_completed`. Completed means that the reload ran, not that interception recovered. +A confirming probe adds its own ID and its interception outcome to the original repair correlation. +The watchdog leaves confirmation to its next scheduled probe. +`budget_exhausted` records a wake repair that the existing budget prevents. +A probe-triggered reload does not count as a missing-anchor repair. + +See [PF DNS interception](pf-dns-intercept.md) and [DNS intercept mode](dns-intercept-mode.md) for the rule lifecycle. + +## Reproduction boundaries + +Compare the affected release with the exact candidate artifact on an owned host. +Keep the configuration unchanged. Capture ordinary DNS and a direct query to the actual listener and port. +Measure recovery after a usable route and resolver return. Exclude unavoidable offline time. +Record source-address changes, PF observations, and upstream results. Success at one layer does not prove success at another. + +Use the recovery mode, ownership, outcome, and effective network-service DNS configuration to assess cancellation. +Never flush global PF state to collect diagnostics. diff --git a/docs/pf-dns-intercept.md b/docs/pf-dns-intercept.md index 1055460c..488f56e4 100644 --- a/docs/pf-dns-intercept.md +++ b/docs/pf-dns-intercept.md @@ -278,10 +278,13 @@ change handler and spawns an asynchronous interception probe monitor: 1. A subprocess sends a DNS query WITHOUT the `_ctrld` group GID, so pf intercept rules apply to it 2. If ctrld receives the query → pf interception is working -3. If the query times out (1s) → pf translation is broken -4. On failure: `forceReloadPFMainRuleset()` does `pfctl -f -` with the current +3. A successful UDP write without local receipt within 1s gives `not_intercepted`. +4. Only this meaningful failure permits probe-triggered repair: `forceReloadPFMainRuleset()` does `pfctl -f -` with the current running ruleset, resetting pf's translation engine +A missing target or helper failure gives `indeterminate` and does not cause a reload. +The probe uses IPv4 only. See [target selection and diagnostics](network-recovery-diagnostics.md#pf-probes-macos). + The monitor probes with exponential backoff (0, 0.5, 1, 2, 4s) to win the race against async pf reloads. Only one monitor runs at a time (singleton). The watchdog also runs the probe every 30s as a safety net. @@ -300,8 +303,8 @@ Some pf.conf files include `set skip on lo0` which tells pf to skip ALL processi **Mitigation:** the interception probe. `probePFIntercept()` sends a real query from outside the `_ctrld` group and confirms the listener received the redirect, which cannot -succeed while pf is bypassing loopback — so a skip on `lo0` shows up as a probe failure -and triggers a full reload. +succeed while pf bypasses loopback. A confirmed send without receipt permits a full reload. +An unsent or unavailable probe is indeterminate and does not trigger repair. **Not implemented, contrary to earlier versions of this document:** ctrld does *not* strip `lo0` from `set skip on` directives, and the watchdog does *not* inspect skip From d1690545f9c6f51de35931a85ccf3a293479dfa1 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Fri, 11 Sep 2026 15:15:40 +0700 Subject: [PATCH 08/29] dns_intercept: don't write a GP sibling on failed NRPT handback A failed handback restored ctrld's rule via addNRPTCatchAllRule, which also writes the GP-path rule whenever another GP rule exists - leaving CtrldCatchAll beside the administrator's catch-all. Restore only the local store while that child is still present. --- .../dns_intercept_lifecycle_windows_test.go | 9 +- cmd/cli/dns_intercept_windows.go | 151 ++++-- cmd/cli/nrpt_handback_windows_test.go | 494 ++++++++++++++++-- docs/dns-intercept-mode.md | 16 +- docs/wfp-dns-intercept.md | 8 +- 5 files changed, 586 insertions(+), 92 deletions(-) diff --git a/cmd/cli/dns_intercept_lifecycle_windows_test.go b/cmd/cli/dns_intercept_lifecycle_windows_test.go index 9abad5d2..55799cd3 100644 --- a/cmd/cli/dns_intercept_lifecycle_windows_test.go +++ b/cmd/cli/dns_intercept_lifecycle_windows_test.go @@ -38,10 +38,11 @@ func newInterceptTestProg(t *testing.T) (*prog, *wfpState, *fakeNRPTOps) { // would have hit the host's real policy. func assertNoNRPTSideEffects(t *testing.T, f *fakeNRPTOps) { t.Helper() - add, remove, signal, _ := f.counts() - if add != 0 || remove != 0 || signal != 0 { - t.Errorf("addRule = %d, removeRule = %d, signal = %d, want 0/0/0: this path must not write NRPT policy", - add, remove, signal) + add, restoreLocal := f.writes() + _, remove, signal, _ := f.counts() + if add != 0 || restoreLocal != 0 || remove != 0 || signal != 0 { + t.Errorf("addRule = %d, restoreLocalRule = %d, removeRule = %d, signal = %d, want 0/0/0/0: this path must not write NRPT policy", + add, restoreLocal, remove, signal) } if flush := f.flushCount(); flush != 0 { t.Errorf("flush calls = %d, want 0: this path must not flush the resolver cache", flush) diff --git a/cmd/cli/dns_intercept_windows.go b/cmd/cli/dns_intercept_windows.go index 77763031..95738621 100644 --- a/cmd/cli/dns_intercept_windows.go +++ b/cmd/cli/dns_intercept_windows.go @@ -399,6 +399,11 @@ const ( // (Get-DnsClientNrptPolicy returns empty). This is a deterministic GUID // so we can reliably find and clean up our own rule. nrptDirectRuleName = `{B2E9A3C1-7F4D-4A8E-9D6B-5C1E0F3A2B8D}` + + // nrptDirectRulePath is the full path of ctrld's local-store catch-all: the + // primary key, written on every start and the only one a failed handback puts + // back. Named once so the writers, the reader and the remover cannot drift. + nrptDirectRulePath = nrptDirectKey + `\` + nrptDirectRuleName ) func (p *prog) nrptListenerIP() string { @@ -522,7 +527,7 @@ func findConflictingGPCatchAll(listenerIP string) (string, string) { // - After GP writes, call RefreshPolicyEx to activate. func addNRPTCatchAllRule(listenerIP string) error { // Always write to local/direct service store path. - if err := writeNRPTRule(nrptDirectKey+`\`+nrptDirectRuleName, listenerIP); err != nil { + if err := writeNRPTRule(nrptDirectRulePath, listenerIP); err != nil { return fmt.Errorf("failed to write NRPT local path rule: %w", err) } @@ -541,6 +546,41 @@ func addNRPTCatchAllRule(listenerIP string) error { return nil } +// restoreNRPTLocalCatchAllRule writes ctrld's catch-all to the local/direct store and +// touches nothing else. It is the only writer ctrld may use while an administrator's +// catch-all is on disk: the handback whose probe found no route, and startup when it must +// install ctrld's own rule beside external policy. +// +// addNRPTCatchAllRule cannot be used there. It writes ctrld's GP-path rule whenever any +// other GP rule exists, and the administrator's catch-all under test is exactly such a +// rule - so "restoring" through it plants CtrldCatchAll beside that child: the second GP +// catch-all that an administrator did not write. +// +// The contract this file keeps, which is narrower than a restore: +// +// - ctrld does not write its GP rule next to an administrator catch-all. +// - This writes back the local GUID key, and only that key. +// - Every other key removeNRPTCatchAllRule deleted stays deleted: the legacy local name +// and ctrld's GP CtrldCatchAll. A GP sibling left behind by an earlier run does not +// come back, so the store after this write is not the store the transition started +// from. +// - What comes back is ownership, not delivery. The branch is reached because the +// post-removal probe failed, and while GP policy is in effect the DNS Client can +// ignore the local key entirely. Nor is the withheld sibling knowably useless: +// isMatchingGPNRPTRule reads only Name and GenericDNSServers, so a child it accepts +// can still be one the DNS Client skips for want of Comment, DisplayName or +// IPSECCARestriction - in which case ctrld's GP rule was the route. +// +// The cost of that contract is that there may be no DNS until the administrator's rule is +// corrected. ctrld logs a warning, keeps ownership, and re-tests the external child after +// nrptHandbackRetryInterval. +func restoreNRPTLocalCatchAllRule(listenerIP string) error { + if err := writeNRPTRule(nrptDirectRulePath, listenerIP); err != nil { + return fmt.Errorf("failed to restore NRPT local path rule: %w", err) + } + return nil +} + // otherGPRulesExist checks if non-ctrld NRPT rules exist in the GP path. // When other software (IT policy, VPN, MDM) has GP rules, DNS Client enters // "GP mode" and ignores ALL local-path rules. @@ -568,6 +608,12 @@ func otherGPRulesExist() bool { // // Do not leave an empty GP parent behind: Windows treats the parent key itself // as the policy store boundary, so an empty key can still hide local-path rules. +// removeCtrldGPRule deletes ctrld's GP-store catch-all and nothing else: not the parent +// key, which an administrator rule still occupies, and not the local store. +func removeCtrldGPRule() { + registry.DeleteKey(registry.LOCAL_MACHINE, nrptBaseKey+`\`+nrptRuleName) +} + func cleanGPPath() bool { // Delete our specific rule. registry.DeleteKey(registry.LOCAL_MACHINE, nrptBaseKey+`\`+nrptRuleName) @@ -635,7 +681,7 @@ func writeNRPTRule(keyPath, listenerIP string) error { // the empty parent on stop, we ensure a clean slate for the next start. func removeNRPTCatchAllRule() error { // Remove our GUID-named rule from local/direct path. - if err := registry.DeleteKey(registry.LOCAL_MACHINE, nrptDirectKey+`\`+nrptDirectRuleName); err != nil { + if err := registry.DeleteKey(registry.LOCAL_MACHINE, nrptDirectRulePath); err != nil { if err != registry.ErrNotExist { return fmt.Errorf("failed to delete NRPT local rule: %w", err) } @@ -665,7 +711,7 @@ func deleteEmptyParentKey(keyPath string) { // in either the local or GP path. func nrptCatchAllRuleExists() bool { for _, path := range []string{ - nrptDirectKey + `\` + nrptDirectRuleName, + nrptDirectRulePath, nrptBaseKey + `\` + nrptRuleName, } { k, err := registry.OpenKey(registry.LOCAL_MACHINE, path, registry.QUERY_VALUE) @@ -877,7 +923,20 @@ func (p *prog) startDNSInterceptLocked() error { owner, _ = state.nrptPolicyOwner() if owner == nrptRuleOwnerNone { - if ops.ruleExists() { + exists := ops.ruleExists() + if exists && ops.findGPRule(listenerIP) != "" { + // An earlier run left ctrld's GP-store catch-all beside an administrator + // catch-all, and adopting would inherit it: nothing downstream removes it. + // Every handback from here stops at the pre-probe while that rule is + // unproven, the monitor calls removeRule only inside a handback, and the + // orphan sweep runs at stop - so the second catch-all would stand for the + // whole process lifetime, through a DNS outage. Drop it before adopting, + // then re-read: ruleExists is also true for the sibling alone, and a store + // holding nothing else must fall through to the writer below. + ops.dropGPRule() + exists = ops.ruleExists() + } + if exists { // A rule from an earlier run already points at this listener. Adopt it rather // than writing a second time: with a GP child present, addNRPTCatchAllRule // would also write ctrld's GP-path sibling. @@ -885,10 +944,22 @@ func (p *prog) startDNSInterceptLocked() error { mainLog.Load().Info().Str("listener", listenerIP). Msg("DNS intercept: adopting the ctrld NRPT catch-all already present from an earlier run") } else { + // Which writer is allowed here is decided the same way the handback decides + // it: with an administrator catch-all on disk, addNRPTCatchAllRule would + // place ctrld's GP rule beside it, so only the local key may be written. + // + // This is not a hypothetical branch. The sibling cleanup above falls into it + // whenever the store held nothing but ctrld's GP key - having just deleted + // that key, the full writer would put it straight back, next to the very + // rule it was removed from. + write := ops.addRule + if ops.findGPRule(listenerIP) != "" { + write = ops.restoreLocalRule + } if ops.cleanParent() { ops.signal() } - if err := ops.addRule(listenerIP); err != nil { + if err := write(listenerIP); err != nil { return fmt.Errorf("dns intercept: failed to add NRPT catch-all rule: %w", err) } logNRPTParentKeyState("post-write") @@ -2085,12 +2156,18 @@ func gpCatchAllConflictBlocksFallback(state *wfpState, reason string) bool { // they cause. Tests substitute it to drive a transition - probe outcomes, registry // state, concurrency - without touching the host's registry or DNS Client. type nrptOps struct { - probe func(state *wfpState) bool - ruleExists func() bool - addRule func(listenerIP string) error - removeRule func() error - signal func() - findGPRule func(listenerIP string) string + probe func(state *wfpState) bool + ruleExists func() bool + addRule func(listenerIP string) error + // restoreLocalRule re-writes only ctrld's local-store catch-all, leaving the GP + // store exactly as found. See restoreNRPTLocalCatchAllRule. + restoreLocalRule func(listenerIP string) error + removeRule func() error + signal func() + findGPRule func(listenerIP string) string + // dropGPRule deletes ctrld's GP-store catch-all, leaving the administrator's rule + // and the local store alone. See removeCtrldGPRule. + dropGPRule func() gpRuleMatches func(ruleName, listenerIP string) bool gpConflicts func(state *wfpState, reason string) bool loopback func(state *wfpState) error @@ -2109,20 +2186,22 @@ func (p *prog) nrptOps() nrptOps { return *nrptOpsForTest } return nrptOps{ - probe: p.probeNRPT, - ruleExists: nrptCatchAllRuleExists, - addRule: addNRPTCatchAllRule, - removeRule: removeNRPTCatchAllRule, - signal: signalNRPTChange, - findGPRule: findMatchingGPNRPTRule, - gpRuleMatches: gpNRPTRuleMatches, - gpConflicts: p.gpCatchAllConflictBlocksFallbackOps, - loopback: p.activateLoopbackWFPProtect, - wait: p.interceptWait, - flush: flushDNSCache, - parentEmpty: nrptParentKeyEmpty, - cleanParent: cleanEmptyNRPTParent, - startWFP: p.startWFPFilters, + probe: p.probeNRPT, + ruleExists: nrptCatchAllRuleExists, + addRule: addNRPTCatchAllRule, + restoreLocalRule: restoreNRPTLocalCatchAllRule, + removeRule: removeNRPTCatchAllRule, + signal: signalNRPTChange, + findGPRule: findMatchingGPNRPTRule, + dropGPRule: removeCtrldGPRule, + gpRuleMatches: gpNRPTRuleMatches, + gpConflicts: p.gpCatchAllConflictBlocksFallbackOps, + loopback: p.activateLoopbackWFPProtect, + wait: p.interceptWait, + flush: flushDNSCache, + parentEmpty: nrptParentKeyEmpty, + cleanParent: cleanEmptyNRPTParent, + startWFP: p.startWFPFilters, } } @@ -2284,10 +2363,22 @@ func (p *prog) nrptHandbackToExternal(state *wfpState, ruleName, reason string) } - // The original child is still exact but cannot carry DNS, or external policy is gone - // altogether: ctrld's route is the one that has to come back. This only restores the - // state the transition started from, so it creates no new sibling. - if err := ops.addRule(state.listenerIP); err != nil { + // ctrld's route has to come back. Class decides the writer: + // + // gpChildGone the namespace is free, so the full writer is correct - it may + // place ctrld's GP rule where unrelated GP rules would otherwise + // hide the local store, or drop an empty parent to leave GP mode. + // anything else an external catch-all is still on disk, so only the local key + // may be written. See restoreNRPTLocalCatchAllRule. + // + // The local writer is the default rather than the exception: reaching the full writer + // with any external catch-all present writes the sibling that must never exist, so a + // class added later must opt in to it deliberately. + restore := ops.restoreLocalRule + if class == gpChildGone { + restore = ops.addRule + } + if err := restore(state.listenerIP); err != nil { mainLog.Load().Error().Err(err).Str("rule", ruleName). Msg("DNS intercept: handback probe failed and the ctrld NRPT rule could not be restored; the health monitor will retry") state.setNRPTPolicyOwner(nrptRuleOwnerNone, "") @@ -2300,7 +2391,7 @@ func (p *prog) nrptHandbackToExternal(state *wfpState, ruleName, reason string) Msg("DNS intercept: GP-managed catch-all disappeared during the handback probe; restored the ctrld fallback and kept ctrld ownership") } else { mainLog.Load().Warn().Str("rule", ruleName).Str("reason", reason). - Msg("DNS intercept: GP-managed catch-all did not carry DNS without ctrld's rule; restored the ctrld fallback and kept ctrld ownership") + Msg("DNS intercept: GP-managed catch-all could not be proved to carry DNS without ctrld's rule; wrote back ctrld's local-store key without writing a GP sibling, and kept ctrld ownership") } return nrptHandbackKeptCtrld } diff --git a/cmd/cli/nrpt_handback_windows_test.go b/cmd/cli/nrpt_handback_windows_test.go index 153376ae..7bea7cbd 100644 --- a/cmd/cli/nrpt_handback_windows_test.go +++ b/cmd/cli/nrpt_handback_windows_test.go @@ -28,11 +28,23 @@ type fakeNRPTOps struct { parentEmpty bool // whether the GP parent key reads as present but empty cleanCalls int gpConflictCalls int - ctrldRule bool // whether ctrld's own catch-all exists - existsCalls int - addCalls int - removeCalls int - signalCalls int + ctrldRule bool // whether ctrld's own catch-all exists in the local store + // ctrldGPRule is whether ctrld's own catch-all also exists in the GP store. The two + // stores are modelled separately because that is where the duplicate lives: the + // production writer adds ctrld's GP rule whenever any other GP rule is present, so a + // fake with a single "ctrld rule exists" flag cannot see a sibling being written + // beside an administrator's catch-all. + ctrldGPRule bool + // otherGPRules models GP rules that are not catch-alls. Production + // otherGPRulesExist is true for any GP rule at all, so the full writer places + // ctrld's GP rule for these too - a shape a gpRule-only fake cannot express. + otherGPRules bool + dropGPCalls int + existsCalls int + addCalls int + restoreLocalCalls int + removeCalls int + signalCalls int addErr error removeErr error @@ -74,7 +86,9 @@ func (f *fakeNRPTOps) ops() *nrptOps { f.mu.Lock() defer f.mu.Unlock() f.existsCalls++ - return f.ctrldRule + // nrptCatchAllRuleExists reads the local GUID key and the GP CtrldCatchAll + // key, so a store holding only the sibling still reports "a rule exists". + return f.ctrldRule || f.ctrldGPRule }, addRule: func(string) error { f.mu.Lock() @@ -82,6 +96,9 @@ func (f *fakeNRPTOps) ops() *nrptOps { f.addCalls++ if err == nil { f.ctrldRule = true + // Mirror addNRPTCatchAllRule: with any other GP rule present it also + // writes ctrld's GP-path rule, otherwise it cleans the GP path. + f.ctrldGPRule = f.gpRule != "" || f.gpConflict || f.otherGPRules } f.mu.Unlock() if beforeAdd != nil { @@ -89,6 +106,16 @@ func (f *fakeNRPTOps) ops() *nrptOps { } return err }, + restoreLocalRule: func(string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.restoreLocalCalls++ + if f.addErr != nil { + return f.addErr + } + f.ctrldRule = true // local store only; the GP store is left as found + return nil + }, removeRule: func() error { f.mu.Lock() defer f.mu.Unlock() @@ -97,6 +124,7 @@ func (f *fakeNRPTOps) ops() *nrptOps { return f.removeErr } f.ctrldRule = false + f.ctrldGPRule = false return nil }, signal: func() { @@ -104,6 +132,12 @@ func (f *fakeNRPTOps) ops() *nrptOps { defer f.mu.Unlock() f.signalCalls++ }, + dropGPRule: func() { + f.mu.Lock() + defer f.mu.Unlock() + f.dropGPCalls++ + f.ctrldGPRule = false // the local store and the administrator's rule are untouched + }, findGPRule: func(string) string { f.mu.Lock() defer f.mu.Unlock() @@ -191,6 +225,37 @@ func (f *fakeNRPTOps) hasCtrldRule() bool { return f.ctrldRule } +// hasCtrldGPSibling reports whether ctrld's own rule is sitting in the GP store, which +// beside an administrator's catch-all is the second GP catch-all that an administrator +// did not write. +func (f *fakeNRPTOps) hasCtrldGPSibling() bool { + f.mu.Lock() + defer f.mu.Unlock() + return f.ctrldGPRule +} + +// writes reports every way this file can put ctrld's catch-all back, so a "nothing was +// written" assertion cannot pass by checking one writer while the other ran. +func (f *fakeNRPTOps) writes() (add, restoreLocal int) { + f.mu.Lock() + defer f.mu.Unlock() + return f.addCalls, f.restoreLocalCalls +} + +// dropGPCount is how many times ctrld's GP-store key was deleted on its own. +func (f *fakeNRPTOps) dropGPCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.dropGPCalls +} + +// restoreLocalCount is how many times the local-store-only restore ran. +func (f *fakeNRPTOps) restoreLocalCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.restoreLocalCalls +} + // installedFakeNRPTOps is the fake currently standing in for the production NRPT // operations, so fixtures can hand it to tests and can tell whether one is installed at // all. @@ -215,6 +280,8 @@ type fakeNRPTSeed struct { gpConflict bool parentEmpty bool ctrldRule bool + ctrldGPRule bool + otherGPRules bool addErr error removeErr error } @@ -229,6 +296,8 @@ func (f *fakeNRPTOps) configure(seed fakeNRPTSeed) { f.gpConflict = seed.gpConflict f.parentEmpty = seed.parentEmpty f.ctrldRule = seed.ctrldRule + f.ctrldGPRule = seed.ctrldGPRule + f.otherGPRules = seed.otherGPRules f.addErr = seed.addErr f.removeErr = seed.removeErr } @@ -306,7 +375,15 @@ func newHandbackTestProg(t *testing.T) (*prog, *wfpState, *fakeNRPTOps) { // Getting this wrong deletes the last working route and declares external ownership: in // hard mode a machine-wide DNS outage, because WFP keeps blocking outbound DNS with // nothing redirecting it to ctrld. +// +// How it comes back matters just as much, and is the #576 owned-lab regression: the +// administrator's child is still on disk, so restoring through addNRPTCatchAllRule leaves +// CtrldCatchAll sitting in the GP store beside it - a machine that started with one GP +// catch-all ends the failed handback with two, both naming the same listener. Only the +// local store may be rewritten here. func TestHandbackRestoresFallbackWhenGPCannotRoute(t *testing.T) { + // The lab fixture: ctrld holds only its local rule, and the GP store holds only the + // administrator's exact catch-all - ctrld has no GP rule to begin with. f := &fakeNRPTOps{probeResults: []bool{true, false}, gpRule: "{GP-RULE}", ctrldRule: true} installFakeNRPTOps(t, f) @@ -323,12 +400,49 @@ func TestHandbackRestoresFallbackWhenGPCannotRoute(t *testing.T) { if remove != 1 { t.Errorf("removeRule calls = %d, want 1: the handback probe must run with ctrld's keys gone", remove) } - if add != 1 { - t.Errorf("addRule calls = %d, want 1: the ctrld fallback must be restored after the probe fails", add) + if f.restoreLocalCount() != 1 { + t.Errorf("restoreLocalRule calls = %d, want 1: the ctrld fallback must be restored after the probe fails", f.restoreLocalCount()) + } + if add != 0 { + t.Errorf("addRule calls = %d, want 0: the full writer adds ctrld's GP rule beside the administrator's child", add) } if !f.hasCtrldRule() { t.Error("ctrld's NRPT rule is missing after a failed handback; the host has no working route") } + if f.hasCtrldGPSibling() { + t.Error("the failed handback left ctrld's rule in the GP store beside the administrator's catch-all") + } +} + +// TestHandbackRestoreDropsAPreexistingGPSibling covers the machine that is already in the +// forbidden state: ctrld wrote its GP rule while only unrelated GP rules existed, and an +// administrator catch-all arrived afterwards. The failed handback must not preserve that +// duplicate on the way out. +// +// It does not, and gets there without a special case. removeRule takes all of ctrld's keys +// for the probe - the local GUID key, the legacy local name and the GP CtrldCatchAll - and +// the local-only write puts back just the first, so the store converges to a single +// catch-all, the administrator's. What is asserted is the shape of the store, not that DNS +// is flowing: this branch exists because the probe failed. +func TestHandbackRestoreDropsAPreexistingGPSibling(t *testing.T) { + p, state, f := newHandbackTestProg(t) + f.configure(fakeNRPTSeed{ + probeResults: []bool{true, false}, + gpRule: "{GP-RULE}", + ctrldRule: true, + ctrldGPRule: true, // the duplicate this run inherited + }) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackKeptCtrld { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackKeptCtrld", got) + } + if f.hasCtrldGPSibling() { + t.Error("the inherited GP sibling survived the handback; the store still holds two catch-alls") + } + if !f.hasCtrldRule() { + t.Error("ctrld's local-store rule was not restored; the transition dropped ctrld's own key instead of putting it back") + } } // TestHandbackAcceptsGPWhenItRoutesWithoutCtrld is the other half of the contract: when @@ -349,8 +463,8 @@ func TestHandbackAcceptsGPWhenItRoutesWithoutCtrld(t *testing.T) { t.Errorf("owner = %v, rule = %q, want GroupPolicy/{GP-RULE}", owner, ruleName) } add, remove, _, _ := f.counts() - if remove != 1 || add != 0 { - t.Errorf("removeRule = %d, addRule = %d, want 1/0: ctrld's rule must be removed and not restored", remove, add) + if remove != 1 || add != 0 || f.restoreLocalCount() != 0 { + t.Errorf("removeRule = %d, addRule = %d, restoreLocalRule = %d, want 1/0/0: ctrld's rule must be removed and not restored", remove, add, f.restoreLocalCount()) } if f.hasCtrldRule() { t.Error("ctrld's rule is still installed beside the adopted GP catch-all") @@ -381,8 +495,9 @@ func TestIneffectiveGPRuleTriggersNoSignalling(t *testing.T) { if signal != 0 { t.Errorf("signal calls = %d, want 0: ctrld must not force GP refresh, paramchange or a cache flush while external policy owns NRPT", signal) } - if add != 0 || remove != 0 { - t.Errorf("addRule = %d, removeRule = %d, want 0/0: external policy must be left untouched", add, remove) + if add != 0 || remove != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, restoreLocalRule = %d, removeRule = %d, want 0/0/0: external policy must be left untouched", + add, f.restoreLocalCount(), remove) } if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerGroupPolicy { t.Errorf("owner = %v, want nrptRuleOwnerGroupPolicy: ctrld must not seize a namespace an administrator owns", owner) @@ -544,9 +659,9 @@ func TestOwnedRecoveryDefersToIneffectiveExternalPolicy(t *testing.T) { p.nrptProbeAndHeal(state) add, remove, signal, _ := f.counts() - if add != 0 || remove != 0 || signal != 0 { - t.Errorf("addRule = %d, removeRule = %d, signal = %d, want 0/0/0: owned recovery must stop once external policy owns the namespace", - add, remove, signal) + if add != 0 || remove != 0 || signal != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, removeRule = %d, signal = %d, restoreLocalRule = %d, want 0/0/0/0: owned recovery must stop once external policy owns the namespace", + add, remove, signal, f.restoreLocalCount()) } if f.hasCtrldRule() { t.Error("owned recovery recreated ctrld's catch-all beside the administrator's rule") @@ -601,8 +716,12 @@ func TestHandbackRestoresFallbackWhenGPChildDisappearsMidProbe(t *testing.T) { if !f.hasCtrldRule() { t.Error("ctrld's rule was not restored after the GP child disappeared; the host has no route") } - if add, remove, _, _ := f.counts(); add != 1 || remove != 1 { - t.Errorf("addRule = %d, removeRule = %d, want 1/1", add, remove) + // A freed namespace is the one case the full writer is still correct for: with no + // external catch-all left to sit beside, it may place ctrld's GP rule where unrelated + // GP rules would otherwise hide the local store, or drop an empty parent to leave GP + // mode. Narrowing this case to a local-only write would leave ctrld invisible there. + if add, remove, _, _ := f.counts(); add != 1 || remove != 1 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, restoreLocalRule = %d, removeRule = %d, want 1/0/1", add, f.restoreLocalCount(), remove) } } @@ -629,8 +748,8 @@ func TestHandbackWritesNoSiblingWhenGPChildTurnsConflicting(t *testing.T) { if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackConflict { t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackConflict", got) } - if add, _, _, _ := f.counts(); add != 0 { - t.Errorf("addRule calls = %d, want 0: restoring here writes a competing rule beside administrator policy", add) + if add, restoreLocal := f.writes(); add != 0 || restoreLocal != 0 { + t.Errorf("addRule calls = %d, restoreLocalRule calls = %d, want 0/0: restoring here writes a competing rule beside administrator policy", add, restoreLocal) } if f.hasCtrldRule() { t.Error("ctrld's rule is installed beside a GP catch-all that targets another resolver") @@ -659,8 +778,8 @@ func TestHandbackWithoutCtrldRuleClassifiesAfterFailedProbe(t *testing.T) { if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackConflict { t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackConflict", got) } - if add, remove, signal, _ := f.counts(); add != 0 || remove != 0 || signal != 0 { - t.Errorf("addRule = %d, removeRule = %d, signal = %d, want 0/0/0", add, remove, signal) + if add, remove, signal, _ := f.counts(); add != 0 || remove != 0 || signal != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, removeRule = %d, signal = %d, restoreLocalRule = %d, want 0/0/0/0", add, remove, signal, f.restoreLocalCount()) } owner, ruleName := state.nrptPolicyOwner() if owner != nrptRuleOwnerNone || ruleName != "" { @@ -686,8 +805,8 @@ func TestHandbackGivesReplacementChildItsOwnPass(t *testing.T) { if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackUnverified { t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackUnverified", got) } - if add, remove, signal, _ := f.counts(); add != 0 || remove != 0 || signal != 0 { - t.Errorf("addRule = %d, removeRule = %d, signal = %d, want 0/0/0", add, remove, signal) + if add, remove, signal, _ := f.counts(); add != 0 || remove != 0 || signal != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, removeRule = %d, signal = %d, restoreLocalRule = %d, want 0/0/0/0", add, remove, signal, f.restoreLocalCount()) } owner, ruleName := state.nrptPolicyOwner() if owner != nrptRuleOwnerGroupPolicy || ruleName != "{NEW-GP-RULE}" { @@ -720,8 +839,9 @@ func TestHandbackWritesNoSiblingWhenReplacementChildTakesOver(t *testing.T) { if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackUnverified { t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackUnverified", got) } - if add, remove, _, _ := f.counts(); add != 0 || remove != 1 { - t.Errorf("addRule = %d, removeRule = %d, want 0/1: ctrld's rule must not be restored beside a replacement catch-all", add, remove) + if add, remove, _, _ := f.counts(); add != 0 || remove != 1 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, restoreLocalRule = %d, removeRule = %d, want 0/0/1: ctrld's rule must not be restored beside a replacement catch-all", + add, f.restoreLocalCount(), remove) } if f.hasCtrldRule() { t.Error("ctrld's rule is installed beside an administrator catch-all that took the namespace") @@ -749,8 +869,8 @@ func TestHandbackAdoptsReplacementChildThatRoutes(t *testing.T) { if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackVerified { t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackVerified", got) } - if add, _, _, _ := f.counts(); add != 0 { - t.Errorf("addRule calls = %d, want 0", add) + if add, restoreLocal := f.writes(); add != 0 || restoreLocal != 0 { + t.Errorf("addRule calls = %d, restoreLocalRule calls = %d, want 0/0", add, restoreLocal) } owner, ruleName := state.nrptPolicyOwner() if owner != nrptRuleOwnerGroupPolicy || ruleName != "{NEW-GP-RULE}" { @@ -788,8 +908,8 @@ func TestHandbackChurnFailsSafeToCurrentExternalOwner(t *testing.T) { if !nrptExternalOwns(got) { t.Error("the churn disposition is not terminal external ownership; callers would fall through and write a sibling") } - if add, _, _, _ := f.counts(); add != 0 { - t.Errorf("addRule calls = %d, want 0: no sibling write while the GP store is churning", add) + if add, restoreLocal := f.writes(); add != 0 || restoreLocal != 0 { + t.Errorf("addRule calls = %d, restoreLocalRule calls = %d, want 0/0: no sibling write while the GP store is churning", add, restoreLocal) } owner, ruleName := state.nrptPolicyOwner() if owner != nrptRuleOwnerGroupPolicy || ruleName != "{GP-RULE-3}" { @@ -817,8 +937,8 @@ func TestHandbackChurnFailsSafeToConflict(t *testing.T) { if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "test"); got != nrptHandbackConflict { t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackConflict", got) } - if add, _, _, _ := f.counts(); add != 0 { - t.Errorf("addRule calls = %d, want 0", add) + if add, _, _, _ := f.counts(); add != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, restoreLocalRule = %d, want 0/0", add, f.restoreLocalCount()) } if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerNone { t.Errorf("owner = %v, want nrptRuleOwnerNone", owner) @@ -878,8 +998,8 @@ func TestActivationWritesNoSiblingWhileGPStoreChurns(t *testing.T) { t.Error("activateCtrldNRPTFallback() = true while an external catch-all owns the namespace") } add, remove, signal, _ := f.counts() - if add != 0 || signal != 0 { - t.Errorf("addRule = %d, signal = %d, want 0/0: ctrld must not write beside the administrator's catch-all", add, signal) + if add != 0 || signal != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, signal = %d, restoreLocalRule = %d, want 0/0/0: ctrld must not write beside the administrator's catch-all", add, signal, f.restoreLocalCount()) } if remove != 0 { t.Errorf("removeRule calls = %d, want 0: there was no ctrld rule to remove", remove) @@ -905,9 +1025,9 @@ func TestStopLeavesGPManagedPolicyAlone(t *testing.T) { } add, remove, signal, _ := f.counts() - if remove != 0 || signal != 0 || add != 0 { - t.Errorf("removeRule = %d, signal = %d, addRule = %d, want 0/0/0: shutdown must not touch externally owned NRPT policy", - remove, signal, add) + if remove != 0 || signal != 0 || add != 0 || f.restoreLocalCount() != 0 { + t.Errorf("removeRule = %d, signal = %d, addRule = %d, restoreLocalRule = %d, want 0/0/0/0: shutdown must not touch externally owned NRPT policy", + remove, signal, add, f.restoreLocalCount()) } if f.flushCount() != 0 { t.Errorf("flush calls = %d, want 0: no cache flush is owed for a rule ctrld does not own", f.flushCount()) @@ -935,8 +1055,8 @@ func TestStopRemovesOrphanWhileLeavingGPPolicy(t *testing.T) { if remove != 1 || signal != 1 { t.Errorf("removeRule = %d, signal = %d, want 1/1: the orphaned ctrld rule must be removed on the way out", remove, signal) } - if add != 0 { - t.Errorf("addRule calls = %d, want 0", add) + if add != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule calls = %d, restoreLocalRule calls = %d, want 0/0", add, f.restoreLocalCount()) } if f.hasCtrldRule() { t.Error("the orphaned ctrld rule survived shutdown; a later GP removal would activate it") @@ -958,8 +1078,8 @@ func TestRemoveOrphanedCtrldNRPTRule(t *testing.T) { p.removeOrphanedCtrldNRPTRule("test") add, remove, signal, _ := f.counts() - if remove != 1 || signal != 1 || add != 0 { - t.Errorf("removeRule = %d, signal = %d, addRule = %d, want 1/1/0", remove, signal, add) + if remove != 1 || signal != 1 || add != 0 || f.restoreLocalCount() != 0 { + t.Errorf("removeRule = %d, signal = %d, addRule = %d, restoreLocalRule = %d, want 1/1/0/0", remove, signal, add, f.restoreLocalCount()) } if f.hasCtrldRule() { t.Error("the ctrld rule is still installed") @@ -974,9 +1094,9 @@ func TestRemoveOrphanedCtrldNRPTRule(t *testing.T) { p.removeOrphanedCtrldNRPTRule("test") add, remove, signal, _ := f.counts() - if remove != 0 || signal != 0 || add != 0 { - t.Errorf("removeRule = %d, signal = %d, addRule = %d, want 0/0/0: nothing to sweep must cost no registry writes and no signalling", - remove, signal, add) + if remove != 0 || signal != 0 || add != 0 || f.restoreLocalCount() != 0 { + t.Errorf("removeRule = %d, signal = %d, addRule = %d, restoreLocalRule = %d, want 0/0/0/0: nothing to sweep must cost no registry writes and no signalling", + remove, signal, add, f.restoreLocalCount()) } }) } @@ -1073,8 +1193,8 @@ func TestHealLadderStopsWhenGPAppearsBeforeTwoPhase(t *testing.T) { if remove != 0 { t.Errorf("removeRule calls = %d, want 0: the ladder must stop before the delete half of the two-phase recovery", remove) } - if add != 0 { - t.Errorf("addRule calls = %d, want 0: the ladder must not recreate ctrld's rule beside a new GP catch-all", add) + if add != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule calls = %d, restoreLocalRule calls = %d, want 0/0: the ladder must not recreate ctrld's rule beside a new GP catch-all", add, f.restoreLocalCount()) } if !f.hasCtrldRule() { t.Error("ctrld's rule was deleted after an administrator catch-all appeared") @@ -1100,8 +1220,8 @@ func TestHealLadderCleansEmptyGPParentBeforeRetrying(t *testing.T) { if f.cleanCalls != 1 { t.Errorf("cleanParent calls = %d, want 1: the empty GP parent must be removed before burning retries", f.cleanCalls) } - if add, remove, _, _ := f.counts(); add != 0 || remove != 0 { - t.Errorf("addRule = %d, removeRule = %d, want 0/0: the probe passed after the cleanup, so no recovery was needed", add, remove) + if add, remove, _, _ := f.counts(); add != 0 || remove != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, removeRule = %d, restoreLocalRule = %d, want 0/0/0: the probe passed after the cleanup, so no recovery was needed", add, remove, f.restoreLocalCount()) } } @@ -1185,8 +1305,8 @@ func TestPhaseTwoRevalidatesOwnershipAfterWaitingForTheLock(t *testing.T) { p.nrptProbeAndHeal(state) - if add, _, _, _ := f.counts(); add != 0 { - t.Errorf("addRule calls = %d, want 0: phase two re-added ctrld's rule beside a GP catch-all that took ownership while it waited for the lock", add) + if add, restoreLocal := f.writes(); add != 0 || restoreLocal != 0 { + t.Errorf("addRule calls = %d, restoreLocalRule calls = %d, want 0/0: phase two re-added ctrld's rule beside a GP catch-all that took ownership while it waited for the lock", add, restoreLocal) } if f.hasCtrldRule() { t.Error("ctrld's rule is installed beside the administrator's catch-all") @@ -1267,8 +1387,9 @@ func TestStartupReportsFailureWhenExternalPolicyNeverRoutes(t *testing.T) { if interceptFailedWithVerifiedExternalDNS(err) { t.Errorf("err = %v; this route was never verified, so it must not read as the verified case", err) } - if add, remove, signal, _ := f.counts(); add != 0 || remove != 0 || signal != 0 { - t.Errorf("addRule = %d, removeRule = %d, signal = %d, want 0/0/0: no owned fallback may be written beside an administrator catch-all", add, remove, signal) + if add, remove, signal, _ := f.counts(); add != 0 || remove != 0 || signal != 0 || f.restoreLocalCount() != 0 { + t.Errorf("addRule = %d, restoreLocalRule = %d, removeRule = %d, signal = %d, want 0/0/0/0: no owned fallback may be written beside an administrator catch-all", + add, f.restoreLocalCount(), remove, signal) } // Recovery must stay live so the rule can be re-tested. @@ -1307,3 +1428,274 @@ func TestFakeNRPTOpsPreconditionDetectsBypass(t *testing.T) { t.Error("a fake that is not the installed one passed the precondition") } } + +// startupInterceptMode puts the globals startDNSInterceptLocked reads into dns mode with a +// single loopback listener, and puts them back afterwards. +func startupInterceptMode(t *testing.T) { + t.Helper() + originalCfg, originalMode, originalIntercept, originalHard := cfg, interceptMode, dnsIntercept, hardIntercept + t.Cleanup(func() { + cfg, interceptMode, dnsIntercept, hardIntercept = originalCfg, originalMode, originalIntercept, originalHard + }) + cfg = ctrld.Config{} + cfg.Listener = map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}} + interceptMode, dnsIntercept, hardIntercept = "dns", true, false +} + +// TestHandbackAbortsWhenTheLocalWriteFails covers the write failure on the put-back path. +// Nothing else exercises it: every other test lets restoreLocalRule succeed. +// +// The rule is gone by then - removeRule ran before the probe - so a failed put-back leaves +// the host with no ctrld route at all. What the transition owes is an accurate report: +// ownership goes back to none so the health monitor treats it as unowned and retries, and +// no signal is sent for a change that did not happen. +func TestHandbackAbortsWhenTheLocalWriteFails(t *testing.T) { + p, state, f := newHandbackTestProg(t) + f.configure(fakeNRPTSeed{ + probeResults: []bool{true, false}, + gpRule: "{GP-RULE}", + ctrldRule: true, + addErr: errors.New("registry write failed"), + }) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "local write fails"); got != nrptHandbackAborted { + t.Errorf("nrptHandbackToExternal() = %v, want nrptHandbackAborted", got) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerNone { + t.Errorf("NRPT owner = %v, want nrptRuleOwnerNone: a failed put-back owns nothing", owner) + } + if f.restoreLocalCount() != 1 { + t.Errorf("restoreLocalRule calls = %d, want 1", f.restoreLocalCount()) + } + if add, _, _, _ := f.counts(); add != 0 { + t.Errorf("addRule calls = %d, want 0: the failing writer must not be retried through the full writer", add) + } + if f.hasCtrldRule() { + t.Error("ctrld's local rule reads as present after the write that was supposed to create it failed") + } + if f.hasCtrldGPSibling() { + t.Error("a failed put-back left ctrld's rule in the GP store") + } +} + +// TestHandbackKeepsCtrldWhenTheStoreReturnsToTheInitialRule covers the spent-budget branch +// of externalAfterRemoval when the churn ends where it began. +// +// The three existing churn tests all end somewhere else - a third rule, a rule targeting +// another resolver, or a free namespace - so none of them reaches this line with the +// original rule name still current. That combination selects the writer: the class is +// gpChildSameExact, the administrator's catch-all is on disk, and only the local key may +// be written. +func TestHandbackKeepsCtrldWhenTheStoreReturnsToTheInitialRule(t *testing.T) { + p, state, f := newHandbackTestProg(t) + f.configure(fakeNRPTSeed{ + probeResults: []bool{true, false, false}, + gpRule: "{GP-RULE}", + ctrldRule: true, + }) + f.onProbe = func(call int) { + switch call { + case 2: + f.setGPRule("{GP-RULE-2}", false) + case 3: + f.setGPRule("{GP-RULE}", false) // back to the rule the transition started on + } + } + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "churn returns to the initial rule"); got != nrptHandbackKeptCtrld { + t.Errorf("nrptHandbackToExternal() = %v, want nrptHandbackKeptCtrld", got) + } + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerCtrld { + t.Errorf("NRPT owner = %v, want nrptRuleOwnerCtrld", owner) + } + if f.restoreLocalCount() != 1 { + t.Errorf("restoreLocalRule calls = %d, want 1", f.restoreLocalCount()) + } + if add, _, _, _ := f.counts(); add != 0 { + t.Errorf("addRule calls = %d, want 0: the spent-budget path must not write a sibling either", add) + } + if f.hasCtrldGPSibling() { + t.Error("the spent-budget path left ctrld's rule in the GP store beside the administrator's catch-all") + } +} + +// TestGPChildGoneWritesTheGPRuleWhenOtherGPRulesRemain is the counterpart to the +// no-sibling tests: with the administrator's catch-all gone but unrelated GP rules still +// present, the full writer is correct and must place ctrld's GP rule, or DNS Client's GP +// mode hides the local store and ctrld routes nothing. +func TestGPChildGoneWritesTheGPRuleWhenOtherGPRulesRemain(t *testing.T) { + p, state, f := newHandbackTestProg(t) + f.configure(fakeNRPTSeed{ + probeResults: []bool{true, false}, + gpRule: "{GP-RULE}", + ctrldRule: true, + otherGPRules: true, // unrelated GP policy that keeps DNS Client in GP mode + }) + f.onProbe = func(call int) { + if call == 2 { + f.setGPRule("", false) // the administrator's catch-all disappears mid-probe + } + } + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "child gone with other GP rules"); got != nrptHandbackKeptCtrld { + t.Errorf("nrptHandbackToExternal() = %v, want nrptHandbackKeptCtrld", got) + } + if add, _, _, _ := f.counts(); add != 1 { + t.Errorf("addRule calls = %d, want 1: a free namespace is what the full writer is for", add) + } + if f.restoreLocalCount() != 0 { + t.Errorf("restoreLocalRule calls = %d, want 0", f.restoreLocalCount()) + } + if !f.hasCtrldGPSibling() { + t.Error("ctrld's GP rule is missing while unrelated GP rules keep DNS Client in GP mode; the local store is hidden and nothing routes") + } +} + +// TestStartupKeepsCtrldRuleWhenExternalPolicyCannotRoute is the owned-lab regression as +// startup runs it: a service start with ctrld's rule from an earlier run and a matching +// administrator catch-all that cannot carry DNS alone. +// +// The existing startup test has no ctrld rule and a single failing probe, so it never +// reaches the put-back. This one does, and the assertion that matters is which writer ran. +// +// Ownership is deliberately not asserted: startup launches nrptProbeAndHeal, whose own +// handback is throttled and whose undecided branch moves the owner to Group Policy within +// seconds, so any owner assertion here is a race. +func TestStartupKeepsCtrldRuleWhenExternalPolicyCannotRoute(t *testing.T) { + startupInterceptMode(t) + + p, _, f := newHandbackTestProg(t) + p.cfg = &cfg + p.dnsInterceptState = nil // startup publishes its own state + f.configure(fakeNRPTSeed{probeResults: []bool{true, false}, gpRule: "{GP-RULE}", ctrldRule: true}) + + if err := p.startDNSInterceptLocked(); err != nil { + t.Fatalf("startDNSInterceptLocked() = %v, want nil: ctrld's own route was put back, so startup succeeded", err) + } + if f.restoreLocalCount() != 1 { + t.Errorf("restoreLocalRule calls = %d, want 1: startup must put back only the local key", f.restoreLocalCount()) + } + if add, _, _, _ := f.counts(); add != 0 { + t.Errorf("addRule calls = %d, want 0: the full writer would place ctrld's GP rule beside the administrator's", add) + } + if !f.hasCtrldRule() { + t.Error("ctrld's local rule is missing after startup; the host has no route") + } + if f.hasCtrldGPSibling() { + t.Error("startup left ctrld's rule in the GP store beside the administrator's catch-all") + } + _ = p.stopDNSIntercept() +} + +// TestStartupDropsAnInheritedGPSibling covers the machine that boots already holding the +// forbidden pair: ctrld's GP rule from an earlier run beside an administrator catch-all. +// +// Adoption used to inherit it. Nothing downstream removes it: every handback from here +// stops at the pre-probe while the administrator's rule stays unproven, the monitor calls +// removeRule only inside a handback, and the orphan sweep runs at stop. The second +// catch-all would therefore stand for the life of the process, through the very outage the +// operator is trying to diagnose. +// +// The GP-key-only case is the one that proves the cleanup is not self-defeating. Dropping +// the sibling empties the store of ctrld keys, so startup has to write its rule after all - +// and the full writer would put the sibling straight back, beside the rule it was just +// removed from. +func TestStartupDropsAnInheritedGPSibling(t *testing.T) { + for _, tc := range []struct { + name string + ctrldRule bool + // wantRestoreLocal is whether startup had to write ctrld's rule itself, which it + // may only ever do into the local store while an administrator rule is present. + wantRestoreLocal int + }{ + {name: "local and GP keys", ctrldRule: true, wantRestoreLocal: 0}, + {name: "GP key only", ctrldRule: false, wantRestoreLocal: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + startupInterceptMode(t) + + p, _, f := newHandbackTestProg(t) + p.cfg = &cfg + p.dnsInterceptState = nil + f.configure(fakeNRPTSeed{ + probeResults: []bool{false}, // the pre-probe never answers, so the handback decides nothing + gpRule: "{GP-RULE}", + ctrldRule: tc.ctrldRule, + ctrldGPRule: true, + }) + + err := p.startDNSInterceptLocked() + if f.hasCtrldGPSibling() { + t.Error("startup left ctrld's rule in the GP store beside the administrator's catch-all; nothing later removes it") + } + if f.dropGPCount() != 1 { + t.Errorf("dropGPRule calls = %d, want 1", f.dropGPCount()) + } + if add, restoreLocal := f.writes(); add != 0 || restoreLocal != tc.wantRestoreLocal { + t.Errorf("addRule calls = %d, restoreLocalRule calls = %d, want 0/%d: the full writer re-creates the sibling", + add, restoreLocal, tc.wantRestoreLocal) + } + if !f.hasCtrldRule() { + t.Error("no ctrld rule after startup; the host has no route") + } + if err == nil { + _ = p.stopDNSIntercept() + } + }) + } +} + +// TestHandbackPutBackSettlesToGroupPolicyOwnership pins the registry and ownership state a +// failed handback actually leaves behind, which is what QA will find on the machine. +// +// After the put-back the health monitor keeps running: its handback is throttled, so the +// heal reaches deferToExternalCatchAll undecided and records Group Policy ownership. The +// end state is ctrld's local key on disk, the administrator's rule on disk, no sibling, +// and no further writes - and stop then removes the local key as an orphan. +func TestHandbackPutBackSettlesToGroupPolicyOwnership(t *testing.T) { + p, state, f := newHandbackTestProg(t) + f.configure(fakeNRPTSeed{probeResults: []bool{true, false}, gpRule: "{GP-RULE}", ctrldRule: true}) + state.setNRPTPolicyOwner(nrptRuleOwnerCtrld, "") + + if got := p.nrptHandbackToExternal(state, "{GP-RULE}", "probe finds no route"); got != nrptHandbackKeptCtrld { + t.Fatalf("nrptHandbackToExternal() = %v, want nrptHandbackKeptCtrld", got) + } + _, _, signalBefore, _ := f.counts() + restoreBefore := f.restoreLocalCount() + + p.nrptProbeAndHeal(state) + + if owner, _ := state.nrptPolicyOwner(); owner != nrptRuleOwnerGroupPolicy { + t.Errorf("NRPT owner = %v, want nrptRuleOwnerGroupPolicy: the throttled heal ends undecided and defers", owner) + } + addAfter, _, signalAfter, _ := f.counts() + // Absolute, not "unchanged": the whole sequence must never reach the full writer, + // so a first call that had already happened would be a defect too. + if addAfter != 0 { + t.Errorf("addRule calls = %d, want 0: neither the handback nor the heal may write a sibling", addAfter) + } + if signalAfter != signalBefore { + t.Errorf("signal calls = %d, want %d: nothing changed, so nothing is signalled", signalAfter, signalBefore) + } + if f.restoreLocalCount() != restoreBefore { + t.Errorf("restoreLocalRule calls = %d, want %d: the rule is already back", f.restoreLocalCount(), restoreBefore) + } + if !f.hasCtrldRule() { + t.Error("ctrld's local key is missing; the put-back did not survive the heal") + } + if f.hasCtrldGPSibling() { + t.Error("the heal wrote ctrld's rule into the GP store") + } + + _, removeBefore, _, _ := f.counts() + _ = p.stopDNSIntercept() + if _, removeAfter, _, _ := f.counts(); removeAfter != removeBefore+1 { + t.Errorf("removeRule calls = %d, want %d: stop must sweep the local key it left on disk", removeAfter, removeBefore+1) + } + if f.hasCtrldRule() { + t.Error("stop left ctrld's local key behind") + } +} diff --git a/docs/dns-intercept-mode.md b/docs/dns-intercept-mode.md index e1724a0b..29463def 100644 --- a/docs/dns-intercept-mode.md +++ b/docs/dns-intercept-mode.md @@ -100,9 +100,14 @@ The **Name Resolution Policy Table** is a Windows feature (originally for Direct | `ConfigOptions` | REG_DWORD | `0x8` | Standard DNS resolution (no DirectAccess) | | `Version` | REG_DWORD | `0x2` | NRPT rule version 2 | -**Registry path**: `HKLM\SOFTWARE\Policies\Microsoft\Windows NT\DNSClient\DnsPolicyConfig\CtrldCatchAll` +**Registry path**: the primary key is the local store, written on every start: +`HKLM\SYSTEM\CurrentControlSet\Services\Dnscache\Parameters\DnsPolicyConfig\{B2E9A3C1-7F4D-4A8E-9D6B-5C1E0F3A2B8D}`. +The GP-store key +`HKLM\SOFTWARE\Policies\Microsoft\Windows NT\DNSClient\DnsPolicyConfig\CtrldCatchAll` +is written only when foreign GP rules would otherwise hide the local store, and never when +the foreign rule is an administrator catch-all for this listener. -**Group Policy refresh**: The DNS Client service only reads NRPT from registry during Group Policy processing cycles (default: every 90 minutes). ctrld calls `RefreshPolicyEx(bMachine=TRUE, dwOptions=RP_FORCE)` when activating or repairing rules it owns. While Group Policy remains the owner, ctrld does not run NRPT activation/heal signaling; the one transition that removes a ctrld fallback is signaled after the external rule has been proven. +**Group Policy refresh**: The DNS Client service only reads NRPT from registry during Group Policy processing cycles (default: every 90 minutes). ctrld calls `RefreshPolicyEx(bMachine=TRUE, dwOptions=RP_FORCE)` when activating or repairing rules it owns. While Group Policy remains the owner, ctrld does not run NRPT activation/heal signaling. For the transition that removes a ctrld fallback, ctrld signals once after removing its keys and then probes. If the proof finds no route, ctrld writes back only its local key and signals again. #### GP-managed NRPT ownership @@ -120,7 +125,7 @@ The health monitor keeps using functional probes: - matching GP rule + failed probe: retry loopback WFP protection, then report the external policy as ineffective without running NRPT heal signals; - matching GP rule disappears: create the normal ctrld-owned fallback and verify it, unless another GP catch-all targets a different resolver; - GP catch-all targets another resolver: report the conflict and do not create a second ambiguous catch-all; -- matching GP rule returns: prove it with a probe, remove only ctrld's deterministic fallback keys, and return ownership to Group Policy. +- matching GP rule returns: prove it with a probe, remove only ctrld's deterministic fallback keys, and return ownership to Group Policy. If the proof finds no route while the same rule stays on disk, ctrld puts back only its local-store key, keeps ownership, and does not write its GP-store rule adjacent to the administrator's rule. Deploy the GPO **before** starting or restarting ctrld if adapter DNS must remain completely untouched. Remove or unlink the GP rule before intentionally removing the ctrld service. A GP catch-all that remains pointed at loopback while no listener is running causes DNS failure by design; ctrld cannot safely delete an administrator-owned policy during uninstall. @@ -174,7 +179,7 @@ A dedicated background goroutine (`nrptHealthMonitor`) runs every 30 seconds and 1. **Ownership check:** Distinguish a matching external GP child from ctrld's deterministic local/GP keys. 2. **Active probe:** Verify Windows DNS Client still routes to the listener. -3. **Transition:** If the external child disappears, activate ctrld's normal fallback. If it returns while the fallback is active, prove it before removing only ctrld's keys. +3. **Transition:** If the external child disappears, activate ctrld's normal fallback. If it returns while the fallback is active, prove it before removing only ctrld's keys. If the proof finds no route while the same rule stays on disk, ctrld puts back only its local-store key, keeps ownership, and does not write its GP-store rule adjacent to the administrator's rule. 4. **Owned recovery:** Restore/heal only when ctrld owns the NRPT rule. 5. **(hard mode)** Verify the WFP sublayer exists and fully restart intercept state on loss. @@ -235,7 +240,8 @@ DNS intercept: pf redirect active — all outbound DNS (port 53) redirected to 1 ### Windows ```powershell -# Check NRPT rules (should show CtrldCatchAll with . → 127.0.0.1) +# Check NRPT rules (should show ctrld's catch-all, . → 127.0.0.1; the primary key is the +# local-store GUID rule, and CtrldCatchAll appears only when ctrld also wrote the GP store) Get-DnsClientNrptRule # Check NRPT registry directly diff --git a/docs/wfp-dns-intercept.md b/docs/wfp-dns-intercept.md index 60b54289..2bf7bd2e 100644 --- a/docs/wfp-dns-intercept.md +++ b/docs/wfp-dns-intercept.md @@ -82,7 +82,9 @@ ctrld uses an adaptive strategy (matching [Tailscale's approach](https://github. on all non-domain machines. 2. **Check if other software has GP NRPT rules** (`otherGPRulesExist()`). If foreign GP rules are present (IT policy, VPN), DNS Client is already in GP mode - and our local rule would be invisible — so we also write to the GP path. + and our local rule would be invisible — so we also write to the GP path. ctrld does + not write to the GP path when the foreign rule is an administrator catch-all for this + listener: a second catch-all in that store is never written, whatever the cost. 3. **If no foreign GP rules exist**, clean any stale ctrld GP rules and delete the empty GP parent key. This ensures DNS Client stays in "local mode" where the local-path rule activates immediately via `paramchange`. @@ -100,7 +102,7 @@ When the rule remains unchanged and the probe arrives, Group Policy owns NRPT: - `RefreshPolicyEx`, Dnscache `paramchange`, and cache flush are not used for that policy; - shutdown/uninstall leave the GP child untouched. -A matching GP child that remains present but fails its probe is reported as ineffective and is not rewritten. If the child disappears, ctrld activates its normal owned fallback. A GP catch-all that instead changes to another resolver is reported as a conflict; ctrld does not create a second ambiguous catch-all. If the matching rule later returns, ctrld probes first, removes only its deterministic fallback keys, signals the ownership transition once, and resumes observing Group Policy. +A matching GP child that remains present but fails its probe is reported as ineffective and is not rewritten. If the proof finds no route while the same rule stays on disk, ctrld puts back only its local-store key, keeps ownership, and does not write its GP-store rule adjacent to the administrator's rule. If the child disappears, ctrld activates its normal owned fallback. A GP catch-all that instead changes to another resolver is reported as a conflict; ctrld does not create a second ambiguous catch-all. If the matching rule later returns, ctrld probes first, removes only its deterministic fallback keys, signals the ownership transition once, and resumes observing Group Policy. **Deployment ordering:** apply the GPO before starting ctrld to guarantee adapter DNS is never reset. Remove/unlink it before intentionally stopping or uninstalling ctrld. A GP catch-all still targeting loopback with no listener running is a deliberate fail-closed state and will break DNS. @@ -304,6 +306,8 @@ Every 30s: │ ├─ ctrld-owned owner │ ├─ Working matching GP child returns → remove only ctrld keys; adopt GP + │ │ └─ Probe finds no route, rule still on disk → write back only the + │ │ local-store key, keep ownership, write no GP sibling │ ├─ ctrld key missing → restore + signal + verify │ └─ ctrld key present → probe; run owned heal sequence on failure │ From 8934e09428a1e6b93a14c08d0653d95930d896b7 Mon Sep 17 00:00:00 2001 From: Anthony Wong Date: Fri, 18 Sep 2026 12:54:35 +0000 Subject: [PATCH 09/29] feat(cli): keep a log journal on disk with file headers and an upload limit ctrld keeps a second log file, ctrld-journal.log, next to ctrld.log. The journal holds all warnings and errors and each event with the field journal=true. It stays on disk after a start of the service and after a self-upgrade. Each log file starts with a Log header line. Each log send upload starts with the same line, made at send time, and has a limit of about 16 MB. --- .gitignore | 1 + cmd/cli/cli.go | 3 +- cmd/cli/commands.go | 68 ++- cmd/cli/commands_cleanup_test.go | 71 +++ cmd/cli/control_client.go | 18 +- cmd/cli/control_server.go | 166 ++++--- cmd/cli/control_server_tail_test.go | 104 ++++ cmd/cli/control_server_test.go | 108 +++++ cmd/cli/dns_proxy.go | 49 +- cmd/cli/dns_proxy_sampling_test.go | 124 +++++ cmd/cli/internal_domains_review_test.go | 2 +- cmd/cli/log_budget.go | 45 ++ cmd/cli/log_budget_test.go | 60 +++ cmd/cli/log_file.go | 416 ++++++++++++++++ cmd/cli/log_file_test.go | 553 +++++++++++++++++++++ cmd/cli/log_header.go | 339 +++++++++++++ cmd/cli/log_header_test.go | 617 ++++++++++++++++++++++++ cmd/cli/log_journal.go | 68 +++ cmd/cli/log_journal_test.go | 255 ++++++++++ cmd/cli/log_path.go | 73 +++ cmd/cli/log_path_rotation_test.go | 187 +++++++ cmd/cli/log_reader.go | 291 +++++++++++ cmd/cli/log_reader_test.go | 572 ++++++++++++++++++++++ cmd/cli/log_tail_follow.go | 80 +++ cmd/cli/log_writer.go | 419 +++++++--------- cmd/cli/log_writer_silent_test.go | 79 ++- cmd/cli/log_writer_test.go | 265 ++++++++-- cmd/cli/main.go | 26 +- cmd/cli/network_transition_test.go | 13 +- cmd/cli/prog.go | 16 +- cmd/cli/query_error_sampler.go | 166 +++++++ cmd/cli/query_error_sampler_test.go | 260 ++++++++++ cmd/cli/service_others.go | 5 +- cmd/cli/upstream_monitor.go | 184 +++++-- cmd/cli/upstream_monitor_test.go | 231 +++++++++ config.go | 2 + config_test.go | 66 +++ docs/config.md | 18 + docs/internal-domains.md | 5 +- docs/network-recovery-diagnostics.md | 19 +- docs/runtime-internal-logging.md | 104 ++-- 41 files changed, 5657 insertions(+), 491 deletions(-) create mode 100644 cmd/cli/commands_cleanup_test.go create mode 100644 cmd/cli/control_server_tail_test.go create mode 100644 cmd/cli/dns_proxy_sampling_test.go create mode 100644 cmd/cli/log_budget.go create mode 100644 cmd/cli/log_budget_test.go create mode 100644 cmd/cli/log_file.go create mode 100644 cmd/cli/log_file_test.go create mode 100644 cmd/cli/log_header.go create mode 100644 cmd/cli/log_header_test.go create mode 100644 cmd/cli/log_journal.go create mode 100644 cmd/cli/log_journal_test.go create mode 100644 cmd/cli/log_path.go create mode 100644 cmd/cli/log_path_rotation_test.go create mode 100644 cmd/cli/log_reader.go create mode 100644 cmd/cli/log_reader_test.go create mode 100644 cmd/cli/log_tail_follow.go create mode 100644 cmd/cli/query_error_sampler.go create mode 100644 cmd/cli/query_error_sampler_test.go create mode 100644 cmd/cli/upstream_monitor_test.go diff --git a/.gitignore b/.gitignore index e9121bf7..290ea923 100644 --- a/.gitignore +++ b/.gitignore @@ -18,3 +18,4 @@ ctrld.exe # Local planning artifacts - never commit /SPEC.md /tasks/ +/SPEC-*.md diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 870d5be6..7e86915c 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -49,7 +49,6 @@ const selfCheckInternalTestDomain = "ctrld" + loopTestDomain const ( windowsForwardersFilename = ".forwarders.txt" oldBinSuffix = "_previous" - oldLogSuffix = ".1" msgExit = "$$EXIT$$" ) @@ -1497,6 +1496,8 @@ func fieldErrorMsg(fe validator.FieldError) string { return fmt.Sprintf("minimum len: %q", fe.Param()) case "gte": return fmt.Sprintf("must be greater than or equal to: %s", fe.Param()) + case "lte": + return fmt.Sprintf("must be less than or equal to: %s", fe.Param()) case "cidr": return fmt.Sprintf("invalid value: %s", fe.Value()) case "required_unless", "required": diff --git a/cmd/cli/commands.go b/cmd/cli/commands.go index 9b057b44..5eee98fc 100644 --- a/cmd/cli/commands.go +++ b/cmd/cli/commands.go @@ -36,11 +36,52 @@ import ( // dialSocketControlServerTimeout is the default timeout to wait when ping control server. const dialSocketControlServerTimeout = 30 * time.Second +// logRequestPath adds the query that makes the control server read every log +// file, not only the newest debug bytes. +func logRequestPath(path string, full bool) string { + if !full { + return path + } + return path + "?full=1" +} + +// removeLogFiles deletes the log files of this installation and returns what +// it could not remove. A missing file is not an error. +func removeLogFiles(logPath string, backups int, internalPaths []string) []error { + for _, path := range internalPaths { + pruneNumberedBackups(path, 0) + } + var errs []error + for _, path := range logFilesToRemove(logPath, backups, internalPaths) { + if err := os.Remove(path); err != nil && !os.IsNotExist(err) { + errs = append(errs, fmt.Errorf("remove %s: %w", path, err)) + } + } + return errs +} + +// logFilesToRemove names the log files of this installation. The log_path +// backups come from the configured count, because log_path can name a file in +// a directory that holds the files of other programs, and a scan of that +// directory would take their files too. An empty logPath leaves the internal +// files alone. +func logFilesToRemove(logPath string, backups int, internalPaths []string) []string { + var paths []string + if logPath != "" { + paths = append(paths, logPath) + for index := 1; index <= backups; index++ { + paths = append(paths, fmt.Sprintf("%s.%d", logPath, index)) + } + } + return append(paths, internalPaths...) +} + func initLogCmd() *cobra.Command { warnRuntimeLoggingNotEnabled := func() { mainLog.Load().Warn().Msg("runtime debug logging is not enabled") mainLog.Load().Warn().Msg(`ctrld may be running without "--cd" flag or logging is already enabled`) } + var sendFullLogs bool logSendCmd := &cobra.Command{ Use: "send", Short: "Send runtime debug logs to ControlD", @@ -68,7 +109,7 @@ func initLogCmd() *cobra.Command { mainLog.Load().Fatal().Err(err).Msg("failed to find ctrld home dir") } cc := newControlClient(filepath.Join(dir, ctrldControlUnixSock)) - resp, err := cc.post(sendLogsPath, nil) + resp, err := cc.post(logRequestPath(sendLogsPath, sendFullLogs), nil) if err != nil { mainLog.Load().Fatal().Err(err).Msg("failed to send logs") } @@ -94,6 +135,9 @@ func initLogCmd() *cobra.Command { } }, } + logSendCmd.Flags().BoolVar(&sendFullLogs, "full", false, "Send every log file, not only the newest 10 MB of debug") + + var viewFullLogs bool logViewCmd := &cobra.Command{ Use: "view", Short: "View current runtime debug logs", @@ -121,7 +165,7 @@ func initLogCmd() *cobra.Command { mainLog.Load().Fatal().Err(err).Msg("failed to find ctrld home dir") } cc := newControlClient(filepath.Join(dir, ctrldControlUnixSock)) - resp, err := cc.post(viewLogsPath, nil) + resp, err := cc.post(logRequestPath(viewLogsPath, viewFullLogs), nil) if err != nil { mainLog.Load().Fatal().Err(err).Msg("failed to get logs") } @@ -148,6 +192,8 @@ func initLogCmd() *cobra.Command { fmt.Println(logs.Data) }, } + logViewCmd.Flags().BoolVar(&viewFullLogs, "full", false, "Show every log file, not only the newest 10 MB of debug") + var tailLines int logTailCmd := &cobra.Command{ Use: "tail", @@ -1247,14 +1293,16 @@ NOTE: Uninstalling will set DNS to values provided by DHCP.`, var files []string // Config file. files = append(files, v.ConfigFileUsed()) - // Log file and backup log file. - // For safety, only process if log file path is absolute. - if logFile := normalizeLogFilePath(cfg.Service.LogPath); filepath.IsAbs(logFile) { - files = append(files, logFile) - oldLogFile := logFile + oldLogSuffix - if _, err := os.Stat(oldLogFile); err == nil { - files = append(files, oldLogFile) - } + // Log files. For safety, only remove the log_path chain if + // that path is absolute. + logFile := normalizeLogFilePath(cfg.Service.LogPath) + if !filepath.IsAbs(logFile) { + logFile = "" + } + backups := debugLogBudget(&cfg.Service, router.Name() != "").backups + internalLogs := []string{absHomeDir(logFileName), absHomeDir(journalLogFileName)} + for _, err := range removeLogFiles(logFile, backups, internalLogs) { + mainLog.Load().Warn().Err(err).Msg("failed to remove log file") } // Socket files. if dir, _ := socketDir(); dir != "" { diff --git a/cmd/cli/commands_cleanup_test.go b/cmd/cli/commands_cleanup_test.go new file mode 100644 index 00000000..36b50abb --- /dev/null +++ b/cmd/cli/commands_cleanup_test.go @@ -0,0 +1,71 @@ +package cli + +import ( + "os" + "path/filepath" + "testing" +) + +// writeCleanupTestFile creates a file that a cleanup test can watch. +func writeCleanupTestFile(t *testing.T, path string) string { + t.Helper() + if err := os.WriteFile(path, []byte("line\n"), 0600); err != nil { + t.Fatalf("write %s: %v", path, err) + } + return path +} + +func assertRemoved(t *testing.T, paths []string) { + t.Helper() + for _, path := range paths { + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("%s is still there: %v", path, err) + } + } +} + +func Test_removeLogFiles(t *testing.T) { + dir := t.TempDir() + logPath := writeCleanupTestFile(t, filepath.Join(dir, "syslog")) + chain := []string{ + writeCleanupTestFile(t, filepath.Join(dir, "syslog.1")), + writeCleanupTestFile(t, filepath.Join(dir, "syslog.2")), + } + // A log_path can name a file in a directory of another program, so a + // number above the configured count belongs to that program. + foreign := writeCleanupTestFile(t, filepath.Join(dir, "syslog.5")) + home := t.TempDir() + internal := []string{ + writeCleanupTestFile(t, filepath.Join(home, logFileName)), + writeCleanupTestFile(t, filepath.Join(home, journalLogFileName)), + } + internalBackups := []string{ + writeCleanupTestFile(t, filepath.Join(home, logFileName+".3")), + writeCleanupTestFile(t, filepath.Join(home, journalLogFileName+".1")), + } + + if errs := removeLogFiles(logPath, len(chain), internal); len(errs) > 0 { + t.Fatalf("removeLogFiles: %v", errs) + } + + assertRemoved(t, append([]string{logPath}, chain...)) + assertRemoved(t, append(internal, internalBackups...)) + if _, err := os.Stat(foreign); err != nil { + t.Fatalf("the cleanup removed %s, which sits above the backup count: %v", foreign, err) + } +} + +func Test_removeLogFilesWithoutALogPath(t *testing.T) { + home := t.TempDir() + internal := []string{writeCleanupTestFile(t, filepath.Join(home, logFileName))} + kept := writeCleanupTestFile(t, filepath.Join(home, "other.log")) + + if errs := removeLogFiles("", 4, internal); len(errs) > 0 { + t.Fatalf("removeLogFiles: %v", errs) + } + + assertRemoved(t, internal) + if _, err := os.Stat(kept); err != nil { + t.Fatalf("the cleanup removed %s, which is not a ctrld log: %v", kept, err) + } +} diff --git a/cmd/cli/control_client.go b/cmd/cli/control_client.go index 0e3f90f0..61c8baa8 100644 --- a/cmd/cli/control_client.go +++ b/cmd/cli/control_client.go @@ -5,6 +5,8 @@ import ( "io" "net" "net/http" + "net/url" + "strings" "time" ) @@ -25,13 +27,25 @@ func newControlClient(addr string) *controlClient { } func (c *controlClient) post(path string, data io.Reader) (*http.Response, error) { - // for log/send, set the timeout to 5 minutes - if path == sendLogsPath { + // A log send uploads many megabytes, so it needs more time than the other + // calls. + if isSendLogsPath(path) { c.c.Timeout = time.Minute * 5 } return c.c.Post("http://unix"+path, contentTypeJson, data) } +// isSendLogsPath reports whether path addresses the log send handler. The path +// can carry a query, so the comparison takes the path alone. A path that does +// not parse keeps the prefix rule, because it still reaches that handler. +func isSendLogsPath(path string) bool { + parsed, err := url.Parse(path) + if err != nil { + return strings.HasPrefix(path, sendLogsPath) + } + return parsed.Path == sendLogsPath +} + // postStream sends a POST request with no timeout, suitable for long-lived streaming connections. func (c *controlClient) postStream(path string, data io.Reader) (*http.Response, error) { c.c.Timeout = 0 diff --git a/cmd/cli/control_server.go b/cmd/cli/control_server.go index 56065216..c6ba7f93 100644 --- a/cmd/cli/control_server.go +++ b/cmd/cli/control_server.go @@ -309,34 +309,13 @@ func (p *prog) registerControlServerHandler() { return } })) - p.cs.register(viewLogsPath, http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { - lr, err := p.logReader() - if err != nil { - http.Error(w, err.Error(), http.StatusBadRequest) - return - } - defer lr.r.Close() - if lr.size == 0 { - w.WriteHeader(http.StatusMovedPermanently) - return - } - data, err := io.ReadAll(lr.r) - if err != nil { - http.Error(w, fmt.Sprintf("could not read log: %v", err), http.StatusInternalServerError) - return - } - if err := json.NewEncoder(w).Encode(&logViewResponse{Data: string(data)}); err != nil { - http.Error(w, err.Error(), http.StatusInternalServerError) - http.Error(w, fmt.Sprintf("could not marshal log data: %v", err), http.StatusInternalServerError) - return - } - })) + p.cs.register(viewLogsPath, http.HandlerFunc(p.handleLogView)) p.cs.register(sendLogsPath, http.HandlerFunc(func(w http.ResponseWriter, request *http.Request) { if time.Since(p.internalLogSent) < logWriterSentInterval { w.WriteHeader(http.StatusServiceUnavailable) return } - r, err := p.logReader() + r, err := p.logReader(wantFullLogs(request)) if err != nil { http.Error(w, err.Error(), http.StatusBadRequest) return @@ -430,48 +409,113 @@ func (p *prog) registerControlServerHandler() { } } else { // File-based logging mode: tail the log file. - logFile := normalizeLogFilePath(p.cfg.Service.LogPath) - f, err := os.Open(logFile) - if err != nil { - // Already committed 200, just return. - return - } - defer f.Close() + followLogFile(request.Context(), w, flusher, normalizeLogFilePath(p.cfg.Service.LogPath), numLines) + } + })) +} - // Seek to show last N lines. - if numLines > 0 { - if tail := tailFileLastLines(f, numLines); len(tail) > 0 { - w.Write(tail) - flusher.Flush() - } - } else { - // Seek to end. - f.Seek(0, io.SeekEnd) - } +// jsonStringChunkSize bounds the memory that one log view answer needs. A log +// of tens of megabytes must not get a second copy in the answer. +const jsonStringChunkSize = 32 * 1024 - // Poll for new data. - buf := make([]byte, 4096) - ticker := time.NewTicker(200 * time.Millisecond) - defer ticker.Stop() - for { - select { - case <-ticker.C: - n, err := f.Read(buf) - if n > 0 { - if _, werr := w.Write(buf[:n]); werr != nil { - return - } - flusher.Flush() - } - if err != nil && err != io.EOF { - return - } - case <-request.Context().Done(): - return - } +// wantFullLogs reports whether the request asks for every log file instead of +// the newest debug bytes. +func wantFullLogs(r *http.Request) bool { + return r.URL.Query().Get("full") == "1" +} + +// handleLogView answers with the log bytes in the data field. It writes the +// body while it reads the log, so the answer needs no copy of the whole log. +func (p *prog) handleLogView(w http.ResponseWriter, r *http.Request) { + lr, err := p.logReader(wantFullLogs(r)) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + defer lr.r.Close() + if lr.size == 0 { + w.WriteHeader(http.StatusMovedPermanently) + return + } + w.Header().Set("Content-Type", contentTypeJson) + if _, err := io.WriteString(w, `{"data":`); err != nil { + return + } + if err := writeJSONString(w, lr.r); err != nil { + // The answer is on its way, so it can only end short. This line tells + // the operator why. + mainLog.Load().Error().Err(err).Msg("could not send log view answer") + return + } + _, _ = io.WriteString(w, `}`) +} + +// writeJSONString writes what r yields as one JSON string, in chunks. +func writeJSONString(w io.Writer, r io.Reader) error { + if _, err := io.WriteString(w, `"`); err != nil { + return err + } + chunk := make([]byte, jsonStringChunkSize) + for { + n, readErr := r.Read(chunk) + if n > 0 { + if err := writeJSONStringChunk(w, chunk[:n]); err != nil { + return err } } - })) + if readErr == io.EOF { + break + } + if readErr != nil { + return readErr + } + } + _, err := io.WriteString(w, `"`) + return err +} + +// writeJSONStringChunk writes one chunk and escapes the bytes that a JSON +// string cannot carry. It writes the bytes between two escapes in one call. +func writeJSONStringChunk(w io.Writer, chunk []byte) error { + start := 0 + for i, b := range chunk { + escaped := jsonStringEscape(b) + if escaped == "" { + continue + } + if _, err := w.Write(chunk[start:i]); err != nil { + return err + } + if _, err := io.WriteString(w, escaped); err != nil { + return err + } + start = i + 1 + } + _, err := w.Write(chunk[start:]) + return err +} + +// jsonStringEscape returns the escape of one byte, or an empty string for a +// byte that a JSON string carries as it is. Every byte of a multi-byte +// character is above 0x1f, so a chunk that ends inside a character still +// writes the character whole. +func jsonStringEscape(b byte) string { + switch b { + case '"': + return `\"` + case '\\': + return `\\` + case '\n': + return `\n` + case '\r': + return `\r` + case '\t': + return `\t` + } + if b < 0x20 { + return fmt.Sprintf(`\u%04x`, b) + } + return "" } // tailFileLastLines reads the last n lines from a file and returns them. diff --git a/cmd/cli/control_server_tail_test.go b/cmd/cli/control_server_tail_test.go new file mode 100644 index 00000000..ec678e50 --- /dev/null +++ b/cmd/cli/control_server_tail_test.go @@ -0,0 +1,104 @@ +package cli + +import ( + "context" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// streamRecorder collects a streamed response under a lock, because the +// handler writes from its own goroutine while the test reads. +type streamRecorder struct { + mu sync.Mutex + header http.Header + body strings.Builder +} + +func (r *streamRecorder) Header() http.Header { return r.header } + +func (r *streamRecorder) Write(p []byte) (int, error) { + r.mu.Lock() + defer r.mu.Unlock() + return r.body.Write(p) +} + +func (r *streamRecorder) WriteHeader(int) {} + +func (r *streamRecorder) Flush() {} + +func (r *streamRecorder) String() string { + r.mu.Lock() + defer r.mu.Unlock() + return r.body.String() +} + +// waitForText polls the recorder until want arrives or the deadline passes. +func waitForText(rec *streamRecorder, want string, timeout time.Duration) bool { + deadline := time.Now().Add(timeout) + for time.Now().Before(deadline) { + if strings.Contains(rec.String(), want) { + return true + } + time.Sleep(20 * time.Millisecond) + } + return false +} + +// Test_logTailFollowsTheRotatedLogPath keeps one tail request open while the +// log_path file rotates. The lines after the rotation must reach the client. +func Test_logTailFollowsTheRotatedLogPath(t *testing.T) { + origUID := cdUID + cdUID = "" + t.Cleanup(func() { cdUID = origUID }) + + path := filepath.Join(t.TempDir(), "ctrld.log") + rf, err := newRotatingFile(path, logBudget{maxSize: 64, backups: 1}, nil) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + if _, err := rf.Write([]byte("before rotation\n")); err != nil { + t.Fatal(err) + } + + p := &prog{ + cfg: &ctrld.Config{Service: ctrld.ServiceConfig{LogPath: path}}, + cs: &controlServer{mux: http.NewServeMux()}, + } + p.registerControlServerHandler() + + ctx, cancel := context.WithCancel(context.Background()) + req := httptest.NewRequest(http.MethodGet, tailLogsPath+"?lines=1", nil).WithContext(ctx) + rec := &streamRecorder{header: http.Header{}} + done := make(chan struct{}) + go func() { + defer close(done) + p.cs.mux.ServeHTTP(rec, req) + }() + t.Cleanup(func() { + cancel() + <-done + }) + + if !waitForText(rec, "before rotation", 2*time.Second) { + t.Fatalf("the first line did not arrive: %q", rec.String()) + } + for i := 0; i < 4; i++ { + if _, err := rf.Write([]byte("filler line that crosses the budget\n")); err != nil { + t.Fatal(err) + } + } + if _, err := rf.Write([]byte("after rotation sentinel\n")); err != nil { + t.Fatal(err) + } + if !waitForText(rec, "after rotation sentinel", 3*time.Second) { + t.Fatalf("the sentinel did not arrive after the rotation: %q", rec.String()) + } +} diff --git a/cmd/cli/control_server_test.go b/cmd/cli/control_server_test.go index 297b37dd..bdafe7b3 100644 --- a/cmd/cli/control_server_test.go +++ b/cmd/cli/control_server_test.go @@ -2,10 +2,14 @@ package cli import ( "bytes" + "encoding/json" "io" "net/http" "os" + "strings" "testing" + + "github.com/Control-D-Inc/ctrld" ) func TestControlServer(t *testing.T) { @@ -52,3 +56,107 @@ func TestControlServer(t *testing.T) { t.Fatal(err) } } + +// jsonStringTestPayload holds every byte class that a JSON string must escape +// and splits a multi-byte character across the chunk boundary, so a chunk that +// ends inside a character is covered. +func jsonStringTestPayload() []byte { + var payload bytes.Buffer + payload.WriteString("quotes \" backslash \\ tab\t return\r newline\n") + payload.Write([]byte{0x00, 0x01, 0x1f}) + for payload.Len() < jsonStringChunkSize-1 { + payload.WriteByte('x') + } + payload.WriteString("é日本語 and an emoji 🙂\n") + payload.WriteString("the tail after the boundary\n") + return payload.Bytes() +} + +func Test_writeJSONString(t *testing.T) { + payload := jsonStringTestPayload() + var written bytes.Buffer + + if err := writeJSONString(&written, bytes.NewReader(payload)); err != nil { + t.Fatalf("writeJSONString: %v", err) + } + + var got string + if err := json.Unmarshal(written.Bytes(), &got); err != nil { + t.Fatalf("unmarshal %d bytes: %v", written.Len(), err) + } + if got != string(payload) { + t.Fatalf("the round trip returned %d bytes, want the %d bytes of the payload", len(got), len(payload)) + } +} + +// newLogViewTestServer serves the log view handler of p on a temporary socket +// and returns a client of it. +func newLogViewTestServer(t *testing.T, p *prog) *controlClient { + t.Helper() + f, err := os.CreateTemp("", "") + if err != nil { + t.Fatalf("temporary socket: %v", err) + } + f.Close() + t.Cleanup(func() { os.Remove(f.Name()) }) + s, err := newControlServer(f.Name()) + if err != nil { + t.Fatalf("newControlServer: %v", err) + } + s.register(viewLogsPath, http.HandlerFunc(p.handleLogView)) + if err := s.start(); err != nil { + t.Fatalf("start: %v", err) + } + t.Cleanup(func() { _ = s.stop() }) + return newControlClient(f.Name()) +} + +func Test_logViewHandler(t *testing.T) { + t.Run("answers the log as one JSON string", func(t *testing.T) { + p := setupMemoryLogReaderTest(t) + line := "a line with \"quotes\", a tab\there and 日本語\n" + if _, err := p.internalLogWriter.Write([]byte(line)); err != nil { + t.Fatalf("write the debug buffer: %v", err) + } + + resp, err := newLogViewTestServer(t, p).post(viewLogsPath, nil) + + if err != nil { + t.Fatalf("post: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusOK) + } + if ct := resp.Header.Get("content-type"); ct != contentTypeJson { + t.Fatalf("content type = %s, want %s", ct, contentTypeJson) + } + var view logViewResponse + if err := json.NewDecoder(resp.Body).Decode(&view); err != nil { + t.Fatalf("decode the answer: %v", err) + } + if !strings.Contains(view.Data, line) { + t.Fatalf("the answer holds %.64q, want the line %q in it", view.Data, line) + } + if !strings.Contains(view.Data, logWriterLogEndMarker) { + t.Fatalf("the answer holds no log end marker: %.64q", view.Data) + } + }) + + t.Run("answers 301 without a log", func(t *testing.T) { + origSilent, origCdUID := silent, cdUID + t.Cleanup(func() { silent, cdUID = origSilent, origCdUID }) + silent, cdUID = false, "" + p := &prog{cfg: &ctrld.Config{}} + + resp, err := newLogViewTestServer(t, p).post(viewLogsPath, nil) + + if err != nil { + t.Fatalf("post: %v", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusMovedPermanently { + t.Fatalf("status = %d, want %d", resp.StatusCode, http.StatusMovedPermanently) + } + }) +} diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index dc8b844c..3be5cfc5 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -270,7 +270,7 @@ func (p *prog) serveDNS(listenerNum string) error { p.forceFetchingAPI(domain) }() if err := w.WriteMsg(answer); err != nil { - ctrld.Log(ctx, mainLog.Load().Error().Err(err), "serveDNS: failed to send DNS response to client") + ctrld.Log(ctx, p.querySampler.event(sampleClassSendResponseFailed, "").Err(err), "serveDNS: failed to send DNS response to client") } }) @@ -842,26 +842,25 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { ctx = context.WithValue(ctx, ctrld.ClientInfoCtxKey{}, req.ci) } answer, err := resolve1(upstream, upstreamConfig, msg) - // if we have an answer, we should reset the failure count - // we dont use reset here since we dont want to prevent failure counts from being incremented + + // reset is not used here, because it also stops the failure count for + // one second, and the failures already in flight must still count. if answer != nil { - p.um.mu.Lock() - p.um.failureReq[upstream] = 0 - p.um.down[upstream] = false - p.um.mu.Unlock() + p.um.noteSuccess(upstream) return answer } + // A failing minute must not push the state events out of the retained log. // A transport error names the endpoint it failed to reach, which for a // generated Internal Domain upstream is the organization's private - // resolver address. Report the classification at error level and keep - // the address-bearing error at debug. + // resolver address. Report the classification through the sampler and + // keep the address-bearing error at debug. if isInternalDomainUpstream(upstream) { ctrld.Log(ctx, mainLog.Load().Debug().Err(err), "failed to resolve query") - ctrld.Log(ctx, mainLog.Load().Error().Str("failure", internalDomainFailureReason(err)), + ctrld.Log(ctx, p.querySampler.event(sampleClassResolveFailed, upstream).Str("failure", internalDomainFailureReason(err)), "failed to resolve query using an Internal Domain resolver") } else { - ctrld.Log(ctx, mainLog.Load().Error().Err(err), "failed to resolve query") + ctrld.Log(ctx, p.querySampler.event(sampleClassResolveFailed, upstream).Err(err), "failed to resolve query") } // increase failure count when there is no answer @@ -973,7 +972,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { res.upstream = upstreamConfig.Endpoint return res } - ctrld.Log(ctx, mainLog.Load().Error(), "all %v endpoints failed", upstreams) + ctrld.Log(ctx, p.querySampler.event(sampleClassAllEndpointsFailed, ""), "all %v endpoints failed", upstreams) // An Internal Domain with explicit resolvers is served only by the // configured resolvers. When all of them are unreachable the query fails @@ -1713,6 +1712,7 @@ func (p *prog) handleNetworkChange(delta *netmon.ChangeDelta, isMajorChange bool validateDefaultLocalIPsFromDelta(sourceState, transitionID) after4, after6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() p.networkSourceMu.Unlock() + p.noteNetworkState(currentState) p.handleDNS64NetworkChange(delta, isMajorChange) validIfaces := networkChangeValidInterfacesFn() @@ -2055,9 +2055,26 @@ func interfaceIPsEqual(a, b []netip.Prefix) bool { var errOsHealthcheckSuppressed = errors.New("upstream os health check suppressed") +// upstreamFailureLog keeps one error line for each upstream of one recovery +// pass. The pass retries every two seconds until the upstream answers, and the +// first line already names the fault, so the failures that follow it go to +// debug. +type upstreamFailureLog struct{ reported bool } + +// report logs one failed check of upstream. +func (l *upstreamFailureLog) report(upstream string, uc *ctrld.UpstreamConfig, err error, duration time.Duration) { + level := mainLog.Load().Error + if l.reported { + level = mainLog.Load().Debug + } + l.reported = true + logUpstreamProbeFailure(isGeneratedInternalDomainUpstreamRef(upstream, uc), uc, err, level, + "Upstream %s check failed after %v", upstream, duration) +} + // checkUpstreamOnce sends a test query to the specified upstream. // Returns nil if the upstream responds successfully. -func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig) error { +func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig, failures *upstreamFailureLog) error { mainLog.Load().Debug().Msgf("Starting check for upstream: %s", upstream) resolver, err := ctrld.NewResolver(uc) @@ -2102,8 +2119,7 @@ func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig) erro mainLog.Load().Debug().Err(err).Msgf("Upstream %s check failed after %v (network unreachable)", upstream, duration) return err } - logUpstreamProbeFailure(isGeneratedInternalDomainUpstreamRef(upstream, uc), uc, err, - mainLog.Load().Error, "Upstream %s check failed after %v", upstream, duration) + failures.report(upstream, uc, err, duration) return err } mainLog.Load().Debug().Msgf("Upstream %s responded successfully in %v", upstream, duration) @@ -2339,6 +2355,7 @@ func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string mainLog.Load().Debug().Msgf("Starting recovery check loop for upstream: %s", name) attempts := 0 unreachableStreak := 0 + var failures upstreamFailureLog for { select { case <-recoveryCtx.Done(): @@ -2347,7 +2364,7 @@ func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string default: attempts++ // checkUpstreamOnce will reset any failure counters on success. - err := p.checkUpstreamOnce(name, uc) + err := p.checkUpstreamOnce(name, uc, &failures) if err == nil || errors.Is(err, errOsHealthcheckSuppressed) { mainLog.Load().Debug().Msgf("Upstream %s recovered successfully", name) select { diff --git a/cmd/cli/dns_proxy_sampling_test.go b/cmd/cli/dns_proxy_sampling_test.go new file mode 100644 index 00000000..00174460 --- /dev/null +++ b/cmd/cli/dns_proxy_sampling_test.go @@ -0,0 +1,124 @@ +package cli + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/miekg/dns" + + "github.com/Control-D-Inc/ctrld" +) + +// deadUpstreamEndpoint is a port that nothing listens on, so every query to it +// fails at once. +const deadUpstreamEndpoint = "127.0.0.1:1" + +// countSamplingLines counts the captured lines of one level whose message +// starts with the given text. A request id rides in front of the message. +func countSamplingLines(t *testing.T, logs *syncBuffer, level, message string) int { + t.Helper() + count := 0 + for _, event := range jsonLogEvents(t, logs, "") { + text, _ := event["message"].(string) + if !strings.HasPrefix(text, message) || event["level"] != level { + continue + } + count++ + } + return count +} + +func Test_querySamplingBoundsPerQueryErrors(t *testing.T) { + logs := captureDebugMainLog(t) + p := &prog{} + // The zero value sampler arms a real timer. Close its window after the + // assertions, so no late summary line reaches another test. + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + for i := 0; i < querySampleLimit+1; i++ { + ctrld.Log(context.Background(), p.querySampler.event(sampleClassResolveFailed, "upstream.0"), samplerTestMessage) + } + + if got := countSamplingLines(t, logs, "error", samplerTestMessage); got != querySampleLimit { + t.Fatalf("error lines: got %d, want %d", got, querySampleLimit) + } + if got := countSamplingLines(t, logs, "debug", samplerTestMessage); got != 1 { + t.Fatalf("debug lines: got %d, want 1", got) + } +} + +// askThroughProxy sends one query through the real proxy path and returns the +// answer. +func askThroughProxy(t *testing.T, p *prog, ctx context.Context) *dns.Msg { + t.Helper() + msg := new(dns.Msg) + msg.SetQuestion("sampling.test.", dns.TypeA) + res := p.proxy(ctx, &proxyRequest{msg: msg, ufr: &upstreamForResult{srcAddr: "192.168.0.1:1234"}}) + if res == nil || res.answer == nil { + t.Fatal("the proxy returned no answer") + } + return res.answer +} + +// Test_querySamplingOnTheQueryPath drives the real query path with an upstream +// that answers nothing: the flood keeps its request id, the sampler bounds it, +// and one state event brackets each end of the outage. +func Test_querySamplingOnTheQueryPath(t *testing.T) { + logs := captureDebugMainLog(t) + cfg := &ctrld.Config{} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg)} + // A new monitor holds back the failure counts for one second. The test + // clears the flag instead of waiting for that timer. + p.um.clearRecovered(upstreamOS) + var armed []func() + p.um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + // The zero value sampler arms a real timer. Close its window after the + // assertions, so no late summary line reaches another test. + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + origOSUpstream := osUpstreamConfig + t.Cleanup(func() { osUpstreamConfig = origOSUpstream }) + osUpstreamConfig = &ctrld.UpstreamConfig{ + Name: "dead resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 500, + } + + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, "r1") + for i := 0; i < querySampleLimit+2; i++ { + if rcode := askThroughProxy(t, p, ctx).Rcode; rcode != dns.RcodeServerFailure { + t.Fatalf("query %d: rcode %s, want SERVFAIL", i+1, dns.RcodeToString[rcode]) + } + } + + const failedMessage = "[r1] " + samplerTestMessage + if got := countSamplingLines(t, logs, "error", failedMessage); got != querySampleLimit { + t.Fatalf("error lines: got %d, want %d", got, querySampleLimit) + } + if got := countSamplingLines(t, logs, "debug", failedMessage); got != 2 { + t.Fatalf("debug lines: got %d, want 2", got) + } + + if len(armed) != 1 { + t.Fatalf("armed timers: got %d, want 1", len(armed)) + } + armed[0]() + startOSResolverStub(t) + if rcode := askThroughProxy(t, p, ctx).Rcode; rcode != dns.RcodeSuccess { + t.Fatalf("the answered query returned %s, want NOERROR", dns.RcodeToString[rcode]) + } + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 2 { + t.Fatalf("state events: got %d, want 2", len(events)) + } + wantField(t, events[0], "upstream", upstreamOS) + wantField(t, events[0], "state", "down") + wantField(t, events[0], "reason", "timer") + wantField(t, events[1], "upstream", upstreamOS) + wantField(t, events[1], "state", "up") +} diff --git a/cmd/cli/internal_domains_review_test.go b/cmd/cli/internal_domains_review_test.go index f26e9cb9..d2ece04f 100644 --- a/cmd/cli/internal_domains_review_test.go +++ b/cmd/cli/internal_domains_review_test.go @@ -592,7 +592,7 @@ func TestInternalDomainsRecoveryCheckHidesResolverAddress(t *testing.T) { } mark := len(logOutput.String()) - if err := p.checkUpstreamOnce("upstream.internal_0", uc); err == nil { + if err := p.checkUpstreamOnce("upstream.internal_0", uc, &upstreamFailureLog{}); err == nil { t.Fatal("expected the probe to fail") } above := logLinesAboveDebug(mark) diff --git a/cmd/cli/log_budget.go b/cmd/cli/log_budget.go new file mode 100644 index 00000000..19c0405f --- /dev/null +++ b/cmd/cli/log_budget.go @@ -0,0 +1,45 @@ +package cli + +import ( + "github.com/Control-D-Inc/ctrld" +) + +// journalLogFileName is the journal file next to the debug log in the ctrld +// home directory. +const journalLogFileName = "ctrld-journal.log" + +// logBudgets splits the disk space by platform. A router keeps the 10 MB total +// that QA measured on an EdgeRouter-X, because its flash is small. A desktop +// holds more, so a log covers more days. +func logBudgets(isRouter bool) (debug, journal logBudget) { + if isRouter { + return logBudget{maxSize: 5 << 20, backups: 1}, logBudget{maxSize: 1 << 20, backups: 1} + } + return logBudget{maxSize: 10 << 20, backups: 4}, logBudget{maxSize: 2 << 20, backups: 2} +} + +const ( + // logMaxSizeMBLimit and logMaxBackupsLimit bound the two keys. The log + // file opens before ctrld validates its configuration, so a key outside + // these bounds reaches this code: a size near the largest integer never + // rotates, and a count of one million backups renames one million files. + // The validation rules are the first line of defense, and these bounds + // are the second. + logMaxSizeMBLimit = 1024 + logMaxBackupsLimit = 64 +) + +// debugLogBudget starts from the platform default, so a router keeps its small +// total until an operator asks for more. The two keys size the debug stream +// only. The journal budget stays fixed. A key outside its bounds keeps the +// default. +func debugLogBudget(svc *ctrld.ServiceConfig, isRouter bool) logBudget { + budget, _ := logBudgets(isRouter) + if mb := svc.LogMaxSizeMB; mb >= 1 && mb <= logMaxSizeMBLimit { + budget.maxSize = int64(mb) << 20 + } + if count := svc.LogMaxBackups; count != nil && *count >= 0 && *count <= logMaxBackupsLimit { + budget.backups = *count + } + return budget +} diff --git a/cmd/cli/log_budget_test.go b/cmd/cli/log_budget_test.go new file mode 100644 index 00000000..e278ff3e --- /dev/null +++ b/cmd/cli/log_budget_test.go @@ -0,0 +1,60 @@ +package cli + +import ( + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +func Test_logBudgets(t *testing.T) { + for _, tc := range []struct { + name string + isRouter bool + wantDebug logBudget + wantJournal logBudget + }{ + {"desktop", false, logBudget{maxSize: 10 << 20, backups: 4}, logBudget{maxSize: 2 << 20, backups: 2}}, + {"router", true, logBudget{maxSize: 5 << 20, backups: 1}, logBudget{maxSize: 1 << 20, backups: 1}}, + } { + t.Run(tc.name, func(t *testing.T) { + debug, journalStream := logBudgets(tc.isRouter) + if debug != tc.wantDebug { + t.Errorf("debug budget = %+v, want %+v", debug, tc.wantDebug) + } + if journalStream != tc.wantJournal { + t.Errorf("journal budget = %+v, want %+v", journalStream, tc.wantJournal) + } + }) + } +} + +func Test_debugLogBudget(t *testing.T) { + noBackup, oneBackup := 0, 1 + negativeBackups, tooManyBackups, mostBackups := -1, logMaxBackupsLimit+1, logMaxBackupsLimit + for _, tc := range []struct { + name string + svc ctrld.ServiceConfig + isRouter bool + want logBudget + }{ + {"both keys set", ctrld.ServiceConfig{LogMaxSizeMB: 20, LogMaxBackups: &oneBackup}, false, logBudget{maxSize: 20 << 20, backups: 1}}, + {"size zero keeps the default size", ctrld.ServiceConfig{LogMaxBackups: &oneBackup}, false, logBudget{maxSize: 10 << 20, backups: 1}}, + {"absent backups keep the default count", ctrld.ServiceConfig{LogMaxSizeMB: 20}, false, logBudget{maxSize: 20 << 20, backups: 4}}, + {"zero backups keep no backup file", ctrld.ServiceConfig{LogMaxBackups: &noBackup}, false, logBudget{maxSize: 10 << 20, backups: 0}}, + {"empty config on a desktop", ctrld.ServiceConfig{}, false, logBudget{maxSize: 10 << 20, backups: 4}}, + {"empty config on a router", ctrld.ServiceConfig{}, true, logBudget{maxSize: 5 << 20, backups: 1}}, + {"keys win on a router", ctrld.ServiceConfig{LogMaxSizeMB: 20, LogMaxBackups: &noBackup}, true, logBudget{maxSize: 20 << 20, backups: 0}}, + {"negative size keeps the default size", ctrld.ServiceConfig{LogMaxSizeMB: -1}, false, logBudget{maxSize: 10 << 20, backups: 4}}, + {"size above the limit keeps the default size", ctrld.ServiceConfig{LogMaxSizeMB: logMaxSizeMBLimit + 1}, false, logBudget{maxSize: 10 << 20, backups: 4}}, + {"the largest size the key takes", ctrld.ServiceConfig{LogMaxSizeMB: logMaxSizeMBLimit}, false, logBudget{maxSize: logMaxSizeMBLimit << 20, backups: 4}}, + {"negative backups keep the default count", ctrld.ServiceConfig{LogMaxBackups: &negativeBackups}, false, logBudget{maxSize: 10 << 20, backups: 4}}, + {"backups above the limit keep the default count", ctrld.ServiceConfig{LogMaxBackups: &tooManyBackups}, false, logBudget{maxSize: 10 << 20, backups: 4}}, + {"the largest backup count the key takes", ctrld.ServiceConfig{LogMaxBackups: &mostBackups}, false, logBudget{maxSize: 10 << 20, backups: logMaxBackupsLimit}}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := debugLogBudget(&tc.svc, tc.isRouter); got != tc.want { + t.Errorf("debug budget = %+v, want %+v", got, tc.want) + } + }) + } +} diff --git a/cmd/cli/log_file.go b/cmd/cli/log_file.go new file mode 100644 index 00000000..dee26e74 --- /dev/null +++ b/cmd/cli/log_file.go @@ -0,0 +1,416 @@ +package cli + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" +) + +// errLogFileNotOpen tells a caller that the file is closed, so a write that +// returns no bytes never looks like a success. +var errLogFileNotOpen = errors.New("log file is not open") + +// retryOpenInterval is the wait between two attempts to open a log file that +// could not open. A file error is often short-lived, and one attempt for each +// line would cost a system call for every line. +var retryOpenInterval = time.Minute + +// logBudget is the disk space of one log stream: the size of the current +// file and the number of numbered backups that follow it. +type logBudget struct { + maxSize int64 + backups int +} + +// logRotation describes the file that a rotation moved away. A set err means +// that the file could not move, so the stream continues in the same file. +type logRotation struct { + file string + bytesWritten int64 + firstWrite time.Time + lastWrite time.Time + backups int + err error +} + +// rotatingFile writes one log file and rotates it by size with numbered +// backups. It holds the header bytes, so every file it opens starts with +// the header, and it records the write times that the rotation event needs. +type rotatingFile struct { + mu sync.Mutex + path string + budget logBudget + open func(string, int) (*os.File, error) + rename func(oldpath, newpath string) error + f *os.File + size int64 + firstWrite time.Time + lastWrite time.Time + header []byte + openErr error + onRotate func(logRotation) + // headerWritten tells whether the file that is open now already holds the + // header. A start wires its logging more than once, and each file takes + // one header. + headerWritten bool + // rotateFailedAt is the size at which a rotation could not move the file. + // The next try waits for another budget of data, so a directory that + // refuses the move does not cost one attempt per line. + rotateFailedAt int64 + // retryOpenAt is the time from which a closed file may open again. + retryOpenAt time.Time + // rotatedInFlight keeps onRotate from reporting a rotation that the + // rotation event itself caused. + rotatedInFlight atomic.Bool +} + +// newRotatingFile opens path in append mode and counts what it already holds, +// so a restart continues the current file instead of starting a new one. +func newRotatingFile(path string, budget logBudget, open func(string, int) (*os.File, error)) (*rotatingFile, error) { + if err := os.MkdirAll(filepath.Dir(path), 0750); err != nil { + return nil, fmt.Errorf("creating log directory: %w", err) + } + rf := &rotatingFile{path: path, budget: budget, open: open} + f, err := rf.openFile(os.O_CREATE | os.O_RDWR | os.O_APPEND) + if err != nil { + return nil, fmt.Errorf("opening log file: %w", err) + } + st, err := f.Stat() + if err != nil { + f.Close() + return nil, fmt.Errorf("stat log file: %w", err) + } + rf.f = f + rf.size = st.Size() + return rf, nil +} + +// openFile opens the file through the platform helper, which holds the flags +// that each platform needs: a share mode on Windows, and a refusal to follow +// a symlink elsewhere. +func (rf *rotatingFile) openFile(flags int) (*os.File, error) { + if rf.open == nil { + return openLogFile(rf.path, flags) + } + return rf.open(rf.path, flags) +} + +func (rf *rotatingFile) renameFile(oldpath, newpath string) error { + if rf.rename == nil { + return os.Rename(oldpath, newpath) + } + return rf.rename(oldpath, newpath) +} + +// writeReporting appends p and reports a rotation that happened first. It +// never calls onRotate, because the caller must log the rotation outside of +// the lock. +func (rf *rotatingFile) writeReporting(p []byte) (int, *logRotation, error) { + rf.mu.Lock() + defer rf.mu.Unlock() + + var rotated *logRotation + if rf.needsRotation(len(p)) { + var err error + if rotated, err = rf.rotateLocked(); err != nil { + return 0, rotated, err + } + } + n, err := rf.writeLocked(p) + if err != nil { + return n, rotated, err + } + rf.noteEventWrite() + return n, rotated, nil +} + +// Write makes the file an io.Writer for the logger. It reports the rotation +// after the lock is free, so the rotation event can go back into this file. +func (rf *rotatingFile) Write(p []byte) (int, error) { + n, rotated, err := rf.writeReporting(p) + if rotated == nil || !rf.rotatedInFlight.CompareAndSwap(false, true) { + return n, err + } + defer rf.rotatedInFlight.Store(false) + rf.mu.Lock() + onRotate := rf.onRotate + rf.mu.Unlock() + if onRotate != nil { + onRotate(*rotated) + } + return n, err +} + +// setHeader stores a copy of the header bytes for the next file. +func (rf *rotatingFile) setHeader(b []byte) { + header := make([]byte, len(b)) + copy(header, b) + rf.mu.Lock() + defer rf.mu.Unlock() + rf.header = header +} + +// writeHeader appends the stored header to the current file, once for each +// file that the writer opens. A restart uses it to mark the point where the +// new run continues an old file. +func (rf *rotatingFile) writeHeader() error { + rf.mu.Lock() + defer rf.mu.Unlock() + if rf.headerWritten { + return nil + } + return rf.writeHeaderLocked() +} + +// setBudget takes the disk space of a later config. The file stays open, so a +// changed key costs no second header. +func (rf *rotatingFile) setBudget(budget logBudget) { + rf.mu.Lock() + defer rf.mu.Unlock() + rf.budget = budget +} + +// rotateNow rotates the file although the budget is not spent. +func (rf *rotatingFile) rotateNow() (*logRotation, error) { + rf.mu.Lock() + defer rf.mu.Unlock() + return rf.rotateLocked() +} + +// paths returns the files that exist, oldest first, for the log readers. +func (rf *rotatingFile) paths() []string { + rf.mu.Lock() + defer rf.mu.Unlock() + files := make([]string, 0, rf.budget.backups+1) + for i := rf.budget.backups; i >= 1; i-- { + backup := rf.backupPath(i) + if _, err := os.Stat(backup); err == nil { + files = append(files, backup) + } + } + if _, err := os.Stat(rf.path); err == nil { + files = append(files, rf.path) + } + return files +} + +func (rf *rotatingFile) currentPath() string { + return rf.path +} + +func (rf *rotatingFile) close() error { + rf.mu.Lock() + defer rf.mu.Unlock() + if rf.f == nil { + return nil + } + err := rf.f.Close() + rf.f = nil + return err +} + +func (rf *rotatingFile) backupPath(index int) string { + return fmt.Sprintf("%s.%d", rf.path, index) +} + +// rotatingSuffix marks the file that a rotation with no backup moves away +// before it opens the new file. The old bytes stay behind the readers that +// hold them open, and the file goes as soon as the new file is open. +const rotatingSuffix = ".rotating" + +func (rf *rotatingFile) rotatingPath() string { + return rf.path + rotatingSuffix +} + +// rotatedPath names the file that the next rotation leaves behind. +func (rf *rotatingFile) rotatedPath() string { + if rf.budget.backups == 0 { + return rf.path + } + return rf.backupPath(1) +} + +// needsRotation reports whether p makes the file cross its budget. A closed +// file never rotates, because a rotation would shift the backups away for a +// write that cannot land. +func (rf *rotatingFile) needsRotation(n int) bool { + if rf.f == nil || rf.budget.maxSize <= 0 { + return false + } + return rf.size+int64(n) > rf.rotateFailedAt+rf.budget.maxSize +} + +// rotateLocked moves the current file away and opens an empty one in its +// place. It must run with rf.mu held. +func (rf *rotatingFile) rotateLocked() (*logRotation, error) { + record := &logRotation{ + file: rf.rotatedPath(), + bytesWritten: rf.size, + firstWrite: rf.firstWrite, + lastWrite: rf.lastWrite, + backups: rf.budget.backups, + } + if rf.f != nil { + rf.f.Close() + rf.f = nil + } + if err := rf.shiftBackups(); err != nil { + record.err = err + return record, rf.keepFileAfterFailedShift() + } + + f, err := rf.openFile(os.O_CREATE | os.O_RDWR | os.O_TRUNC) + if err != nil { + rf.size = 0 + return record, rf.noteOpenFailure(err) + } + if rf.budget.backups == 0 { + // The readers that hold the old file keep their bytes; the name goes. + _ = os.Remove(rf.rotatingPath()) + } + rf.f = f + rf.openErr = nil + rf.size = 0 + rf.rotateFailedAt = 0 + rf.firstWrite = time.Time{} + rf.lastWrite = time.Time{} + rf.headerWritten = false + return record, rf.writeHeaderLocked() +} + +// keepFileAfterFailedShift opens the file that could not move. The append flag +// keeps the lines that are already in it, which a truncating open would drop +// although no backup holds them. It must run with rf.mu held. +func (rf *rotatingFile) keepFileAfterFailedShift() error { + f, err := rf.openFile(os.O_CREATE | os.O_RDWR | os.O_APPEND) + if err != nil { + rf.size = 0 + return rf.noteOpenFailure(err) + } + rf.f = f + rf.openErr = nil + rf.rotateFailedAt = rf.size + return nil +} + +// noteOpenFailure keeps the error of a failed open and holds the next attempt +// back. It must run with rf.mu held. +func (rf *rotatingFile) noteOpenFailure(err error) error { + rf.openErr = fmt.Errorf("reopening log file: %w", err) + rf.retryOpenAt = time.Now().Add(retryOpenInterval) + return rf.openErr +} + +// shiftBackups moves each backup one number up and drops the oldest one. A +// missing backup is normal here, so those moves are best effort: a rotation +// must continue even when one backup cannot move. The move of the current +// file decides the rotation, so its error goes back to the caller. With no +// backup allowed, the current file still moves away, because a truncating +// reopen would change the bytes under an open upload reader. +func (rf *rotatingFile) shiftBackups() error { + if rf.budget.backups == 0 { + return rf.moveCurrentFile(rf.rotatingPath()) + } + for i := rf.budget.backups - 1; i >= 1; i-- { + _ = rf.renameFile(rf.backupPath(i), rf.backupPath(i+1)) + } + return rf.moveCurrentFile(rf.backupPath(1)) +} + +// moveCurrentFile renames the current file. A file that is not there is not +// an error, because a fresh path has nothing to move. +func (rf *rotatingFile) moveCurrentFile(to string) error { + if err := rf.renameFile(rf.path, to); err != nil && !os.IsNotExist(err) { + return err + } + return nil +} + +// pruneNumberedBackups removes the backups of path above keep. A lowered +// log_max_backups would otherwise leave old files that no reader lists. It +// reads the directory, because the numbers can have gaps. Only a name with an +// exact number after the path goes, because the directory can hold the files +// of another writer. +func pruneNumberedBackups(path string, keep int) { + dir := filepath.Dir(path) + entries, err := os.ReadDir(dir) + if err != nil { + return + } + prefix := filepath.Base(path) + "." + for _, entry := range entries { + suffix, found := strings.CutPrefix(entry.Name(), prefix) + if !found { + continue + } + index, err := strconv.Atoi(suffix) + if err != nil || index <= keep { + continue + } + os.Remove(filepath.Join(dir, entry.Name())) + } +} + +// writeHeaderLocked must run with rf.mu held. +func (rf *rotatingFile) writeHeaderLocked() error { + if len(rf.header) == 0 { + return nil + } + rf.headerWritten = true + _, err := rf.writeLocked(rf.header) + return err +} + +// writeLocked appends to the open file and counts the bytes. It must run +// with rf.mu held. +func (rf *rotatingFile) writeLocked(p []byte) (int, error) { + if rf.f == nil { + if rf.openErr == nil { + return 0, errLogFileNotOpen + } + if err := rf.retryOpenLocked(); err != nil { + return 0, err + } + } + n, err := rf.f.Write(p) + rf.size += int64(n) + return n, err +} + +// retryOpenLocked opens a file again that could not open. A file whose open +// failed once must not stay closed for the life of the process, because the +// cause is often a full disk or a directory that comes back. It must run with +// rf.mu held. +func (rf *rotatingFile) retryOpenLocked() error { + if time.Now().Before(rf.retryOpenAt) { + return rf.openErr + } + f, err := rf.openFile(os.O_CREATE | os.O_RDWR | os.O_APPEND) + if err != nil { + return rf.noteOpenFailure(err) + } + rf.f = f + rf.openErr = nil + rf.size = 0 + if st, err := f.Stat(); err == nil { + rf.size = st.Size() + } + rf.headerWritten = false + return rf.writeHeaderLocked() +} + +// noteEventWrite keeps the times that the rotation event reports. A header +// is not an event, so it does not move these times. +func (rf *rotatingFile) noteEventWrite() { + now := time.Now() + if rf.firstWrite.IsZero() { + rf.firstWrite = now + } + rf.lastWrite = now +} diff --git a/cmd/cli/log_file_test.go b/cmd/cli/log_file_test.go new file mode 100644 index 00000000..428be98e --- /dev/null +++ b/cmd/cli/log_file_test.go @@ -0,0 +1,553 @@ +package cli + +import ( + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "sync/atomic" + "testing" + "time" +) + +const rotatingFileTestLineSize = 1024 + +// rotatingFileTestLine returns a line of a fixed size that names its +// generation, so a test can tell the content of one file from another. +func rotatingFileTestLine(generation, seq int) []byte { + head := fmt.Sprintf("gen=%d seq=%d ", generation, seq) + return []byte(head + strings.Repeat("x", rotatingFileTestLineSize-len(head)-1) + "\n") +} + +func newTestRotatingFile(t *testing.T, budget logBudget) *rotatingFile { + t.Helper() + path := filepath.Join(t.TempDir(), "ctrld.log") + rf, err := newRotatingFile(path, budget, nil) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + return rf +} + +func rotatingFileTestContent(t *testing.T, path string) string { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatalf("read %s: %v", path, err) + } + return string(b) +} + +func writeRotatingFileTestLine(t *testing.T, rf *rotatingFile, generation, seq int) *logRotation { + t.Helper() + line := rotatingFileTestLine(generation, seq) + n, rotated, err := rf.writeReporting(line) + if err != nil { + t.Fatalf("write gen=%d seq=%d: %v", generation, seq, err) + } + if n != len(line) { + t.Fatalf("write gen=%d seq=%d wrote %d bytes, want %d", generation, seq, n, len(line)) + } + return rotated +} + +func Test_rotatingFileKeepsNumberedBackups(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 4096, backups: 4}) + rotations := 0 + for generation := 1; generation <= 7; generation++ { + for seq := 1; seq <= 4; seq++ { + if writeRotatingFileTestLine(t, rf, generation, seq) != nil { + rotations++ + } + } + } + if rotations != 6 { + t.Fatalf("rotations = %d, want 6", rotations) + } + + generationOf := map[string]int{"": 7, ".1": 6, ".2": 5, ".3": 4, ".4": 3} + for suffix, generation := range generationOf { + content := rotatingFileTestContent(t, rf.path+suffix) + if want := fmt.Sprintf("gen=%d seq=1 ", generation); !strings.HasPrefix(content, want) { + t.Fatalf("file %q starts with %.16q, want %q", rf.path+suffix, content, want) + } + if newer := fmt.Sprintf("gen=%d ", generation+1); strings.Contains(content, newer) { + t.Fatalf("file %q holds %q", rf.path+suffix, newer) + } + } + if _, err := os.Stat(rf.path + ".5"); !os.IsNotExist(err) { + t.Fatalf("stat %s.5 = %v, want a not exist error", rf.path, err) + } +} + +func Test_rotatingFileHeaderStartsEveryFile(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 4096, backups: 1}) + header := []byte(`{"level":"info","journal":true,"message":"Log header"}` + "\n") + rf.setHeader(header) + if err := rf.writeHeader(); err != nil { + t.Fatalf("writeHeader: %v", err) + } + if rf.size != int64(len(header)) { + t.Fatalf("size = %d, want %d", rf.size, len(header)) + } + if got := strings.Count(rotatingFileTestContent(t, rf.path), string(header)); got != 1 { + t.Fatalf("header count in the first file = %d, want 1", got) + } + + rotations := 0 + for seq := 1; seq <= 5; seq++ { + if writeRotatingFileTestLine(t, rf, 1, seq) != nil { + rotations++ + } + } + if rotations != 1 { + t.Fatalf("rotations = %d, want 1", rotations) + } + if !strings.HasPrefix(rotatingFileTestContent(t, rf.path+".1"), string(header)) { + t.Fatal("the backup file does not start with the header") + } + content := rotatingFileTestContent(t, rf.path) + if !strings.HasPrefix(content, string(header)) { + t.Fatal("the new file does not start with the header") + } + if got := strings.Count(content, string(header)); got != 1 { + t.Fatalf("header count in the new file = %d, want 1", got) + } + if want := int64(len(header) + 2*rotatingFileTestLineSize); rf.size != want { + t.Fatalf("size = %d, want %d", rf.size, want) + } +} + +func Test_rotatingFileRotationRecord(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 4096, backups: 2}) + if rf.currentPath() != rf.path { + t.Fatalf("currentPath = %q, want %q", rf.currentPath(), rf.path) + } + start := time.Now() + for seq := 1; seq <= 3; seq++ { + writeRotatingFileTestLine(t, rf, 1, seq) + } + sizeBeforeRotation := rf.size + + record, err := rf.rotateNow() + if err != nil { + t.Fatalf("rotateNow: %v", err) + } + if record == nil { + t.Fatal("rotateNow returned no record") + } + if want := rf.path + ".1"; record.file != want { + t.Fatalf("record.file = %q, want %q", record.file, want) + } + if record.backups != 2 { + t.Fatalf("record.backups = %d, want 2", record.backups) + } + if record.bytesWritten != sizeBeforeRotation { + t.Fatalf("record.bytesWritten = %d, want %d", record.bytesWritten, sizeBeforeRotation) + } + st, err := os.Stat(record.file) + if err != nil { + t.Fatalf("stat %s: %v", record.file, err) + } + if st.Size() != record.bytesWritten { + t.Fatalf("%s holds %d bytes, record.bytesWritten = %d", record.file, st.Size(), record.bytesWritten) + } + if record.firstWrite.Before(start) { + t.Fatalf("record.firstWrite = %v, want at or after %v", record.firstWrite, start) + } + if record.lastWrite.Before(record.firstWrite) { + t.Fatalf("record.lastWrite = %v is before record.firstWrite = %v", record.lastWrite, record.firstWrite) + } + + want := []string{rf.path + ".1", rf.path} + if got := rf.paths(); !slices.Equal(got, want) { + t.Fatalf("paths = %v, want %v", got, want) + } + writeRotatingFileTestLine(t, rf, 2, 1) + if _, err := rf.rotateNow(); err != nil { + t.Fatalf("second rotateNow: %v", err) + } + want = []string{rf.path + ".2", rf.path + ".1", rf.path} + if got := rf.paths(); !slices.Equal(got, want) { + t.Fatalf("paths after two rotations = %v, want %v", got, want) + } +} + +func Test_rotatingFileZeroBackupsTruncatesInPlace(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 2048, backups: 0}) + var record *logRotation + for seq := 1; seq <= 3; seq++ { + if rotated := writeRotatingFileTestLine(t, rf, 1, seq); rotated != nil { + record = rotated + } + } + if record == nil { + t.Fatal("no rotation happened") + } + if record.file != rf.path { + t.Fatalf("record.file = %q, want %q", record.file, rf.path) + } + if record.backups != 0 { + t.Fatalf("record.backups = %d, want 0", record.backups) + } + if want := int64(2 * rotatingFileTestLineSize); record.bytesWritten != want { + t.Fatalf("record.bytesWritten = %d, want %d", record.bytesWritten, want) + } + if _, err := os.Stat(rf.path + ".1"); !os.IsNotExist(err) { + t.Fatalf("stat %s.1 = %v, want a not exist error", rf.path, err) + } + if content := rotatingFileTestContent(t, rf.path); strings.Contains(content, "seq=1 ") { + t.Fatal("the truncated file still holds the old content") + } + if got, want := rf.paths(), []string{rf.path}; !slices.Equal(got, want) { + t.Fatalf("paths = %v, want %v", got, want) + } +} + +func Test_rotatingFileWriteCallsOnRotate(t *testing.T) { + t.Run("one call per rotation", func(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 4096, backups: 4}) + var rotations atomic.Int64 + rf.onRotate = func(logRotation) { rotations.Add(1) } + for generation := 1; generation <= 7; generation++ { + for seq := 1; seq <= 4; seq++ { + if _, err := rf.Write(rotatingFileTestLine(generation, seq)); err != nil { + t.Fatalf("Write: %v", err) + } + } + } + if got := rotations.Load(); got != 6 { + t.Fatalf("onRotate calls = %d, want 6", got) + } + }) + + t.Run("reentrant callback does not deadlock", func(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 4096, backups: 1}) + const reentrantLine = "written from onRotate\n" + var rotations atomic.Int64 + rf.onRotate = func(logRotation) { + rotations.Add(1) + if _, err := rf.Write([]byte(reentrantLine)); err != nil { + t.Errorf("reentrant Write: %v", err) + } + } + done := make(chan struct{}) + go func() { + defer close(done) + for seq := 1; seq <= 5; seq++ { + if _, err := rf.Write(rotatingFileTestLine(1, seq)); err != nil { + t.Errorf("Write: %v", err) + return + } + } + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("Write did not return, a reentrant onRotate deadlocks it") + } + if got := rotations.Load(); got != 1 { + t.Fatalf("onRotate calls = %d, want 1", got) + } + if !strings.Contains(rotatingFileTestContent(t, rf.path), reentrantLine) { + t.Fatal("the new file does not hold the reentrant line") + } + }) +} + +func Test_rotatingFileCountsExistingSize(t *testing.T) { + path := filepath.Join(t.TempDir(), "logs", "ctrld.log") + budget := logBudget{maxSize: 8192, backups: 1} + rf, err := newRotatingFile(path, budget, nil) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + for seq := 1; seq <= 3; seq++ { + writeRotatingFileTestLine(t, rf, 1, seq) + } + if err := rf.close(); err != nil { + t.Fatalf("close: %v", err) + } + + reopened, err := newRotatingFile(path, budget, nil) + if err != nil { + t.Fatalf("newRotatingFile on an existing file: %v", err) + } + t.Cleanup(func() { _ = reopened.close() }) + if want := int64(3 * rotatingFileTestLineSize); reopened.size != want { + t.Fatalf("size = %d, want %d", reopened.size, want) + } + record, err := reopened.rotateNow() + if err != nil { + t.Fatalf("rotateNow: %v", err) + } + if want := int64(3 * rotatingFileTestLineSize); record.bytesWritten != want { + t.Fatalf("record.bytesWritten = %d, want %d", record.bytesWritten, want) + } + st, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + if st.Size() != 0 { + t.Fatalf("the new file holds %d bytes, want 0", st.Size()) + } +} + +func Test_rotatingFileKeepsReopenError(t *testing.T) { + reopenErr := errors.New("reopen refused") + var opens atomic.Int64 + open := func(path string, flags int) (*os.File, error) { + if opens.Add(1) > 1 { + return nil, reopenErr + } + return os.OpenFile(path, flags, 0600) + } + path := filepath.Join(t.TempDir(), "ctrld.log") + rf, err := newRotatingFile(path, logBudget{maxSize: 1024, backups: 1}, open) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + + writeRotatingFileTestLine(t, rf, 1, 1) + if _, _, err := rf.writeReporting(rotatingFileTestLine(1, 2)); !errors.Is(err, reopenErr) { + t.Fatalf("write after a failed reopen = %v, want %v", err, reopenErr) + } + if rf.f != nil { + t.Fatal("the file is still open after a failed reopen") + } + if _, _, err := rf.writeReporting(rotatingFileTestLine(1, 3)); !errors.Is(err, reopenErr) { + t.Fatalf("later write = %v, want %v", err, reopenErr) + } + + oversized := make([]byte, 2*rotatingFileTestLineSize) + _, rotated, err := rf.writeReporting(oversized) + if !errors.Is(err, reopenErr) { + t.Fatalf("oversized write = %v, want %v", err, reopenErr) + } + if rotated != nil { + t.Fatalf("a closed file reported a rotation of %q", rotated.file) + } + if _, err := os.Stat(path + ".2"); !os.IsNotExist(err) { + t.Fatalf("stat %s.2 = %v, want a not exist error: a closed file shifted the backups", path, err) + } +} + +func Test_rotatingFileRetriesAFailedOpen(t *testing.T) { + origInterval := retryOpenInterval + retryOpenInterval = 10 * time.Millisecond + t.Cleanup(func() { retryOpenInterval = origInterval }) + + openErr := errors.New("open refused") + opens := 0 + open := func(path string, flags int) (*os.File, error) { + opens++ + if opens == 2 { + return nil, openErr + } + return os.OpenFile(path, flags, 0600) + } + path := filepath.Join(t.TempDir(), "ctrld.log") + rf, err := newRotatingFile(path, logBudget{maxSize: rotatingFileTestLineSize, backups: 1}, open) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + header := []byte(`{"level":"info","journal":true,"message":"Log header"}` + "\n") + rf.setHeader(header) + + writeRotatingFileTestLine(t, rf, 1, 1) + if _, _, err := rf.writeReporting(rotatingFileTestLine(1, 2)); !errors.Is(err, openErr) { + t.Fatalf("write after a failed reopen = %v, want %v", err, openErr) + } + if _, _, err := rf.writeReporting(rotatingFileTestLine(1, 3)); !errors.Is(err, openErr) { + t.Fatalf("write inside the retry interval = %v, want %v", err, openErr) + } + if opens != 2 { + t.Fatalf("opens = %d, want 2: a write inside the retry interval opened the file again", opens) + } + + time.Sleep(2 * retryOpenInterval) + writeRotatingFileTestLine(t, rf, 1, 4) + + content := rotatingFileTestContent(t, path) + if !strings.HasPrefix(content, string(header)) { + t.Fatalf("the reopened file starts with %.40q, want the header", content) + } + if !strings.Contains(content, "gen=1 seq=4 ") { + t.Fatalf("the reopened file misses the line that followed the retry: %.80q", content) + } +} + +func Test_pruneNumberedBackups(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, logFileName) + numbered := []string{path + ".1", path + ".2", path + ".3", path + ".4"} + // A prune must keep every file that another writer owns, because log_path + // can name a file in a directory full of them. + foreign := []string{path + ".bak", filepath.Join(dir, journalLogFileName+".3"), filepath.Join(dir, "syslog.5")} + for _, name := range append(slices.Clone(numbered), foreign...) { + if err := os.WriteFile(name, []byte("old\n"), 0600); err != nil { + t.Fatalf("write %s: %v", name, err) + } + } + + pruneNumberedBackups(path, 1) + + if _, err := os.Stat(numbered[0]); err != nil { + t.Fatalf("the backup inside the count went away: %v", err) + } + for _, name := range numbered[1:] { + if _, err := os.Stat(name); !os.IsNotExist(err) { + t.Fatalf("%s still exists after a prune that keeps one backup (err %v)", name, err) + } + } + for _, name := range foreign { + if _, err := os.Stat(name); err != nil { + t.Fatalf("the prune removed %s, which it does not own: %v", name, err) + } + } +} + +func Test_rotatingFileKeepsFilesAboveTheCount(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 2 * rotatingFileTestLineSize, backups: 1}) + late := rf.backupPath(3) + if err := os.WriteFile(late, []byte("late\n"), 0600); err != nil { + t.Fatalf("write the late backup: %v", err) + } + + for seq := 1; seq <= 3; seq++ { + if _, err := rf.Write(rotatingFileTestLine(1, seq)); err != nil { + t.Fatalf("write %d: %v", seq, err) + } + } + + if _, err := os.Stat(late); err != nil { + t.Fatalf("a rotation removed %s, which sits above the backup count: %v", late, err) + } + if _, err := os.Stat(rf.backupPath(1)); err != nil { + t.Fatalf("the rotation left no .1 file: %v", err) + } +} + +func Test_rotatingFileDoesNotFollowASymlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the Windows opener takes no symlink flag") + } + dir := t.TempDir() + planted := filepath.Join(dir, "planted.log") + if err := os.WriteFile(planted, []byte("planted\n"), 0600); err != nil { + t.Fatalf("write the planted file: %v", err) + } + path := filepath.Join(dir, "ctrld.log") + if err := os.Symlink(planted, path); err != nil { + t.Fatalf("symlink: %v", err) + } + + rf, err := newRotatingFile(path, logBudget{maxSize: 4096, backups: 1}, nil) + + if err == nil { + _ = rf.close() + t.Fatal("newRotatingFile followed the symlink") + } + if got := rotatingFileTestContent(t, planted); got != "planted\n" { + t.Fatalf("the planted file = %q, want it untouched", got) + } +} + +func Test_rotatingFileKeepsContentWhenRenameFails(t *testing.T) { + renameErr := errors.New("rename refused") + renameFails := true + path := filepath.Join(t.TempDir(), "ctrld.log") + rf, err := newRotatingFile(path, logBudget{maxSize: 2 * rotatingFileTestLineSize, backups: 1}, nil) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + rf.rename = func(oldpath, newpath string) error { + if renameFails { + return renameErr + } + return os.Rename(oldpath, newpath) + } + + for seq := 1; seq <= 2; seq++ { + writeRotatingFileTestLine(t, rf, 1, seq) + } + record := writeRotatingFileTestLine(t, rf, 1, 3) + if record == nil { + t.Fatal("the write past the budget reported no rotation") + } + if !errors.Is(record.err, renameErr) { + t.Fatalf("record.err = %v, want %v", record.err, renameErr) + } + content := rotatingFileTestContent(t, path) + for seq := 1; seq <= 3; seq++ { + if want := fmt.Sprintf("gen=1 seq=%d ", seq); !strings.Contains(content, want) { + t.Fatalf("the file lost %q after a failed rename", want) + } + } + if _, err := os.Stat(path + ".1"); !os.IsNotExist(err) { + t.Fatalf("stat %s.1 = %v, want a not exist error", path, err) + } + if writeRotatingFileTestLine(t, rf, 1, 4) != nil { + t.Fatal("the next write rotated again instead of waiting for another budget") + } + + renameFails = false + var second *logRotation + for seq := 5; seq <= 6; seq++ { + if rotated := writeRotatingFileTestLine(t, rf, 1, seq); rotated != nil { + second = rotated + } + } + if second == nil { + t.Fatal("no rotation happened after the rename worked again") + } + if second.err != nil { + t.Fatalf("record.err = %v, want no error", second.err) + } + if _, err := os.Stat(path + ".1"); err != nil { + t.Fatalf("stat %s.1 after a working rename: %v", path, err) + } +} + +// Test_zeroBackupRotationKeepsTheOpenReader opens an upload reader over the +// current file, then rotates with no backup. The reader must return the bytes +// and the size that it advertised, and the current file holds the new +// generation only. +func Test_zeroBackupRotationKeepsTheOpenReader(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 1 << 20, backups: 0}) + if _, err := rf.Write([]byte("old generation!\n")); err != nil { + t.Fatal(err) + } + var upload logParts + upload.addBoundedFiles(rf.paths(), 0) + t.Cleanup(upload.close) + + if _, err := rf.rotateNow(); err != nil { + t.Fatalf("rotate: %v", err) + } + if _, err := rf.Write([]byte("new\n")); err != nil { + t.Fatal(err) + } + + got, err := io.ReadAll(io.MultiReader(upload.readers...)) + if err != nil { + t.Fatal(err) + } + if upload.size != 16 || string(got) != "old generation!\n" { + t.Fatalf("reader gave %q (%d bytes advertised), want the old generation of 16 bytes", got, upload.size) + } + if content := rotatingFileTestContent(t, rf.currentPath()); content != "new\n" { + t.Fatalf("current file = %q, want the new generation only", content) + } + if _, err := os.Stat(rf.currentPath() + rotatingSuffix); !os.IsNotExist(err) { + t.Fatalf("the moved file is still on disk: %v", err) + } +} diff --git a/cmd/cli/log_header.go b/cmd/cli/log_header.go new file mode 100644 index 00000000..f9a58f67 --- /dev/null +++ b/cmd/cli/log_header.go @@ -0,0 +1,339 @@ +package cli + +import ( + "bytes" + "cmp" + "net" + "net/netip" + "os" + "runtime" + "slices" + "strconv" + "sync" + "time" + + "github.com/rs/zerolog" + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// processStartTime is the time this process started. A reader compares it with +// the time of the header line to tell a restart from a rotation. +var processStartTime = time.Now() + +const ( + // logHeaderMessage is the message of the header line. Log tools select the + // header by this message. + logHeaderMessage = "Log header" + + // interceptModeOff is the mode word for a ctrld that does not intercept. + interceptModeOff = "off" + + // logHeaderTriggerSend marks the header that a log request renders. It + // tells a reader that the line came from an upload, not from a file open. + logHeaderTriggerSend = "send" + + // upstreamTypeUnspecified names an upstream that has no type in the config. + // ctrld resolves the type at start, so the header must not claim one. + upstreamTypeUnspecified = "unspecified" +) + +// headerOSVersion reads the OS version once. The read starts a subprocess on +// macOS, and a header refresh follows every network change. +var headerOSVersion = sync.OnceValue(osVersion) + +// logHeaderInterface is one interface of the header network snapshot. +type logHeaderInterface struct { + Name string + Up bool + IPs []string +} + +// logHeaderNetwork is the minimal network snapshot of a header line. +type logHeaderNetwork struct { + DefaultRouteInterface string + HaveV4 bool + HaveV6 bool + Interfaces []logHeaderInterface +} + +// logHeaderInput holds every value of a header line. The caller collects the +// values, so the renderer stays free of locks and of system calls. +type logHeaderInput struct { + Version string + Commit string + OS string + Arch string + PID int + StartTime time.Time + InterceptMode string + Listeners []string + UpstreamCount int + UpstreamTypes []string + ResolverUID string + LogFiles []string + Network logHeaderNetwork + Trigger string +} + +// renderLogHeader renders the header as one log line. A file writer stores the +// bytes and writes them itself, so the header reaches a file that the logger +// never fills. +func renderLogHeader(in logHeaderInput) []byte { + var buf bytes.Buffer + logger := zerolog.New(&buf).With().Timestamp().Logger() + event := journal(logHeaderEvent(&logger)). + Str("version", in.Version). + Str("commit", in.Commit). + Str("os", in.OS). + Str("arch", in.Arch). + Int("pid", in.PID). + Str("start_time", in.StartTime.Format(time.RFC3339)). + Str("intercept_mode", in.InterceptMode). + Strs("listeners", in.Listeners). + Int("upstream_count", in.UpstreamCount). + Strs("upstream_types", in.UpstreamTypes). + Str("resolver_uid", in.ResolverUID). + Strs("log_files", in.LogFiles). + Dict("network", logHeaderNetworkDict(in.Network)) + if in.Trigger != "" { + event = event.Str("trigger", in.Trigger) + } + event.Msg(logHeaderMessage) + + // The header is the first line of every file and of every upload, and it + // bypasses the writers, so it passes the journal redaction here. + return redactRetainedLine(buf.Bytes()) +} + +// logHeaderEvent starts the header event at info level. The global level sits +// above info on a default start, and a log file must hold its header at every +// level, so the event names its level itself. +func logHeaderEvent(logger *zerolog.Logger) *zerolog.Event { + return logger.Log().Str(zerolog.LevelFieldName, zerolog.LevelInfoValue) +} + +func logHeaderNetworkDict(network logHeaderNetwork) *zerolog.Event { + interfaces := zerolog.Arr() + for _, iface := range network.Interfaces { + interfaces = interfaces.Dict(zerolog.Dict(). + Str("name", iface.Name). + Bool("up", iface.Up). + Strs("ips", iface.IPs)) + } + return zerolog.Dict(). + Str("default_route_interface", network.DefaultRouteInterface). + Bool("have_v4", network.HaveV4). + Bool("have_v6", network.HaveV6). + Array("interfaces", interfaces) +} + +// logHeaderNetworkFromState makes the header snapshot from a netmon state. +func logHeaderNetworkFromState(st *netmon.State) logHeaderNetwork { + if st == nil { + return logHeaderNetwork{} + } + network := logHeaderNetwork{ + DefaultRouteInterface: st.DefaultRouteInterface, + HaveV4: st.HaveV4, + HaveV6: st.HaveV6, + } + for name, iface := range st.Interface { + network.Interfaces = append(network.Interfaces, logHeaderInterface{ + Name: name, + Up: iface.Interface != nil && iface.IsUp(), + IPs: prefixStrings(st.InterfaceIPs[name]), + }) + } + slices.SortFunc(network.Interfaces, func(a, b logHeaderInterface) int { + return cmp.Compare(a.Name, b.Name) + }) + return network +} + +// logHeaderNetworkFresh reads the network now. A header that no network +// callback reached yet, and the header of an upload, need the live picture. +func logHeaderNetworkFresh() logHeaderNetwork { + state, err := readNetworkSourceStateFn() + if err != nil { + state = nil + } + network := logHeaderNetworkFromState(state) + if route, err := netmon.DefaultRouteInterface(); err == nil { + network.DefaultRouteInterface = route + } + network.HaveV4, network.HaveV6 = globalAddressFamilies(state) + return network +} + +// globalAddressFamilies reports which families the host can reach. A loopback +// or a link-local address reaches no other host, so it proves nothing. +func globalAddressFamilies(st *netmon.State) (haveV4, haveV6 bool) { + if st == nil { + return false, false + } + for name, iface := range st.Interface { + if iface.Interface == nil || !iface.IsUp() { + continue + } + for _, prefix := range st.InterfaceIPs[name] { + addr := prefix.Addr().Unmap() + if !addr.IsGlobalUnicast() || addr.IsLoopback() || addr.IsLinkLocalUnicast() { + continue + } + if addr.Is4() { + haveV4 = true + continue + } + haveV6 = true + } + } + return haveV4, haveV6 +} + +// logHeaderInput collects every header value of the running program. +func (p *prog) logHeaderInput() logHeaderInput { + in := logHeaderInput{ + Version: curVersion(), + Commit: commit, + OS: headerOSVersion(), + Arch: runtime.GOARCH, + PID: os.Getpid(), + StartTime: processStartTime, + // The resolver UID is a secret that the provisioning code strips from + // every artifact, so the header keeps its first characters only. + ResolverUID: redactToken(cdUID), + LogFiles: p.logHeaderFiles(), + Network: p.logHeaderNetwork(), + } + p.headerConfigInto(&in) + return in +} + +// headerConfigInto reads the config under p.mu, because a reload replaces the +// config under that lock while log view and log send render a header. +func (p *prog) headerConfigInto(in *logHeaderInput) { + p.mu.Lock() + defer p.mu.Unlock() + logHeaderFromConfig(p.cfg, in) +} + +// sendLogHeader renders the header that leads one upload. +func (p *prog) sendLogHeader() []byte { + in := p.logHeaderInput() + // The stored state can be older than the request, and support reads this + // line to learn the network of the moment the user asked. + in.Network = logHeaderNetworkFresh() + in.Trigger = logHeaderTriggerSend + return renderLogHeader(in) +} + +// noteNetworkState keeps the newest network that ctrld saw and puts it in the +// header bytes of every open log file. The caller must hold no lock, because +// the render reads the config and the network. +func (p *prog) noteNetworkState(st *netmon.State) { + if st != nil { + p.lastNetworkState.Store(st) + } + p.refreshLogHeader() +} + +// logHeaderNetwork takes the network that the last monitor callback reported. +func (p *prog) logHeaderNetwork() logHeaderNetwork { + state := p.lastNetworkState.Load() + if state == nil { + return logHeaderNetworkFresh() + } + return logHeaderNetworkFromState(state) +} + +// logHeaderFiles names the files that this run writes, debug first. +func (p *prog) logHeaderFiles() []string { + files := make([]string, 0, 3) + for _, rf := range p.openLogFiles() { + files = append(files, rf.currentPath()) + } + return files +} + +// refreshLogHeader renders the header and hands the bytes to every open file. +// The caller must hold no writer lock, because the render reads the config +// and the network. +func (p *prog) refreshLogHeader() { + header := renderLogHeader(p.logHeaderInput()) + for _, rf := range p.openLogFiles() { + rf.setHeader(header) + } +} + +// writeLogHeaders puts the header at the append point of each open file. A +// header that cannot land leaves the file usable, so the error only earns a +// warning. +func (p *prog) writeLogHeaders() { + for _, rf := range p.openLogFiles() { + if err := rf.writeHeader(); err != nil { + mainLog.Load().Warn().Err(err).Msg("could not write log header") + } + } +} + +func prefixStrings(prefixes []netip.Prefix) []string { + ips := make([]string, 0, len(prefixes)) + for _, prefix := range prefixes { + ips = append(ips, prefix.String()) + } + return ips +} + +// logHeaderFromConfig fills the header fields that the config holds. It keeps +// every upstream endpoint, name, and bootstrap IP out of them, because a token +// can hide in each of them. +func logHeaderFromConfig(cfg *ctrld.Config, in *logHeaderInput) { + if cfg == nil { + in.InterceptMode = interceptModeOff + return + } + in.InterceptMode = headerInterceptMode(cfg) + in.Listeners = listenerAddresses(cfg) + in.UpstreamCount = len(cfg.Upstream) + in.UpstreamTypes = upstreamTypeWords(cfg) +} + +// headerInterceptMode names the mode for the header. An unset mode means that +// ctrld does not intercept, which reads as off. +func headerInterceptMode(cfg *ctrld.Config) string { + if mode := listenerInterceptMode(cfg); mode != "" { + return mode + } + return interceptModeOff +} + +func listenerAddresses(cfg *ctrld.Config) []string { + addresses := make([]string, 0, len(cfg.Listener)) + for _, listener := range cfg.Listener { + if listener == nil { + continue + } + addresses = append(addresses, net.JoinHostPort(listener.IP, strconv.Itoa(listener.Port))) + } + slices.Sort(addresses) + return addresses +} + +func upstreamTypeWords(cfg *ctrld.Config) []string { + words := make([]string, 0, len(cfg.Upstream)) + for _, upstream := range cfg.Upstream { + if upstream == nil { + continue + } + word := upstream.Type + if word == "" { + word = upstreamTypeUnspecified + } + if !slices.Contains(words, word) { + words = append(words, word) + } + } + slices.Sort(words) + return words +} diff --git a/cmd/cli/log_header_test.go b/cmd/cli/log_header_test.go new file mode 100644 index 00000000..48d49680 --- /dev/null +++ b/cmd/cli/log_header_test.go @@ -0,0 +1,617 @@ +package cli + +import ( + "encoding/json" + "io" + "net" + "net/netip" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "github.com/rs/zerolog" + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// logHeaderTestSecret is the string that must never reach a header line. +const logHeaderTestSecret = "secret-token" + +// logHeaderTestLine is the parsed form of a header line. The json tags name +// the fields that a log tool reads, so a renamed field fails the test. +type logHeaderTestLine struct { + Level string `json:"level"` + Time string `json:"time"` + Journal bool `json:"journal"` + Message string `json:"message"` + Version string `json:"version"` + Commit string `json:"commit"` + OS string `json:"os"` + Arch string `json:"arch"` + PID int `json:"pid"` + StartTime string `json:"start_time"` + InterceptMode string `json:"intercept_mode"` + Listeners []string `json:"listeners"` + UpstreamCount int `json:"upstream_count"` + UpstreamTypes []string `json:"upstream_types"` + ResolverUID string `json:"resolver_uid"` + LogFiles []string `json:"log_files"` + Network struct { + DefaultRouteInterface string `json:"default_route_interface"` + HaveV4 bool `json:"have_v4"` + HaveV6 bool `json:"have_v6"` + Interfaces []struct { + Name string `json:"name"` + Up bool `json:"up"` + IPs []string `json:"ips"` + } `json:"interfaces"` + } `json:"network"` + Trigger string `json:"trigger"` +} + +func logHeaderTestInput() logHeaderInput { + return logHeaderInput{ + Version: "1.5.8", + Commit: "abc1234", + OS: "darwin 15.6", + Arch: "arm64", + PID: 4242, + StartTime: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC), + InterceptMode: "dns", + Listeners: []string{"127.0.0.1:53", "127.0.0.1:5354"}, + UpstreamCount: 2, + UpstreamTypes: []string{"doh", "os"}, + ResolverUID: redactToken("abcd1234"), + LogFiles: []string{"/tmp/ctrld.log", "/tmp/ctrld-journal.log"}, + Network: logHeaderNetwork{ + DefaultRouteInterface: "en0", + HaveV4: true, + Interfaces: []logHeaderInterface{ + {Name: "en0", Up: true, IPs: []string{"192.0.2.10/24"}}, + }, + }, + } +} + +// parseLogHeader checks that the render is one line and returns it twice: as +// values, and as raw fields for the tests that ask whether a field is there. +func parseLogHeader(t *testing.T, line []byte) (logHeaderTestLine, map[string]json.RawMessage) { + t.Helper() + if len(line) == 0 { + t.Fatal("renderLogHeader returned no bytes") + } + if line[len(line)-1] != '\n' { + t.Fatalf("header does not end with a newline: %q", line) + } + if count := strings.Count(string(line), "\n"); count != 1 { + t.Fatalf("header holds %d newlines, want 1: %q", count, line) + } + var parsed logHeaderTestLine + if err := json.Unmarshal(line, &parsed); err != nil { + t.Fatalf("parse header %q: %v", line, err) + } + fields := map[string]json.RawMessage{} + if err := json.Unmarshal(line, &fields); err != nil { + t.Fatalf("parse header fields %q: %v", line, err) + } + return parsed, fields +} + +func setLogHeaderTestGlobalLevel(t *testing.T, level zerolog.Level) { + t.Helper() + old := zerolog.GlobalLevel() + zerolog.SetGlobalLevel(level) + t.Cleanup(func() { zerolog.SetGlobalLevel(old) }) +} + +func Test_renderLogHeaderHoldsEveryField(t *testing.T) { + setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + in := logHeaderTestInput() + + parsed, fields := parseLogHeader(t, renderLogHeader(in)) + + checks := []struct { + field string + got any + want any + }{ + {"level", parsed.Level, "info"}, + {"journal", parsed.Journal, true}, + {"message", parsed.Message, "Log header"}, + {"version", parsed.Version, in.Version}, + {"commit", parsed.Commit, in.Commit}, + {"os", parsed.OS, in.OS}, + {"arch", parsed.Arch, in.Arch}, + {"pid", parsed.PID, in.PID}, + {"start_time", parsed.StartTime, in.StartTime.Format(time.RFC3339)}, + {"intercept_mode", parsed.InterceptMode, in.InterceptMode}, + {"listeners", parsed.Listeners, in.Listeners}, + {"upstream_count", parsed.UpstreamCount, in.UpstreamCount}, + {"upstream_types", parsed.UpstreamTypes, in.UpstreamTypes}, + // The resolver UID is a secret, so the header keeps the first four + // characters only. + {"resolver_uid", parsed.ResolverUID, "abcd***"}, + {"log_files", parsed.LogFiles, in.LogFiles}, + {"network.default_route_interface", parsed.Network.DefaultRouteInterface, in.Network.DefaultRouteInterface}, + {"network.have_v4", parsed.Network.HaveV4, in.Network.HaveV4}, + {"network.have_v6", parsed.Network.HaveV6, in.Network.HaveV6}, + } + for _, c := range checks { + if !reflect.DeepEqual(c.got, c.want) { + t.Errorf("%s = %v, want %v", c.field, c.got, c.want) + } + } + + if len(parsed.Network.Interfaces) != 1 { + t.Fatalf("network.interfaces holds %d entries, want 1", len(parsed.Network.Interfaces)) + } + iface := parsed.Network.Interfaces[0] + if iface.Name != "en0" || !iface.Up || !reflect.DeepEqual(iface.IPs, []string{"192.0.2.10/24"}) { + t.Errorf("network.interfaces[0] = %+v, want en0 up with 192.0.2.10/24", iface) + } + if parsed.Time == "" { + t.Error("time is empty") + } + if _, ok := fields["trigger"]; ok { + t.Errorf("trigger is present for an empty Trigger: %s", fields["trigger"]) + } +} + +func Test_renderLogHeaderAddsTriggerWhenSet(t *testing.T) { + setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + in := logHeaderTestInput() + in.Trigger = "send" + + parsed, _ := parseLogHeader(t, renderLogHeader(in)) + + if parsed.Trigger != "send" { + t.Errorf("trigger = %q, want %q", parsed.Trigger, "send") + } +} + +// Test_renderLogHeaderIgnoresGlobalLevel guards the promise that every log file +// starts with a header. The global level sits above info on a default start. +func Test_renderLogHeaderIgnoresGlobalLevel(t *testing.T) { + setLogHeaderTestGlobalLevel(t, zerolog.WarnLevel) + + parsed, _ := parseLogHeader(t, renderLogHeader(logHeaderTestInput())) + + if parsed.Level != "info" { + t.Errorf("level = %q, want %q", parsed.Level, "info") + } + if parsed.Message != "Log header" { + t.Errorf("message = %q, want %q", parsed.Message, "Log header") + } +} + +func Test_logHeaderFromConfigKeepsUpstreamSecretsOut(t *testing.T) { + setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + old := interceptMode + t.Cleanup(func() { interceptMode = old }) + interceptMode = "" + + cfg := &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{ + "0": {IP: "127.0.0.1", Port: 53}, + logHeaderTestSecret + "-l": {IP: "10.0.0.1", Port: 5354}, + }, + Upstream: map[string]*ctrld.UpstreamConfig{ + "0": { + Name: logHeaderTestSecret + "-name", + Type: "doh", + Endpoint: "https://dns.example/" + logHeaderTestSecret, + BootstrapIP: "192.0.2.53", + }, + "1": {Endpoint: "https://dns.example/" + logHeaderTestSecret}, + }, + } + cfg.Service.InterceptMode = "dns" + + in := logHeaderTestInput() + logHeaderFromConfig(cfg, &in) + + if in.InterceptMode != "dns" { + t.Errorf("intercept mode = %q, want %q", in.InterceptMode, "dns") + } + if want := []string{"10.0.0.1:5354", "127.0.0.1:53"}; !reflect.DeepEqual(in.Listeners, want) { + t.Errorf("listeners = %v, want %v", in.Listeners, want) + } + if in.UpstreamCount != 2 { + t.Errorf("upstream count = %d, want 2", in.UpstreamCount) + } + if want := []string{"doh", "unspecified"}; !reflect.DeepEqual(in.UpstreamTypes, want) { + t.Errorf("upstream types = %v, want %v", in.UpstreamTypes, want) + } + + line := renderLogHeader(in) + if strings.Contains(string(line), logHeaderTestSecret) { + t.Errorf("header holds %q: %s", logHeaderTestSecret, line) + } +} + +func Test_logHeaderFromConfigReportsOffWhenUnset(t *testing.T) { + old := interceptMode + t.Cleanup(func() { interceptMode = old }) + interceptMode = "" + + var in logHeaderInput + logHeaderFromConfig(&ctrld.Config{}, &in) + + if in.InterceptMode != "off" { + t.Errorf("intercept mode = %q, want %q", in.InterceptMode, "off") + } +} + +func Test_logHeaderNetworkFromState(t *testing.T) { + setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + state := &netmon.State{ + DefaultRouteInterface: "en0", + HaveV4: true, + Interface: map[string]netmon.Interface{ + "en0": {Interface: &net.Interface{Name: "en0", Flags: net.FlagUp}}, + "awdl0": {Interface: &net.Interface{Name: "awdl0"}}, + }, + InterfaceIPs: map[string][]netip.Prefix{ + "en0": {netip.MustParsePrefix("192.0.2.10/24"), netip.MustParsePrefix("2001:db8::1/64")}, + }, + } + + network := logHeaderNetworkFromState(state) + + want := logHeaderNetwork{ + DefaultRouteInterface: "en0", + HaveV4: true, + Interfaces: []logHeaderInterface{ + {Name: "awdl0", Up: false, IPs: []string{}}, + {Name: "en0", Up: true, IPs: []string{"192.0.2.10/24", "2001:db8::1/64"}}, + }, + } + if !reflect.DeepEqual(network, want) { + t.Fatalf("network = %+v, want %+v", network, want) + } + + in := logHeaderTestInput() + in.Network = network + parsed, _ := parseLogHeader(t, renderLogHeader(in)) + if parsed.Network.DefaultRouteInterface != "en0" { + t.Errorf("network.default_route_interface = %q, want %q", parsed.Network.DefaultRouteInterface, "en0") + } + if len(parsed.Network.Interfaces) != 2 { + t.Fatalf("network.interfaces holds %d entries, want 2", len(parsed.Network.Interfaces)) + } + if parsed.Network.Interfaces[0].Name != "awdl0" || parsed.Network.Interfaces[0].Up { + t.Errorf("network.interfaces[0] = %+v, want awdl0 down", parsed.Network.Interfaces[0]) + } + if got := parsed.Network.Interfaces[1].IPs; !reflect.DeepEqual(got, []string{"192.0.2.10/24", "2001:db8::1/64"}) { + t.Errorf("network.interfaces[1].ips = %v", got) + } +} + +func Test_logHeaderNetworkFromStateWithoutState(t *testing.T) { + if network := logHeaderNetworkFromState(nil); !reflect.DeepEqual(network, logHeaderNetwork{}) { + t.Errorf("network = %+v, want the zero value", network) + } +} + +// setupLogHeaderWiringTest puts ctrld in cd mode with a temporary home +// directory, so initInternalLogging writes its files there. Every global that +// the logging setup touches goes back when the test ends. +func setupLogHeaderWiringTest(t *testing.T) string { + t.Helper() + origSilent, origCdUID, origHomedir, origVerbose := silent, cdUID, homedir, verbose + origMainLog, origProxyLogger := mainLog.Load(), ctrld.ProxyLogger.Load() + origLevel := zerolog.GlobalLevel() + origLogPathFile := logPathFile.Load() + t.Cleanup(func() { + silent, cdUID, homedir, verbose = origSilent, origCdUID, origHomedir, origVerbose + mainLog.Store(origMainLog) + ctrld.ProxyLogger.Store(origProxyLogger) + zerolog.SetGlobalLevel(origLevel) + logPathFile.Store(origLogPathFile) + }) + + homedir = t.TempDir() + cdUID = "header-wiring-uid" + silent, verbose = false, 0 + discard := zerolog.New(io.Discard) + mainLog.Store(&discard) + logPathFile.Store(nil) + return homedir +} + +// startLogHeaderWiringProg runs the internal logging setup and closes both +// files when the test ends. +func startLogHeaderWiringProg(t *testing.T) *prog { + t.Helper() + p := &prog{cfg: &ctrld.Config{}} + p.initInternalLogging(nil) + t.Cleanup(func() { + p.internalLogWriter.closeLogFile() + p.internalJournalWriter.closeLogFile() + }) + return p +} + +// logHeaderFileLines parses every line of a log file. It returns line 1 and +// the number of header lines, so a test can tell a missing header from a +// repeated one. +func logHeaderFileLines(t *testing.T, path string) (logHeaderTestLine, int) { + t.Helper() + content := strings.TrimRight(rotatingFileTestContent(t, path), "\n") + if content == "" { + t.Fatalf("%s is empty", path) + } + var firstLine logHeaderTestLine + headers := 0 + for i, line := range strings.Split(content, "\n") { + var parsed logHeaderTestLine + if err := json.Unmarshal([]byte(line), &parsed); err != nil { + t.Fatalf("parse line %d of %s (%q): %v", i+1, path, line, err) + } + if i == 0 { + firstLine = parsed + } + if parsed.Message == logHeaderMessage { + headers++ + } + } + return firstLine, headers +} + +func Test_logHeaderWiring_writesHeaderOnStart(t *testing.T) { + dir := setupLogHeaderWiringTest(t) + + startLogHeaderWiringProg(t) + + for _, name := range []string{logFileName, journalLogFileName} { + path := filepath.Join(dir, name) + firstLine, headers := logHeaderFileLines(t, path) + if firstLine.Message != logHeaderMessage { + t.Errorf("line 1 of %s = %q, want %q", path, firstLine.Message, logHeaderMessage) + } + if !firstLine.Journal { + t.Errorf("line 1 of %s carries no journal marker", path) + } + if want := redactToken(cdUID); firstLine.ResolverUID != want { + t.Errorf("resolver_uid in %s = %q, want %q", path, firstLine.ResolverUID, want) + } + if headers != 1 { + t.Errorf("%s holds %d header lines, want 1", path, headers) + } + } +} + +func Test_logHeaderInput_keepsSecretsOut(t *testing.T) { + setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + dir := setupLogHeaderWiringTest(t) + origInterceptMode := interceptMode + t.Cleanup(func() { interceptMode = origInterceptMode }) + interceptMode = "" + p := &prog{cfg: &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{ + logHeaderTestSecret + "-listener": {IP: "127.0.0.1", Port: 53}, + }, + Upstream: map[string]*ctrld.UpstreamConfig{ + "0": { + Name: logHeaderTestSecret + "-name", + Type: "doh", + Endpoint: "https://dns.example/" + logHeaderTestSecret, + BootstrapIP: "192.0.2.53", + Domain: logHeaderTestSecret + ".example", + }, + }, + }} + p.initInternalLogging(nil) + t.Cleanup(func() { + p.internalLogWriter.closeLogFile() + p.internalJournalWriter.closeLogFile() + }) + + if line := renderLogHeader(p.logHeaderInput()); strings.Contains(string(line), logHeaderTestSecret) { + t.Errorf("the header holds %q: %s", logHeaderTestSecret, line) + } + + for _, name := range []string{logFileName, journalLogFileName} { + path := filepath.Join(dir, name) + if content := rotatingFileTestContent(t, path); strings.Contains(content, logHeaderTestSecret) { + t.Errorf("%s holds %q: %s", path, logHeaderTestSecret, content) + } + } +} + +func Test_logHeaderWiring_secondInitAddsNoHeader(t *testing.T) { + dir := setupLogHeaderWiringTest(t) + p := startLogHeaderWiringProg(t) + + p.initInternalLogging(nil) + + for _, name := range []string{logFileName, journalLogFileName} { + path := filepath.Join(dir, name) + if _, headers := logHeaderFileLines(t, path); headers != 1 { + t.Errorf("%s holds %d header lines after a second start, want 1", path, headers) + } + } +} + +func Test_logHeaderWiring_headerLeadsRotatedFile(t *testing.T) { + setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + captureMainLog(t) + lw, path := newTestFileLogWriter(t, testLogWriterBudget) + header := string(renderLogHeader(logHeaderInput{Version: "v1"})) + + rf := lw.rotating() + rf.setHeader([]byte(header)) + if err := rf.writeHeader(); err != nil { + t.Fatalf("writeHeader: %v", err) + } + linesPerFile := int(testLogWriterBudget.maxSize) / rotatingFileTestLineSize + for seq := 1; seq <= linesPerFile+1; seq++ { + if _, err := lw.Write(rotatingFileTestLine(1, seq)); err != nil { + t.Fatalf("write line %d: %v", seq, err) + } + } + + for _, name := range []string{path + ".1", path} { + if content := rotatingFileTestContent(t, name); !strings.HasPrefix(content, header) { + t.Errorf("%s starts with %q, want the header", name, strings.SplitN(content, "\n", 2)[0]) + } + } +} + +func Test_logHeaderWiring_restartAppendsHeader(t *testing.T) { + dir := setupLogHeaderWiringTest(t) + path := filepath.Join(dir, logFileName) + const seed = `{"level":"info","message":"line of the run before"}` + if err := os.WriteFile(path, []byte(seed+"\n"), 0o600); err != nil { + t.Fatalf("seed %s: %v", path, err) + } + + startLogHeaderWiringProg(t) + + lines := strings.Split(strings.TrimRight(rotatingFileTestContent(t, path), "\n"), "\n") + if len(lines) < 2 { + t.Fatalf("%s holds %d lines, want the seed line and the header", path, len(lines)) + } + if lines[0] != seed { + t.Errorf("line 1 of %s = %q, want the seed line", path, lines[0]) + } + var header logHeaderTestLine + if err := json.Unmarshal([]byte(lines[1]), &header); err != nil { + t.Fatalf("parse line 2 of %s (%q): %v", path, lines[1], err) + } + if header.Message != logHeaderMessage { + t.Errorf("line 2 of %s = %q, want %q", path, header.Message, logHeaderMessage) + } + if _, headers := logHeaderFileLines(t, path); headers != 1 { + t.Errorf("%s holds %d header lines, want 1", path, headers) + } +} + +func Test_logHeaderWiring_usesLastNetworkState(t *testing.T) { + setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + origRead := readNetworkSourceStateFn + t.Cleanup(func() { readNetworkSourceStateFn = origRead }) + p := &prog{cfg: &ctrld.Config{}} + + p.lastNetworkState.Store(&netmon.State{DefaultRouteInterface: "en7"}) + + if got := p.logHeaderInput().Network.DefaultRouteInterface; got != "en7" { + t.Errorf("default_route_interface = %q, want %q", got, "en7") + } + + p.lastNetworkState.Store(nil) + readNetworkSourceStateFn = func() (*netmon.State, error) { + return &netmon.State{ + Interface: map[string]netmon.Interface{"en9": {Interface: &net.Interface{Name: "en9", Flags: net.FlagUp}}}, + InterfaceIPs: map[string][]netip.Prefix{"en9": {netip.MustParsePrefix("192.0.2.9/24")}}, + }, nil + } + + network := p.logHeaderInput().Network + if len(network.Interfaces) != 1 || network.Interfaces[0].Name != "en9" || !network.Interfaces[0].Up { + t.Fatalf("interfaces = %+v, want one up en9", network.Interfaces) + } + if !network.HaveV4 { + t.Error("have_v4 = false, want true for a global unicast v4 address") + } + if network.HaveV6 { + t.Error("have_v6 = true, want false without a v6 address") + } +} + +// stubLogHeaderNetworkRead makes the fresh network read return one up +// interface, so a header test never depends on the host network. +func stubLogHeaderNetworkRead(t *testing.T, name string) { + t.Helper() + origRead := readNetworkSourceStateFn + t.Cleanup(func() { readNetworkSourceStateFn = origRead }) + readNetworkSourceStateFn = func() (*netmon.State, error) { + return &netmon.State{ + Interface: map[string]netmon.Interface{name: {Interface: &net.Interface{Name: name, Flags: net.FlagUp}}}, + InterfaceIPs: map[string][]netip.Prefix{name: {netip.MustParsePrefix("192.0.2.9/24")}}, + }, nil + } +} + +func Test_sendLogHeader_usesFreshNetworkAndTrigger(t *testing.T) { + setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + stubLogHeaderNetworkRead(t, "en9") + p := &prog{cfg: &ctrld.Config{}} + p.lastNetworkState.Store(&netmon.State{ + DefaultRouteInterface: "en7", + Interface: map[string]netmon.Interface{"en7": {Interface: &net.Interface{Name: "en7", Flags: net.FlagUp}}}, + InterfaceIPs: map[string][]netip.Prefix{"en7": {netip.MustParsePrefix("192.0.2.7/24")}}, + }) + + parsed, _ := parseLogHeader(t, p.sendLogHeader()) + + if parsed.Trigger != "send" { + t.Errorf("trigger = %q, want %q", parsed.Trigger, "send") + } + if parsed.Message != logHeaderMessage { + t.Errorf("message = %q, want %q", parsed.Message, logHeaderMessage) + } + names := make([]string, 0, len(parsed.Network.Interfaces)) + for _, iface := range parsed.Network.Interfaces { + names = append(names, iface.Name) + } + if want := []string{"en9"}; !reflect.DeepEqual(names, want) { + t.Fatalf("network.interfaces = %v, want %v from the fresh read", names, want) + } +} + +// Test_logHeaderInputReadsTheConfigUnderTheLock renders headers beside the +// config swap of a reload. The race detector fails the test when the render +// reads the config outside the lock. +func Test_logHeaderInputReadsTheConfigUnderTheLock(t *testing.T) { + stubLogHeaderNetworkRead(t, "en0") + p := &prog{cfg: &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}}, + Upstream: map[string]*ctrld.UpstreamConfig{"0": {Type: "doh"}}, + }} + reloaded := &ctrld.Config{ + Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 5354}}, + Upstream: map[string]*ctrld.UpstreamConfig{"0": {Type: "doq"}, "1": {Type: "legacy"}}, + } + + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + for { + select { + case <-stop: + return + default: + } + p.mu.Lock() + *p.cfg = *reloaded + p.mu.Unlock() + } + }() + for i := 0; i < 300; i++ { + if in := p.logHeaderInput(); len(in.Listeners) != 1 { + t.Fatalf("listeners = %v, want one", in.Listeners) + } + } + close(stop) + <-done +} + +// Test_renderLogHeaderPassesTheSink puts a provisioning secret where the +// header does not expect one. The header is the first line of every file and +// of every upload, so it must pass the same redaction as every other line. +func Test_renderLogHeaderPassesTheSink(t *testing.T) { + origUID := cdUID + cdUID = "abcd1234SECRETUID" + t.Cleanup(func() { cdUID = origUID }) + + line := string(renderLogHeader(logHeaderInput{Version: "1.0", LogFiles: []string{"/var/log/" + cdUID + "/ctrld.log"}})) + + if strings.Contains(line, cdUID) { + t.Fatalf("the header holds the secret: %s", line) + } +} diff --git a/cmd/cli/log_journal.go b/cmd/cli/log_journal.go new file mode 100644 index 00000000..0f851be8 --- /dev/null +++ b/cmd/cli/log_journal.go @@ -0,0 +1,68 @@ +package cli + +import ( + "bytes" + "io" + "regexp" + + "github.com/rs/zerolog" +) + +// journalField marks an event for the retained journal stream. +const journalField = "journal" + +// journalMarker is the serialized form of the journal field. The journal +// writer keeps a line below warn level only when the line contains it. +var journalMarker = []byte(`"` + journalField + `":true`) + +// retainedURLPath matches the part of a URL that follows the host. The class +// stops the match at the characters that end a value inside a JSON line. +var retainedURLPath = regexp.MustCompile(`(https?://[^/?"\\\s]+)[/?][^"\\\s]*`) + +// redactRetainedLine strips the secrets of one line that the journal keeps. +// Support downloads the journal whole, and any error text can carry a DoH URL +// with the resolver path and the packed query of the user. +func redactRetainedLine(p []byte) []byte { + stripped := retainedURLPath.ReplaceAll(p, []byte(`${1}/[redacted]`)) + return []byte(redactSecrets(string(stripped), provisionSecrets()...)) +} + +// journal marks an event for the retained journal stream. The event keeps +// its own level, so an info event stays info on the console. +func journal(e *zerolog.Event) *zerolog.Event { + return e.Bool(journalField, true) +} + +// journalLevelWriter passes on the lines that the journal keeps and drops the +// rest, so the journal holds a small history that survives a restart. +type journalLevelWriter struct { + w io.Writer +} + +var _ zerolog.LevelWriter = (*journalLevelWriter)(nil) + +func newJournalLevelWriter(w io.Writer) *journalLevelWriter { + return &journalLevelWriter{w: w} +} + +// WriteLevel reads the level first, because a compare costs less than a scan +// of the serialized line. +func (j *journalLevelWriter) WriteLevel(l zerolog.Level, p []byte) (int, error) { + if l < zerolog.WarnLevel && !bytes.Contains(p, journalMarker) { + // A dropped line reports a full write, because zerolog.MultiLevelWriter + // turns a short write into an error for the whole group of writers. + return len(p), nil + } + if _, err := j.w.Write(redactRetainedLine(p)); err != nil { + return 0, err + } + // The redacted line is shorter than the line the logger handed over, and a + // short write is an error for the whole group of writers. + return len(p), nil +} + +// Write has no level to read, so the marker alone decides. Every level below +// warn takes that path. +func (j *journalLevelWriter) Write(p []byte) (int, error) { + return j.WriteLevel(zerolog.DebugLevel, p) +} diff --git a/cmd/cli/log_journal_test.go b/cmd/cli/log_journal_test.go new file mode 100644 index 00000000..ff6fd499 --- /dev/null +++ b/cmd/cli/log_journal_test.go @@ -0,0 +1,255 @@ +package cli + +import ( + "bytes" + "io" + "path/filepath" + "strings" + "testing" + + "github.com/rs/zerolog" + + "github.com/Control-D-Inc/ctrld" +) + +func Test_journalSetsMarker(t *testing.T) { + var buf bytes.Buffer + logger := zerolog.New(&buf) + journal(logger.Info()).Msg("marked") + if !bytes.Contains(buf.Bytes(), journalMarker) { + t.Fatalf("journal event lacks the marker: %s", buf.String()) + } + if !bytes.Contains(buf.Bytes(), []byte(`"level":"info"`)) { + t.Fatalf("journal event changed its level: %s", buf.String()) + } +} + +// journalTestLine has the shape of a serialized event, so the direct write +// path sees the marker where zerolog puts it. +func journalTestLine(marked bool) []byte { + if marked { + return []byte(`{"level":"info","` + journalField + `":true,"message":"state"}` + "\n") + } + return []byte(`{"level":"info","message":"state"}` + "\n") +} + +func Test_journalLevelWriterKeepsWarningsAndMarkedLines(t *testing.T) { + level := zerolog.GlobalLevel() + zerolog.SetGlobalLevel(zerolog.DebugLevel) + t.Cleanup(func() { zerolog.SetGlobalLevel(level) }) + + for _, tc := range []struct { + name string + log func(logger zerolog.Logger) + kept bool + }{ + {"info with the marker", func(logger zerolog.Logger) { journal(logger.Info()).Msg("state") }, true}, + {"info without the marker", func(logger zerolog.Logger) { logger.Info().Msg("state") }, false}, + {"warn without the marker", func(logger zerolog.Logger) { logger.Warn().Msg("slow") }, true}, + {"debug with the marker", func(logger zerolog.Logger) { journal(logger.Debug()).Msg("state") }, true}, + {"notice without the marker", func(logger zerolog.Logger) { logger.Notice().Msg("note") }, false}, + {"error without the marker", func(logger zerolog.Logger) { logger.Error().Msg("failed") }, true}, + } { + t.Run(tc.name, func(t *testing.T) { + var buf bytes.Buffer + tc.log(zerolog.New(zerolog.MultiLevelWriter(newJournalLevelWriter(&buf)))) + if kept := buf.Len() > 0; kept != tc.kept { + t.Fatalf("kept = %v, want %v, line: %q", kept, tc.kept, buf.String()) + } + }) + } +} + +func Test_journalLevelWriterWriteKeepsMarkedLineOnly(t *testing.T) { + var buf bytes.Buffer + writer := newJournalLevelWriter(&buf) + + marked := journalTestLine(true) + n, err := writer.Write(marked) + if err != nil || n != len(marked) { + t.Fatalf("write of a marked line = (%d, %v), want (%d, nil)", n, err, len(marked)) + } + if buf.String() != string(marked) { + t.Fatalf("journal holds %q, want %q", buf.String(), marked) + } + + plain := journalTestLine(false) + n, err = writer.Write(plain) + if err != nil || n != len(plain) { + t.Fatalf("write of a plain line = (%d, %v), want (%d, nil)", n, err, len(plain)) + } + if buf.String() != string(marked) { + t.Fatalf("journal kept an unmarked line: %q", buf.String()) + } +} + +// journalDoHError has the shape of a resolver error text: the DoH URL carries +// the resolver path and the packed query. +const journalDoHError = `Get "https://dns.example/org-v1-SECRET0123?dns=pOgBAAAB": dial tcp` + +// journalRedactedURL is what the sink keeps of journalDoHError. +const journalRedactedURL = "https://dns.example/[redacted]" + +// journalTestLogger returns a logger that writes through the journal sink. +func journalTestLogger(t *testing.T) (*bytes.Buffer, zerolog.Logger) { + t.Helper() + level := zerolog.GlobalLevel() + zerolog.SetGlobalLevel(zerolog.DebugLevel) + t.Cleanup(func() { zerolog.SetGlobalLevel(level) }) + var buf bytes.Buffer + return &buf, zerolog.New(zerolog.MultiLevelWriter(newJournalLevelWriter(&buf))) +} + +func Test_journalLevelWriterRedactsAKeptLine(t *testing.T) { + buf, logger := journalTestLogger(t) + + logger.Warn().Str("error", journalDoHError).Msg("upstream failed") + + line := buf.String() + if !strings.Contains(line, journalRedactedURL) { + t.Fatalf("the sink kept the URL path: %s", line) + } + for _, secret := range []string{"SECRET0123", "pOgBAAAB"} { + if strings.Contains(line, secret) { + t.Fatalf("the sink kept %q: %s", secret, line) + } + } +} + +func Test_journalLevelWriterRedactsTheProvisionSecrets(t *testing.T) { + origCdUID := cdUID + t.Cleanup(func() { cdUID = origCdUID }) + cdUID = "org-v1-uidofthisdevice" + buf, logger := journalTestLogger(t) + + logger.Warn().Str("detail", "device "+cdUID+" is gone").Msg("provision failed") + + line := buf.String() + if strings.Contains(line, cdUID) { + t.Fatalf("the sink kept the resolver uid: %s", line) + } + if !strings.Contains(line, "[redacted]") { + t.Fatalf("the sink wrote no redaction mark: %s", line) + } +} + +func Test_journalLevelWriterDropsADebugLineWithAURL(t *testing.T) { + buf, logger := journalTestLogger(t) + + logger.Debug().Str("error", journalDoHError).Msg("upstream failed") + + if buf.Len() != 0 { + t.Fatalf("the sink kept a debug line: %s", buf.String()) + } +} + +func Test_journalLevelWriterRedactsAMarkedInfoLine(t *testing.T) { + buf, logger := journalTestLogger(t) + + journal(logger.Info()).Str("error", journalDoHError).Msg("upstream failed") + + line := buf.String() + if !strings.Contains(line, journalRedactedURL) { + t.Fatalf("the sink kept the URL path: %s", line) + } + if !strings.Contains(line, `"level":"info"`) { + t.Fatalf("the sink changed the level: %s", line) + } +} + +func Test_journalLevelWriterReportsTheOriginalLength(t *testing.T) { + var buf bytes.Buffer + writer := newJournalLevelWriter(&buf) + line := []byte(`{"level":"warn","error":"https://dns.example/org-v1-SECRET0123","message":"state"}` + "\n") + + n, err := writer.WriteLevel(zerolog.WarnLevel, line) + + if err != nil || n != len(line) { + t.Fatalf("write of a kept line = (%d, %v), want (%d, nil)", n, err, len(line)) + } + if buf.Len() >= len(line) { + t.Fatalf("the sink wrote no shorter line: %s", buf.String()) + } +} + +// startInternalLogging drives the real cd mode setup and returns the program +// and the directory that holds the debug file and the journal file. +func startInternalLogging(t *testing.T) (*prog, string) { + t.Helper() + origSilent, origCdUID, origHomedir, origVerbose := silent, cdUID, homedir, verbose + origMainLog, origProxyLog := mainLog.Load(), ctrld.ProxyLogger.Load() + origLevel := zerolog.GlobalLevel() + t.Cleanup(func() { + silent, cdUID, homedir, verbose = origSilent, origCdUID, origHomedir, origVerbose + mainLog.Store(origMainLog) + ctrld.ProxyLogger.Store(origProxyLog) + zerolog.SetGlobalLevel(origLevel) + }) + if origMainLog == nil { + discard := zerolog.New(io.Discard) + mainLog.Store(&discard) + } + + dir := t.TempDir() + homedir = dir + cdUID = "test-uid" + silent = false + verbose = 0 + + p := &prog{cfg: &ctrld.Config{}} + p.initInternalLogging(nil) + t.Cleanup(p.closeInternalLogs) + return p, dir +} + +// Test_initInternalLogging_redactsTheJournalFile drives a cd mode run: support +// downloads the journal file, so it holds the redacted URL, while the debug +// file stays raw for the developer who reads it on the host. +func Test_initInternalLogging_redactsTheJournalFile(t *testing.T) { + _, dir := startInternalLogging(t) + + mainLog.Load().Warn().Str("error", journalDoHError).Msg("upstream failed") + + journalText := rotatingFileTestContent(t, filepath.Join(dir, journalLogFileName)) + if !strings.Contains(journalText, journalRedactedURL) { + t.Fatalf("journal file misses the redacted URL: %s", journalText) + } + if strings.Contains(journalText, "SECRET0123") { + t.Fatalf("journal file kept the resolver path: %s", journalText) + } + + debugText := rotatingFileTestContent(t, filepath.Join(dir, logFileName)) + if !strings.Contains(debugText, "org-v1-SECRET0123?dns=pOgBAAAB") { + t.Fatalf("debug file lost the raw URL: %s", debugText) + } +} + +// Test_stopClosesTheInternalLogFiles proves that a stopped service leaves no +// open handle on the debug file and the journal file. +func Test_stopClosesTheInternalLogFiles(t *testing.T) { + origPin := cdDeactivationPin.Load() + t.Cleanup(func() { cdDeactivationPin.Store(origPin) }) + cdDeactivationPin.Store(defaultDeactivationPin) + + p, _ := startInternalLogging(t) + p.stopCh = make(chan struct{}) + p.dnsWatcherStopCh = make(chan struct{}) + for _, writer := range []*logWriter{p.internalLogWriter, p.internalJournalWriter} { + if writer.rotating() == nil { + t.Fatal("the setup opened no file") + } + } + + if err := p.Stop(nil); err != nil { + t.Fatalf("Stop: %v", err) + } + + for name, writer := range map[string]*logWriter{ + logFileName: p.internalLogWriter, + journalLogFileName: p.internalJournalWriter, + } { + if writer.rotating() != nil { + t.Fatalf("%s stayed open after the stop", name) + } + } +} diff --git a/cmd/cli/log_path.go b/cmd/cli/log_path.go new file mode 100644 index 00000000..b4eb6c4e --- /dev/null +++ b/cmd/cli/log_path.go @@ -0,0 +1,73 @@ +package cli + +import ( + "os" + "sync/atomic" + + "github.com/Control-D-Inc/ctrld/internal/router" +) + +// logPathFile is the open log_path file. The header renderer and the log +// readers need it, and both run outside this file. +var logPathFile atomic.Pointer[rotatingFile] + +// pendingLogPathRotation holds the rotation of a start-time backup. The +// logger reaches no file at that point, so initLogging reports it later. +var pendingLogPathRotation atomic.Pointer[logRotation] + +// openLogPathWriter opens the log_path file of this run and stores it for the +// header renderer and the log readers. It returns nil when no log_path is set. +// A start calls it more than once, so a call for the file that is already open +// keeps that file: a second open would put a second header in it. +func openLogPathWriter(doBackup bool) *rotatingFile { + logFilePath := normalizeLogFilePath(cfg.Service.LogPath) + if logFilePath == "" { + return nil + } + budget := debugLogBudget(&cfg.Service, router.Name() != "") + rf := logPathFile.Load() + if rf == nil || rf.currentPath() != logFilePath { + rf = newLogPathFile(logFilePath, budget) + } else { + rf.setBudget(budget) + } + if doBackup { + backupLogPathFile(rf) + } + return rf +} + +// newLogPathFile opens path and puts it in place of the file of an earlier +// call. A ctrld that cannot write the log file its operator asked for leaves +// no record of the run, so the open failure ends the process. +func newLogPathFile(path string, budget logBudget) *rotatingFile { + rf, err := newRotatingFile(path, budget, nil) + if err != nil { + mainLog.Load().Error().Msgf("failed to create log file: %v", err) + os.Exit(1) + } + rf.onRotate = emitLogRotated + if old := logPathFile.Swap(rf); old != nil { + _ = old.close() + } + return rf +} + +// backupLogPathFile moves the file of an earlier run away, so no run +// overwrites another. +func backupLogPathFile(rf *rotatingFile) { + if !logFileHasContent(rf.currentPath()) { + return + } + record, err := rf.rotateNow() + if err != nil { + mainLog.Load().Error().Msgf("could not backup old log file: %v", err) + return + } + pendingLogPathRotation.Store(record) +} + +func logFileHasContent(path string) bool { + st, err := os.Stat(path) + return err == nil && st.Size() > 0 +} diff --git a/cmd/cli/log_path_rotation_test.go b/cmd/cli/log_path_rotation_test.go new file mode 100644 index 00000000..aad30ded --- /dev/null +++ b/cmd/cli/log_path_rotation_test.go @@ -0,0 +1,187 @@ +package cli + +import ( + "encoding/json" + "io" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/rs/zerolog" + + "github.com/Control-D-Inc/ctrld" +) + +const ( + logPathTestBackups = 2 + logPathTestBudgetMB = 1 + logPathTestLineSize = 4 * 1024 + logPathTestLineCount = 300 +) + +// setupLogPathTest points log_path at a temporary file and puts every global +// that initLoggingWithBackup touches back when the test ends. The globals are +// the reason why no test of this package calls t.Parallel. +// +// The temporary directory comes first: its own cleanup removes the directory, +// and a cleanup runs before every cleanup that was added earlier, so the file +// must close before that. +func setupLogPathTest(t *testing.T) string { + t.Helper() + dir := t.TempDir() + origLogPath, origLogLevel := cfg.Service.LogPath, cfg.Service.LogLevel + origMaxSizeMB, origMaxBackups := cfg.Service.LogMaxSizeMB, cfg.Service.LogMaxBackups + origSilent, origVerbose := silent, verbose + origConsoleWriter := consoleWriter + origMainLog, origProxyLogger := mainLog.Load(), ctrld.ProxyLogger.Load() + origLevel, origTimeFormat := zerolog.GlobalLevel(), zerolog.TimeFieldFormat + origLogPathFile, origPendingRotation := logPathFile.Load(), pendingLogPathRotation.Load() + t.Cleanup(func() { + if rf := logPathFile.Load(); rf != nil && rf != origLogPathFile { + _ = rf.close() + } + logPathFile.Store(origLogPathFile) + pendingLogPathRotation.Store(origPendingRotation) + cfg.Service.LogPath, cfg.Service.LogLevel = origLogPath, origLogLevel + cfg.Service.LogMaxSizeMB, cfg.Service.LogMaxBackups = origMaxSizeMB, origMaxBackups + silent, verbose = origSilent, origVerbose + consoleWriter = origConsoleWriter + mainLog.Store(origMainLog) + ctrld.ProxyLogger.Store(origProxyLogger) + zerolog.SetGlobalLevel(origLevel) + zerolog.TimeFieldFormat = origTimeFormat + }) + + backups := logPathTestBackups + cfg.Service.LogPath = filepath.Join(dir, "ctrld.log") + cfg.Service.LogLevel = "info" + cfg.Service.LogMaxSizeMB = logPathTestBudgetMB + cfg.Service.LogMaxBackups = &backups + silent, verbose = false, 0 + consoleWriter = zerolog.NewConsoleWriter(func(w *zerolog.ConsoleWriter) { w.Out = io.Discard }) + discard := zerolog.New(io.Discard) + mainLog.Store(&discard) + logPathFile.Store(nil) + pendingLogPathRotation.Store(nil) + return cfg.Service.LogPath +} + +func Test_logPathBackupChainOnStart(t *testing.T) { + path := setupLogPathTest(t) + if err := os.WriteFile(path, []byte("run1\n"), 0o600); err != nil { + t.Fatalf("seed log file: %v", err) + } + + initLoggingWithBackup(true) + if _, err := logPathFile.Load().Write([]byte("run2\n")); err != nil { + t.Fatalf("write run2: %v", err) + } + initLoggingWithBackup(true) + + newest := rotatingFileTestContent(t, path+".1") + if !strings.Contains(newest, "run2") { + t.Fatalf("%s.1 = %q, want the run2 line", path, newest) + } + if strings.Contains(newest, "run1") { + t.Fatalf("%s.1 = %q, want no run1 line", path, newest) + } + oldest := rotatingFileTestContent(t, path+".2") + if !strings.Contains(oldest, "run1") { + t.Fatalf("%s.2 = %q, want the run1 line", path, oldest) + } + current := rotatingFileTestContent(t, path) + if strings.Contains(current, "run1") || strings.Contains(current, "run2") { + t.Fatalf("%s = %q, want neither run line", path, current) + } +} + +func Test_logPathRotatesBySize(t *testing.T) { + path := setupLogPathTest(t) + initLoggingWithBackup(false) + + line := strings.Repeat("x", logPathTestLineSize) + for i := 0; i < logPathTestLineCount; i++ { + mainLog.Load().Info().Msg(line) + } + + if _, err := os.Stat(path + ".2"); !os.IsNotExist(err) { + t.Fatalf("%s.2 exists although the lines fill one backup (stat err = %v)", path, err) + } + want := []string{path + ".1", path} + if got := logPathFile.Load().paths(); !slices.Equal(got, want) { + t.Fatalf("paths() = %v, want %v", got, want) + } + + rotatedLines := 0 + for _, logLine := range strings.Split(rotatingFileTestContent(t, path), "\n") { + if !strings.Contains(logLine, "Log rotated") { + continue + } + rotatedLines++ + if !strings.Contains(logLine, string(journalMarker)) { + t.Fatalf("Log rotated line %q carries no journal marker", logLine) + } + } + if rotatedLines != 1 { + t.Fatalf("Log rotated lines in %s = %d, want 1", path, rotatedLines) + } +} + +func Test_logPathHeaderLeadsEveryFile(t *testing.T) { + path := setupLogPathTest(t) + stubLogHeaderNetworkRead(t, "en0") + p := &prog{cfg: &cfg} + + // A start wires its logging twice: once for the config it reads from + // disk, and once more for the config that the API sends. + p.initLogging(true) + p.initLogging(false) + + line := strings.Repeat("x", logPathTestLineSize) + for i := 0; i < logPathTestLineCount; i++ { + mainLog.Load().Info().Msg(line) + } + + for _, name := range []string{path + ".1", path} { + firstLine, headers := logHeaderFileLines(t, name) + if firstLine.Message != logHeaderMessage { + t.Errorf("line 1 of %s = %q, want %q", name, firstLine.Message, logHeaderMessage) + } + if headers != 1 { + t.Errorf("%s holds %d header lines, want 1", name, headers) + } + } +} + +func Test_logPathStartBackupLogsRotation(t *testing.T) { + path := setupLogPathTest(t) + stubLogHeaderNetworkRead(t, "en0") + if err := os.WriteFile(path, []byte(`{"level":"info","message":"run1"}`+"\n"), 0o600); err != nil { + t.Fatalf("seed log file: %v", err) + } + p := &prog{cfg: &cfg} + + p.initLogging(true) + + rotatedLines := 0 + for _, logLine := range strings.Split(rotatingFileTestContent(t, path), "\n") { + if !strings.Contains(logLine, "Log rotated") { + continue + } + rotatedLines++ + var event struct { + File string `json:"file"` + } + if err := json.Unmarshal([]byte(logLine), &event); err != nil { + t.Fatalf("parse %q: %v", logLine, err) + } + if event.File != path+".1" { + t.Errorf("file = %q, want %q", event.File, path+".1") + } + } + if rotatedLines != 1 { + t.Fatalf("Log rotated lines in %s = %d, want 1", path, rotatedLines) + } +} diff --git a/cmd/cli/log_reader.go b/cmd/cli/log_reader.go new file mode 100644 index 00000000..fd0ce402 --- /dev/null +++ b/cmd/cli/log_reader.go @@ -0,0 +1,291 @@ +package cli + +import ( + "bytes" + "errors" + "io" + "os" + "strings" +) + +// logSendDebugBudget bounds the debug bytes of one upload. The journal is +// always whole, so an upload stays small and keeps the retained stream. A test +// lowers it, because a few small files must cross it. +var logSendDebugBudget int64 = 10 << 20 + +// lineScanBufferSize bounds the memory that the search for a line boundary +// needs, whatever the length of the line. +const lineScanBufferSize = 32 * 1024 + +var ( + errInternalLogEmpty = errors.New("internal log is empty") + errLogFileEmpty = errors.New("log file is empty") +) + +type logReader struct { + r io.ReadCloser + size int64 +} + +// logFilePart is the part of one open log file that an upload carries. An +// offset of 0 means the whole file. size is the size at the open, so a write +// after the selection stays out of the upload and the budget holds. +type logFilePart struct { + path string + f *os.File + offset int64 + size int64 +} + +// logReader composes the upload of the current logging mode and leads it with +// the header of this request. full lifts the bound on the debug bytes. +func (p *prog) logReader(full bool) (*logReader, error) { + lr, err := p.modeLogReader(full) + if err != nil { + return nil, err + } + // An empty upload must stay empty, because the handlers answer 301 on a + // size of 0. + if lr.size == 0 { + return lr, nil + } + header := p.sendLogHeader() + return &logReader{ + r: &multiCloser{ + Reader: io.MultiReader(bytes.NewReader(header), lr.r), + closers: []io.Closer{lr.r}, + }, + size: lr.size + int64(len(header)), + }, nil +} + +// modeLogReader composes the body of the current logging mode. +func (p *prog) modeLogReader(full bool) (*logReader, error) { + if p.needInternalLogging() { + return p.internalLogReader(full) + } + return p.logPathReader(full) +} + +// internalLogReader puts the newest debug bytes first, then the marker, then +// the journal, so a support reader finds the retained stream after the marker. +func (p *prog) internalLogReader(full bool) (*logReader, error) { + lw, jlw := p.internalWriters() + if lw == nil { + return nil, errors.New("nil internal log writer") + } + if jlw == nil { + return nil, errors.New("nil internal journal writer") + } + + var upload logParts + debugBytes := upload.addStream(lw, logSendBudget(full)) + upload.addBytes([]byte(logWriterLogEndMarker)) + journalBytes := upload.addStream(jlw, 0) + // The marker alone is not a log, and the handlers answer 301 on an empty + // upload. + if debugBytes+journalBytes == 0 { + upload.close() + return nil, errInternalLogEmpty + } + return upload.reader(errInternalLogEmpty) +} + +// logPathReader composes the upload of the log_path stream. The journal needs +// internal logging, so this upload holds the debug files alone. A run that +// opened no log_path file has nothing to send. +func (p *prog) logPathReader(full bool) (*logReader, error) { + rf := logPathFile.Load() + if p.cfg.Service.LogPath == "" || rf == nil { + return emptyLogReader(), nil + } + var upload logParts + upload.addBoundedFiles(rf.paths(), logSendBudget(full)) + return upload.reader(errLogFileEmpty) +} + +// emptyLogReader yields no bytes, so the handlers answer 301 instead of +// sending an upload without a line. +func emptyLogReader() *logReader { + return &logReader{r: io.NopCloser(strings.NewReader(""))} +} + +// logSendBudget reports the debug bytes that one upload may carry. Zero means +// every file. +func logSendBudget(full bool) int64 { + if full { + return 0 + } + return logSendDebugBudget +} + +// boundedLogFiles opens the files, reads each size once, and picks the newest +// bytes that fit in budget. files come oldest first, and so do the parts. The +// oldest part starts at a line boundary, so the upload holds whole lines only. +// A budget of 0 or less takes every file. The files that fall out of the +// budget close here; the parts own the files they hold. +func boundedLogFiles(files []string, budget int64) []logFilePart { + parts := openLogParts(files) + remaining := budget + first := len(parts) + for first > 0 { + part := &parts[first-1] + if budget <= 0 || part.size <= remaining { + remaining -= part.size + first-- + continue + } + if offset, ok := lineStartAt(part.f, part.size-remaining); ok { + part.offset = offset + first-- + } + // Without a line boundary in the tail, the upload would start inside a + // line, so the file stays out. + break + } + for _, part := range parts[:first] { + part.f.Close() + } + return parts[first:] +} + +// openLogParts opens each file through the platform opener, whose share flags +// let a rotation move the file away while Windows reads it. A rotation can +// also move a file away between the listing and the open, so a file that does +// not open must not stop the rest. +func openLogParts(files []string) []logFilePart { + parts := make([]logFilePart, 0, len(files)) + for _, path := range files { + f, err := openLogFile(path, os.O_RDONLY) + if err != nil { + continue + } + st, err := f.Stat() + if err != nil { + f.Close() + continue + } + parts = append(parts, logFilePart{path: path, f: f, size: st.Size()}) + } + return parts +} + +// lineStartAt returns the offset where a whole line starts at or after from. +// A cut that already lands on a line start keeps the line that follows it. It +// reports false when no newline follows. +func lineStartAt(f *os.File, from int64) (int64, bool) { + if startsLine(f, from) { + return from, true + } + if _, err := f.Seek(from, io.SeekStart); err != nil { + return 0, false + } + buf := make([]byte, lineScanBufferSize) + offset := from + for { + n, readErr := f.Read(buf) + if index := bytes.IndexByte(buf[:n], '\n'); index >= 0 { + return offset + int64(index) + 1, true + } + offset += int64(n) + if readErr != nil { + return 0, false + } + } +} + +// startsLine reports whether from is the first byte of a line. +func startsLine(f *os.File, from int64) bool { + if from == 0 { + return true + } + var previous [1]byte + if _, err := f.ReadAt(previous[:], from-1); err != nil { + return false + } + return previous[0] == '\n' +} + +// logParts collects the readers of one upload and the files it must close. +type logParts struct { + readers []io.Reader + closers []io.Closer + size int64 +} + +func (lp *logParts) addBytes(b []byte) { + if len(b) == 0 { + return + } + lp.readers = append(lp.readers, bytes.NewReader(b)) + lp.size += int64(len(b)) +} + +// addFiles takes the open parts. Each reader ends at the size of the open, so +// the bytes that a writer adds later stay out of this upload. +func (lp *logParts) addFiles(parts []logFilePart) { + for _, part := range parts { + length := part.size - part.offset + if length <= 0 { + part.f.Close() + continue + } + lp.readers = append(lp.readers, io.NewSectionReader(part.f, part.offset, length)) + lp.closers = append(lp.closers, part.f) + lp.size += length + } +} + +// addBoundedFiles adds the newest bytes of files that fit in budget. A budget +// of 0 or less takes every file. +func (lp *logParts) addBoundedFiles(files []string, budget int64) { + lp.addFiles(boundedLogFiles(files, budget)) +} + +// addStream adds the bytes of one stream and reports how many it added. A +// stream that has a file takes its files, and a stream that keeps its lines in +// memory takes its buffer, so one failed file never hides the other stream. +func (lp *logParts) addStream(lw *logWriter, budget int64) int64 { + before := lp.size + if rf := lw.rotating(); rf != nil { + lp.addBoundedFiles(rf.paths(), budget) + return lp.size - before + } + lp.addBytes(lw.bufferedBytes()) + return lp.size - before +} + +// reader joins the parts. The size is the number of bytes that the reader +// yields, so the caller reports what it uploads. +func (lp *logParts) reader(empty error) (*logReader, error) { + if lp.size == 0 { + lp.close() + return nil, empty + } + return &logReader{ + r: &multiCloser{Reader: io.MultiReader(lp.readers...), closers: lp.closers}, + size: lp.size, + }, nil +} + +func (lp *logParts) close() { + for _, c := range lp.closers { + c.Close() + } +} + +// multiCloser wraps an io.Reader and closes multiple underlying closers. +type multiCloser struct { + io.Reader + closers []io.Closer +} + +func (mc *multiCloser) Close() error { + var firstErr error + for _, c := range mc.closers { + if err := c.Close(); err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr +} diff --git a/cmd/cli/log_reader_test.go b/cmd/cli/log_reader_test.go new file mode 100644 index 00000000..369401a4 --- /dev/null +++ b/cmd/cli/log_reader_test.go @@ -0,0 +1,572 @@ +package cli + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "slices" + "strings" + "testing" + "time" + + "github.com/rs/zerolog" + + "github.com/Control-D-Inc/ctrld" +) + +const ( + boundedLogTestLineSize = 100 + boundedLogTestLineCount = 30 + boundedLogTestFileCount = 5 + boundedLogTestBudget = 10000 +) + +// boundedLogTestLine returns a line of a fixed size that names its file, so a +// test can tell one part from another. +func boundedLogTestLine(file, line int) []byte { + head := fmt.Sprintf("file=%d line=%d ", file, line) + return []byte(head + strings.Repeat("x", boundedLogTestLineSize-len(head)-1) + "\n") +} + +// writeBoundedLogTestFiles creates the log files of one stream, oldest first. +func writeBoundedLogTestFiles(t *testing.T) []string { + t.Helper() + dir := t.TempDir() + files := make([]string, 0, boundedLogTestFileCount) + for file := 0; file < boundedLogTestFileCount; file++ { + var content bytes.Buffer + for line := 0; line < boundedLogTestLineCount; line++ { + content.Write(boundedLogTestLine(file, line)) + } + path := filepath.Join(dir, fmt.Sprintf("ctrld.log.%d", boundedLogTestFileCount-file)) + if err := os.WriteFile(path, content.Bytes(), 0o600); err != nil { + t.Fatalf("write %s: %v", path, err) + } + files = append(files, path) + } + return files +} + +func logFilesContent(t *testing.T, files []string) []byte { + t.Helper() + var joined []byte + for _, path := range files { + joined = append(joined, []byte(rotatingFileTestContent(t, path))...) + } + return joined +} + +// boundedLogTestFileSize is the size of each file that the bounded reader +// tests select from. +const boundedLogTestFileSize = int64(boundedLogTestLineCount * boundedLogTestLineSize) + +// wantTailLine reports the first line that fits in budget, for a budget that +// ends inside the fourth newest file. A cut that lands on a line start keeps +// that line. +func wantTailLine(budget int64) int { + cut := boundedLogTestFileSize - (budget - 3*boundedLogTestFileSize) + line := cut / boundedLogTestLineSize + if cut%boundedLogTestLineSize != 0 { + line++ + } + return int(line) +} + +// assertTailPart checks that a part starts at a line boundary and that it +// starts with the expected whole line. +func assertTailPart(t *testing.T, part logFilePart, file, line int) { + t.Helper() + content := []byte(rotatingFileTestContent(t, part.path)) + if part.offset <= 0 || part.offset >= int64(len(content)) { + t.Fatalf("offset = %d, want a position inside %s", part.offset, part.path) + } + if content[part.offset-1] != '\n' { + t.Fatalf("offset %d in %s does not follow a newline", part.offset, part.path) + } + if want := boundedLogTestLine(file, line); !bytes.HasPrefix(content[part.offset:], want) { + t.Fatalf("tail of %s starts with %.24q, want %.24q", part.path, content[part.offset:], want) + } +} + +// closeLogParts closes the files of parts that no upload took. +func closeLogParts(parts []logFilePart) { + for _, part := range parts { + part.f.Close() + } +} + +// partsSize reports the bytes that the parts hold on disk. +func partsSize(t *testing.T, parts []logFilePart) int64 { + t.Helper() + var size int64 + for _, part := range parts { + content := rotatingFileTestContent(t, part.path) + size += int64(len(content)) - part.offset + } + return size +} + +// assertBoundedParts checks the shape that every bounded selection shares: +// the newest three files whole, oldest first, after the tail of the fourth. +func assertBoundedParts(t *testing.T, files []string, parts []logFilePart, tailLine int) { + t.Helper() + wantPaths := files[1:] + if len(parts) != len(wantPaths) { + t.Fatalf("parts = %d, want %d", len(parts), len(wantPaths)) + } + for i, part := range parts { + if part.path != wantPaths[i] { + t.Fatalf("parts[%d].path = %s, want %s", i, part.path, wantPaths[i]) + } + } + assertTailPart(t, parts[0], 1, tailLine) + for i, part := range parts[1:] { + if part.offset != 0 { + t.Fatalf("parts[%d].offset = %d, want the whole file", i+1, part.offset) + } + } +} + +func Test_boundedLogFiles(t *testing.T) { + files := writeBoundedLogTestFiles(t) + + t.Run("budget ends at a line boundary", func(t *testing.T) { + parts := boundedLogFiles(files, boundedLogTestBudget) + t.Cleanup(func() { closeLogParts(parts) }) + + assertBoundedParts(t, files, parts, wantTailLine(boundedLogTestBudget)) + // The cut lands on a line start, so the tail begins at the cut and the + // parts fill the budget exactly. + if size := partsSize(t, parts); size != boundedLogTestBudget { + t.Fatalf("size = %d, want the whole budget of %d", size, boundedLogTestBudget) + } + }) + + t.Run("budget ends inside a line", func(t *testing.T) { + const budget = boundedLogTestBudget + boundedLogTestLineSize/2 + parts := boundedLogFiles(files, budget) + t.Cleanup(func() { closeLogParts(parts) }) + + assertBoundedParts(t, files, parts, wantTailLine(budget)) + if size := partsSize(t, parts); size > budget { + t.Fatalf("size = %d, want at most %d", size, budget) + } + }) + + t.Run("no budget keeps every file", func(t *testing.T) { + parts := boundedLogFiles(files, 0) + t.Cleanup(func() { closeLogParts(parts) }) + + if len(parts) != len(files) { + t.Fatalf("parts = %d, want %d", len(parts), len(files)) + } + for i, part := range parts { + if part.path != files[i] || part.offset != 0 { + t.Fatalf("parts[%d] = %+v, want %s whole", i, part, files[i]) + } + } + want := int64(boundedLogTestFileCount) * boundedLogTestFileSize + if size := partsSize(t, parts); size != want { + t.Fatalf("size = %d, want %d", size, want) + } + }) +} + +// Test_logReaderDoesNotFollowASymlink covers the one open of the reader. A +// symlink in place of a log file points at a file of another owner, and the +// writer refuses it in the same way. +func Test_logReaderDoesNotFollowASymlink(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("the Windows opener takes no symlink flag") + } + dir := t.TempDir() + planted := filepath.Join(dir, "planted.log") + if err := os.WriteFile(planted, []byte("planted line\n"), 0600); err != nil { + t.Fatalf("write the planted file: %v", err) + } + path := filepath.Join(dir, "ctrld.log") + if err := os.Symlink(planted, path); err != nil { + t.Fatalf("symlink: %v", err) + } + + parts := boundedLogFiles([]string{path}, 4) + if len(parts) != 0 { + t.Fatalf("the selection opened a symlink: %+v", parts) + } + var upload logParts + upload.addFiles(parts) + if upload.size != 0 { + t.Fatalf("the upload holds %d bytes of a symlink, want none", upload.size) + } +} + +// internalLogReaderTestBudget keeps the files small: four lines fill one file, +// so a few writes build the whole backup chain. +var internalLogReaderTestBudget = logBudget{maxSize: 4 * rotatingFileTestLineSize, backups: 2} + +const internalLogReaderTestLines = 12 + +// newTestStreamWriter opens one persisted stream in the temporary home +// directory. +func newTestStreamWriter(t *testing.T, name string) *logWriter { + t.Helper() + lw := newLogWriterWithSize(logWriterSize) + if err := lw.setLogFile(absHomeDir(name), internalLogReaderTestBudget); err != nil { + t.Fatalf("setLogFile %s: %v", name, err) + } + t.Cleanup(lw.closeLogFile) + return lw +} + +// internalLogReaderTestSendBudget holds one debug file and a half, so the +// default upload cuts inside the oldest of the three files. +var internalLogReaderTestSendBudget = internalLogReaderTestBudget.maxSize * 3 / 2 + +// useTestSendBudget lowers the bound on the debug bytes of one upload, so a +// few small files cross it. +func useTestSendBudget(t *testing.T, budget int64) { + t.Helper() + origBudget := logSendDebugBudget + t.Cleanup(func() { logSendDebugBudget = origBudget }) + logSendDebugBudget = budget +} + +// setupInternalLogReaderTest returns a cd mode prog whose debug stream and +// journal stream both hold a full backup chain on disk. +func setupInternalLogReaderTest(t *testing.T) *prog { + t.Helper() + origSilent, origCdUID, origHomedir := silent, cdUID, homedir + t.Cleanup(func() { silent, cdUID, homedir = origSilent, origCdUID, origHomedir }) + captureMainLog(t) + setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + stubLogHeaderNetworkRead(t, "en9") + silent, cdUID, homedir = false, "test-uid", t.TempDir() + + p := &prog{cfg: &ctrld.Config{}} + p.internalLogWriter = newTestStreamWriter(t, logFileName) + p.internalJournalWriter = newTestStreamWriter(t, journalLogFileName) + for seq := 1; seq <= internalLogReaderTestLines; seq++ { + if _, err := p.internalLogWriter.Write(rotatingFileTestLine(1, seq)); err != nil { + t.Fatalf("write debug line %d: %v", seq, err) + } + if _, err := p.internalJournalWriter.Write(rotatingFileTestLine(2, seq)); err != nil { + t.Fatalf("write journal line %d: %v", seq, err) + } + } + return p +} + +// readLogReader reads one upload and checks that the reported size is the +// number of bytes it yields. +func readLogReader(t *testing.T, p *prog, full bool) []byte { + t.Helper() + lr, err := p.logReader(full) + if err != nil { + t.Fatalf("logReader(%v): %v", full, err) + } + defer lr.r.Close() + data, err := io.ReadAll(lr.r) + if err != nil { + t.Fatalf("read upload: %v", err) + } + if lr.size != int64(len(data)) { + t.Fatalf("size = %d, want %d", lr.size, len(data)) + } + return data +} + +// splitSendHeader takes the header line off the front of an upload and checks +// that it names this request. It returns the rest of the upload. +func splitSendHeader(t *testing.T, upload []byte, since time.Time) []byte { + t.Helper() + end := bytes.IndexByte(upload, '\n') + if end < 0 { + t.Fatalf("upload holds no header line: %.64q", upload) + } + parsed, _ := parseLogHeader(t, upload[:end+1]) + if parsed.Message != logHeaderMessage { + t.Fatalf("line 1 of the upload = %q, want %q", parsed.Message, logHeaderMessage) + } + if parsed.Trigger != "send" { + t.Fatalf("trigger = %q, want %q", parsed.Trigger, "send") + } + rendered, err := time.Parse(time.RFC3339, parsed.Time) + if err != nil { + t.Fatalf("parse header time %q: %v", parsed.Time, err) + } + // The header format drops the fraction of a second, so the window starts + // at the second that holds since. + if rendered.Before(since.Truncate(time.Second)) || rendered.After(time.Now().Add(time.Second)) { + t.Fatalf("header time = %s, want a time between %s and now", rendered, since) + } + return upload[end+1:] +} + +// splitUpload takes the header off an upload and cuts the rest at the log end +// marker, which parts the debug bytes from the journal bytes. +func splitUpload(t *testing.T, upload []byte, since time.Time) (debugPart, journalPart []byte) { + t.Helper() + body := splitSendHeader(t, upload, since) + debugPart, journalPart, found := bytes.Cut(body, []byte(logWriterLogEndMarker)) + if !found { + t.Fatalf("upload holds no log end marker: %.64q", body) + } + return debugPart, journalPart +} + +func Test_logRequestPath(t *testing.T) { + if got := logRequestPath(sendLogsPath, false); got != sendLogsPath { + t.Fatalf("logRequestPath(%q, false) = %q, want the path itself", sendLogsPath, got) + } + want := viewLogsPath + "?full=1" + if got := logRequestPath(viewLogsPath, true); got != want { + t.Fatalf("logRequestPath(%q, true) = %q, want %q", viewLogsPath, got, want) + } +} + +func Test_logReader_internalComposition(t *testing.T) { + p := setupInternalLogReaderTest(t) + useTestSendBudget(t, internalLogReaderTestSendBudget) + + debugFiles := p.internalLogWriter.rotating().paths() + journalFiles := p.internalJournalWriter.rotating().paths() + wantFiles := internalLogReaderTestBudget.backups + 1 + if len(debugFiles) != wantFiles || len(journalFiles) != wantFiles { + t.Fatalf("files: debug %v, journal %v, want %d of each", debugFiles, journalFiles, wantFiles) + } + wholeDebug := logFilesContent(t, debugFiles) + wholeJournal := logFilesContent(t, journalFiles) + + start := time.Now() + debugPart, journalPart := splitUpload(t, readLogReader(t, p, false), start) + + if int64(len(debugPart)) > logSendDebugBudget { + t.Fatalf("debug part = %d bytes, want at most the budget of %d", len(debugPart), logSendDebugBudget) + } + if len(debugPart) >= len(wholeDebug) { + t.Fatalf("debug part = %d bytes, want fewer than the %d bytes of %v", len(debugPart), len(wholeDebug), debugFiles) + } + if !bytes.HasSuffix(wholeDebug, debugPart) { + t.Fatalf("debug part = %.24q, want the newest %d bytes of %v", debugPart, len(debugPart), debugFiles) + } + cut := len(wholeDebug) - len(debugPart) + if wholeDebug[cut-1] != '\n' { + t.Fatalf("the debug part starts at %d, which does not follow a newline", cut) + } + if !bytes.Equal(journalPart, wholeJournal) { + t.Fatalf("journal part = %d bytes, want the %d bytes of %v", len(journalPart), len(wholeJournal), journalFiles) + } + + // The full upload lifts the bound, so it holds every debug file. + fullDebug, fullJournal := splitUpload(t, readLogReader(t, p, true), start) + if !bytes.Equal(fullDebug, wholeDebug) { + t.Fatalf("full debug part = %d bytes, want the %d bytes of %v", len(fullDebug), len(wholeDebug), debugFiles) + } + if !bytes.Equal(fullJournal, wholeJournal) { + t.Fatalf("full journal part = %d bytes, want the %d bytes of %v", len(fullJournal), len(wholeJournal), journalFiles) + } +} + +func Test_logReader_logPathComposition(t *testing.T) { + path := setupLogPathTest(t) + stubLogHeaderNetworkRead(t, "en9") + initLoggingWithBackup(false) + + line := strings.Repeat("x", logPathTestLineSize) + for i := 0; i < logPathTestLineCount; i++ { + mainLog.Load().Info().Msg(line) + } + + files := logPathFile.Load().paths() + if want := []string{path + ".1", path}; !slices.Equal(files, want) { + t.Fatalf("paths() = %v, want %v", files, want) + } + + p := &prog{cfg: &cfg} + start := time.Now() + got := splitSendHeader(t, readLogReader(t, p, false), start) + if want := logFilesContent(t, files); !bytes.Equal(got, want) { + t.Fatalf("upload = %d bytes, want the %d bytes of %v", len(got), len(want), files) + } + if bytes.Contains(got, []byte(logWriterLogEndMarker)) { + t.Fatalf("log_path upload holds the log end marker") + } +} + +// setupMemoryLogReaderTest returns a cd mode prog whose streams keep their +// lines in memory, because no file is open. +func setupMemoryLogReaderTest(t *testing.T) *prog { + t.Helper() + origSilent, origCdUID := silent, cdUID + t.Cleanup(func() { silent, cdUID = origSilent, origCdUID }) + captureMainLog(t) + setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + stubLogHeaderNetworkRead(t, "en9") + silent, cdUID = false, "test-uid" + + p := &prog{cfg: &ctrld.Config{}} + p.internalLogWriter = newLogWriterWithSize(logWriterSize) + p.internalJournalWriter = newLogWriterWithSize(logWriterSize) + return p +} + +func Test_logReader_memoryFallbackStartsWithHeader(t *testing.T) { + p := setupMemoryLogReaderTest(t) + debugLine := []byte("debug line\n") + journalLine := []byte("journal line\n") + if _, err := p.internalLogWriter.Write(debugLine); err != nil { + t.Fatalf("write the debug buffer: %v", err) + } + if _, err := p.internalJournalWriter.Write(journalLine); err != nil { + t.Fatalf("write the journal buffer: %v", err) + } + + start := time.Now() + got := splitSendHeader(t, readLogReader(t, p, false), start) + + var want []byte + want = append(want, debugLine...) + want = append(want, []byte(logWriterLogEndMarker)...) + want = append(want, journalLine...) + if !bytes.Equal(got, want) { + t.Fatalf("upload after the header = %q, want %q", got, want) + } +} + +// useTestHomeDir points the ctrld home directory at a temporary directory, so +// the internal log files of a test never reach the real one. +func useTestHomeDir(t *testing.T) { + t.Helper() + origHomedir := homedir + t.Cleanup(func() { homedir = origHomedir }) + homedir = t.TempDir() +} + +// newSeededJournalWriter writes the journal files of an earlier run and +// returns a new writer over them. Its memory buffer is empty, which is what a +// restart finds, so a rule that reads the buffer loses these lines. +func newSeededJournalWriter(t *testing.T, lines int) *logWriter { + t.Helper() + previous := newTestStreamWriter(t, journalLogFileName) + for seq := 1; seq <= lines; seq++ { + if _, err := previous.Write(rotatingFileTestLine(2, seq)); err != nil { + t.Fatalf("write journal line %d: %v", seq, err) + } + } + previous.closeLogFile() + return newTestStreamWriter(t, journalLogFileName) +} + +// mixedStreamJournalLines fill two journal files, because four lines fill one. +const mixedStreamJournalLines = 6 + +func Test_logReader_mixedStreams(t *testing.T) { + p := setupMemoryLogReaderTest(t) + useTestHomeDir(t) + p.internalJournalWriter = newSeededJournalWriter(t, mixedStreamJournalLines) + debugLine := []byte("debug line\n") + if _, err := p.internalLogWriter.Write(debugLine); err != nil { + t.Fatalf("write the debug buffer: %v", err) + } + journalFiles := p.internalJournalWriter.rotating().paths() + if len(journalFiles) != 2 { + t.Fatalf("journal files = %v, want two", journalFiles) + } + + start := time.Now() + got := splitSendHeader(t, readLogReader(t, p, false), start) + + var want []byte + want = append(want, debugLine...) + want = append(want, []byte(logWriterLogEndMarker)...) + want = append(want, logFilesContent(t, journalFiles)...) + if !bytes.Equal(got, want) { + t.Fatalf("upload after the header = %d bytes, want the %d bytes of the debug buffer, the marker and %v", len(got), len(want), journalFiles) + } +} + +func Test_logReader_emptyStaysEmpty(t *testing.T) { + t.Run("internal mode without a line", func(t *testing.T) { + p := setupMemoryLogReaderTest(t) + + lr, err := p.logReader(false) + + if !errors.Is(err, errInternalLogEmpty) { + t.Fatalf("logReader error = %v, want %v", err, errInternalLogEmpty) + } + if lr != nil { + t.Fatalf("logReader = %+v, want no reader", lr) + } + }) + + t.Run("internal mode with empty files", func(t *testing.T) { + p := setupMemoryLogReaderTest(t) + useTestHomeDir(t) + p.internalLogWriter = newTestStreamWriter(t, logFileName) + p.internalJournalWriter = newTestStreamWriter(t, journalLogFileName) + + lr, err := p.logReader(false) + + if !errors.Is(err, errInternalLogEmpty) { + t.Fatalf("logReader error = %v, want %v", err, errInternalLogEmpty) + } + if lr != nil { + t.Fatalf("logReader = %+v, want no reader", lr) + } + }) + + t.Run("no log_path to read", func(t *testing.T) { + origSilent, origCdUID := silent, cdUID + t.Cleanup(func() { silent, cdUID = origSilent, origCdUID }) + setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + stubLogHeaderNetworkRead(t, "en9") + silent, cdUID = false, "" + + p := &prog{cfg: &ctrld.Config{}} + lr, err := p.logReader(false) + if err != nil { + t.Fatalf("logReader: %v", err) + } + defer lr.r.Close() + + if lr.size != 0 { + t.Fatalf("size = %d, want 0 so the handler answers 301", lr.size) + } + data, err := io.ReadAll(lr.r) + if err != nil { + t.Fatalf("read the upload: %v", err) + } + if len(data) != 0 { + t.Fatalf("upload = %q, want no bytes", data) + } + }) +} + +// Test_boundedLogFilesFixTheSizeAtSelection appends through the real writer +// between the selection and the reader creation. The upload must hold the +// bytes that the selection counted, so the budget holds under a live writer. +func Test_boundedLogFilesFixTheSizeAtSelection(t *testing.T) { + rf := newTestRotatingFile(t, logBudget{maxSize: 1 << 20, backups: 1}) + if _, err := rf.Write([]byte("one\n")); err != nil { + t.Fatal(err) + } + parts := boundedLogFiles(rf.paths(), 4) + if _, err := rf.Write([]byte("two\n")); err != nil { + t.Fatal(err) + } + + var upload logParts + upload.addFiles(parts) + t.Cleanup(upload.close) + got, err := io.ReadAll(io.MultiReader(upload.readers...)) + if err != nil { + t.Fatal(err) + } + if upload.size != 4 || string(got) != "one\n" { + t.Fatalf("upload = %q (%d bytes), want %q within the budget of 4", got, upload.size, "one\n") + } +} diff --git a/cmd/cli/log_tail_follow.go b/cmd/cli/log_tail_follow.go new file mode 100644 index 00000000..045b4e18 --- /dev/null +++ b/cmd/cli/log_tail_follow.go @@ -0,0 +1,80 @@ +package cli + +import ( + "context" + "io" + "net/http" + "os" + "time" +) + +// logTailPollInterval is the time between two reads of a tailed log file. +const logTailPollInterval = 200 * time.Millisecond + +// followLogFile streams the lines of path until ctx ends. It first sends the +// last numLines lines, or starts at the end when numLines is 0. A rotation +// moves the file away or truncates it, so the poll reopens the path when the +// file behind it changes or shrinks. +func followLogFile(ctx context.Context, w io.Writer, flusher http.Flusher, path string, numLines int) { + f, err := os.Open(path) + if err != nil { + return + } + defer func() { f.Close() }() + + if numLines > 0 { + if tail := tailFileLastLines(f, numLines); len(tail) > 0 { + w.Write(tail) + flusher.Flush() + } + } else { + f.Seek(0, io.SeekEnd) + } + + buf := make([]byte, 4096) + ticker := time.NewTicker(logTailPollInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + n, err := f.Read(buf) + if n > 0 { + if _, werr := w.Write(buf[:n]); werr != nil { + return + } + flusher.Flush() + } + if err != nil && err != io.EOF { + return + } + if n == 0 && logFileReplaced(f, path) { + reopened, err := os.Open(path) + if err != nil { + continue + } + f.Close() + f = reopened + } + case <-ctx.Done(): + return + } + } +} + +// logFileReplaced reports whether path names another file than f, or a file +// shorter than the read position. Both mean that a rotation happened. +func logFileReplaced(f *os.File, path string) bool { + current, err := os.Stat(path) + if err != nil { + return false + } + open, err := f.Stat() + if err != nil { + return true + } + if !os.SameFile(current, open) { + return true + } + position, err := f.Seek(0, io.SeekCurrent) + return err == nil && current.Size() < position +} diff --git a/cmd/cli/log_writer.go b/cmd/cli/log_writer.go index 201968a3..73b7f307 100644 --- a/cmd/cli/log_writer.go +++ b/cmd/cli/log_writer.go @@ -2,18 +2,14 @@ package cli import ( "bytes" - "errors" - "fmt" "io" - "os" - "path/filepath" - "strings" "sync" "time" "github.com/rs/zerolog" "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/router" ) const ( @@ -24,8 +20,7 @@ const ( logWriterInitEndMarker = "\n\n=== INIT_END ===\n\n" logWriterLogEndMarker = "\n\n=== LOG_END ===\n\n" - logFileName = "ctrld.log" - logFileMaxSize = 1024 * 1024 * 5 // 5 MB + logFileName = "ctrld.log" ) type logViewResponse struct { @@ -37,29 +32,21 @@ type logSentResponse struct { Error string `json:"error"` } -type logReader struct { - r io.ReadCloser - size int64 -} - // logSubscriber represents a subscriber to live log output. type logSubscriber struct { ch chan []byte } // logWriter is an internal buffer to keep track of runtime log when no logging is enabled. -// When a file path is configured via setLogFile, writes are also persisted to -// a rotated file on disk (max logFileMaxSize, 1 backup) so logs survive restarts. +// When a file is configured via setLogFile, writes also go to that file, which +// rotates within its budget, so logs survive restarts. type logWriter struct { mu sync.Mutex buf bytes.Buffer size int subscribers []*logSubscriber - // File persistence fields. - logFile *os.File - logFilePath string - logFileSize int64 + file *rotatingFile } // newLogWriter creates an internal log writer. @@ -74,82 +61,50 @@ func newSmallLogWriter() *logWriter { // newLogWriterWithSize creates an internal log writer with a given buffer size. func newLogWriterWithSize(size int) *logWriter { - lw := &logWriter{size: size} - return lw + return &logWriter{size: size} } -// setLogFile configures file-backed persistence for the log writer. -// The directory is created if it does not exist. An existing file is -// opened in append mode and its current size is tracked for rotation. -func (lw *logWriter) setLogFile(path string) error { - dir := filepath.Dir(path) - if err := os.MkdirAll(dir, 0750); err != nil { - return fmt.Errorf("creating log directory: %w", err) - } - f, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0600) +// setLogFile persists the stream to a file that rotates within the budget. +func (lw *logWriter) setLogFile(path string, budget logBudget) error { + rf, err := newRotatingFile(path, budget, nil) if err != nil { - return fmt.Errorf("opening log file: %w", err) - } - st, err := f.Stat() - if err != nil { - f.Close() - return fmt.Errorf("stat log file: %w", err) + return err } + rf.onRotate = emitLogRotated lw.mu.Lock() defer lw.mu.Unlock() - lw.logFile = f - lw.logFilePath = path - lw.logFileSize = st.Size() + lw.file = rf return nil } -// rotateLogFile rotates the current log file to a .1 backup. -// It returns true if lw.logFile is usable after the call, false otherwise. -// Must be called with lw.mu held. -func (lw *logWriter) rotateLogFile() bool { - if lw.logFile == nil { - return false - } - lw.logFile.Close() - backupPath := lw.logFilePath + ".1" - // Best effort: rename current to backup (overwrites old backup). - os.Rename(lw.logFilePath, backupPath) - f, err := os.OpenFile(lw.logFilePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) - if err != nil { - // If we can't reopen, disable file logging. - lw.logFile = nil - lw.logFileSize = 0 - return false - } - lw.logFile = f - lw.logFileSize = 0 - return true -} - // closeLogFile closes the backing file if open. func (lw *logWriter) closeLogFile() { lw.mu.Lock() defer lw.mu.Unlock() - if lw.logFile != nil { - lw.logFile.Close() - lw.logFile = nil + if lw.file == nil { + return } + lw.file.close() + lw.file = nil } -// logFilePaths returns the paths to the current log file and its backup -// (if they exist) for inclusion in log send payloads. -func (lw *logWriter) logFilePaths() (current, backup string) { +// rotating returns the backing file, so a caller works with the file itself +// instead of one pass-through method for each of its calls. A stream that +// keeps its lines in memory only has no file. +func (lw *logWriter) rotating() *rotatingFile { lw.mu.Lock() defer lw.mu.Unlock() - if lw.logFilePath == "" { - return "", "" - } - current = lw.logFilePath - bp := lw.logFilePath + ".1" - if _, err := os.Stat(bp); err == nil { - backup = bp - } - return current, backup + return lw.file +} + +// bufferedBytes copies the memory buffer, so a reader of the copy holds no +// writer lock while it works. +func (lw *logWriter) bufferedBytes() []byte { + lw.mu.Lock() + defer lw.mu.Unlock() + buffered := make([]byte, lw.buf.Len()) + copy(buffered, lw.buf.Bytes()) + return buffered } // Subscribe returns a channel that receives new log data as it's written, @@ -200,6 +155,18 @@ func (lw *logWriter) tailLastLines(n int) []byte { } func (lw *logWriter) Write(p []byte) (int, error) { + // The file takes the line outside lw.mu: it holds its own lock, and it + // reports its rotation with an event that comes back through this writer. + // A file error must not stop the memory buffer, which is the fallback for + // the log readers. + if rf := lw.rotating(); rf != nil { + _, _ = rf.Write(p) + } + return lw.writeStreams(p) +} + +// writeStreams feeds the subscribers and the memory buffer. +func (lw *logWriter) writeStreams(p []byte) (int, error) { lw.mu.Lock() defer lw.mu.Unlock() @@ -216,16 +183,6 @@ func (lw *logWriter) Write(p []byte) (int, error) { } } - // Write to backing file if configured. - if lw.logFile != nil { - needsRotation := lw.logFileSize+int64(len(p)) > logFileMaxSize - if !needsRotation || lw.rotateLogFile() { - if n, err := lw.logFile.Write(p); err == nil { - lw.logFileSize += int64(n) - } - } - } - // If writing p causes overflows, discard old data. if lw.buf.Len()+len(p) > lw.size { buf := lw.buf.Bytes() @@ -254,6 +211,29 @@ func (lw *logWriter) Write(p []byte) (int, error) { return lw.buf.Write(p) } +// emitLogRotated logs a rotation. The file that rotated guards this call, +// because the event is a line that can rotate that file again. A rotation +// that could not move the file keeps the lines it holds, so it earns a +// warning instead of the normal report. +func emitLogRotated(r logRotation) { + event := journal(mainLog.Load().Info()) + message := "Log rotated" + if r.err != nil { + event = journal(mainLog.Load().Warn()).Err(r.err) + message = "Log rotation failed" + } + event = event.Str("file", r.file).Int64("bytes_written", r.bytesWritten) + // A file that holds a header only carries no event, and the year 1 of the + // zero time reads as a wrong date. + if !r.firstWrite.IsZero() { + event = event.Time("first_event_at", r.firstWrite) + } + if !r.lastWrite.IsZero() { + event = event.Time("last_event_at", r.lastWrite) + } + event.Int("backups", r.backups).Msg(message) +} + // initLogging initializes global logging setup. func (p *prog) initLogging(backup bool) { zerolog.TimeFieldFormat = time.RFC3339 + ".000" @@ -261,12 +241,18 @@ func (p *prog) initLogging(backup bool) { // Initializing internal logging after global logging. p.initInternalLogging(logWriters) -} -// internalLogFilePath returns the path for persisted internal logs. -// The file lives in the ctrld home directory alongside other runtime state. -func internalLogFilePath() string { - return absHomeDir(logFileName) + if rf := logPathFile.Load(); rf != nil { + p.refreshLogHeader() + if err := rf.writeHeader(); err != nil { + mainLog.Load().Warn().Err(err).Msg("could not write log header") + } + } + // A start-time backup happens before the logger reaches the new file, so + // its event waits here for a logger that can report it. + if record := pendingLogPathRotation.Swap(nil); record != nil { + emitLogRotated(*record) + } } // initInternalLogging performs internal logging if there's no log enabled. @@ -274,200 +260,129 @@ func (p *prog) initInternalLogging(writers []io.Writer) { if !p.needInternalLogging() { return } + headersWritten := false p.initInternalLogWriterOnce.Do(func() { mainLog.Load().Notice().Msg("internal logging enabled") p.internalLogWriter = newLogWriter() p.internalLogSent = time.Now().Add(-logWriterSentInterval) - p.internalWarnLogWriter = newSmallLogWriter() - // Persist internal logs to disk so they survive restarts. - if path := internalLogFilePath(); path != "" { - if err := p.internalLogWriter.setLogFile(path); err != nil { - mainLog.Load().Warn().Err(err).Msg("could not enable persistent internal logging") - } else { - mainLog.Load().Notice().Msgf("internal log file: %s", path) - } - } + p.internalJournalWriter = newSmallLogWriter() + p.openInternalLogFiles() + // A restart appends to the file it finds, so the header marks the + // point where this run starts. + p.refreshLogHeader() + p.writeLogHeaders() + headersWritten = true }) - p.mu.Lock() - lw := p.internalLogWriter - wlw := p.internalWarnLogWriter - p.mu.Unlock() + lw, jlw := p.internalWriters() // If ctrld was run without explicit verbose level, // run the internal logging at debug level, so we could // have enough information for troubleshooting. if verbose == 0 { - for i := range writers { - w := &zerolog.FilteredLevelWriter{ - Writer: zerolog.LevelWriterAdapter{Writer: writers[i]}, - Level: zerolog.NoticeLevel, - } - writers[i] = w - } + wrapConsoleWritersAtNotice(writers) zerolog.SetGlobalLevel(zerolog.DebugLevel) } - writers = append(writers, lw) - writers = append(writers, &zerolog.FilteredLevelWriter{ - Writer: zerolog.LevelWriterAdapter{Writer: wlw}, - Level: zerolog.WarnLevel, - }) + writers = append(writers, lw, newJournalLevelWriter(jlw)) multi := zerolog.MultiLevelWriter(writers...) l := mainLog.Load().Output(multi).With().Logger() mainLog.Store(&l) ctrld.ProxyLogger.Store(&l) -} - -// needInternalLogging reports whether prog needs to run internal logging. -func (p *prog) needInternalLogging() bool { - // Do not run in silent mode: the user explicitly asked for no logging, so - // ctrld must not create or write the persisted internal log file (nor reset - // the global level back to debug). See https://github.com/Control-D-Inc/ctrld/issues/320. - if silent { - return false - } - // Do not run in non-cd mode. - if cdUID == "" { - return false - } - // Do not run if there's already log file. - if p.cfg.Service.LogPath != "" { - return false + if headersWritten { + return } - return true + // A later call reaches a changed config, so the stored bytes need the new + // values. The files keep the header they already hold. + p.refreshLogHeader() } -func (p *prog) logReader() (*logReader, error) { - if p.needInternalLogging() { - p.mu.Lock() - lw := p.internalLogWriter - wlw := p.internalWarnLogWriter - p.mu.Unlock() - if lw == nil { - return nil, errors.New("nil internal log writer") - } - if wlw == nil { - return nil, errors.New("nil internal warn log writer") +// wrapConsoleWritersAtNotice holds the console at notice level while the files +// take every level. A run without -v needs the detail for troubleshooting, and +// the same detail on the console is noise. +func wrapConsoleWritersAtNotice(writers []io.Writer) { + for i := range writers { + writers[i] = &zerolog.FilteredLevelWriter{ + Writer: zerolog.LevelWriterAdapter{Writer: writers[i]}, + Level: zerolog.NoticeLevel, } + } +} - // If we have a persisted log file, read from disk (includes data - // from previous runs that the in-memory buffer wouldn't have). - current, backup := lw.logFilePaths() - if current != "" { - return p.logReaderFromFiles(current, backup, wlw) - } +// openInternalLogFiles persists both internal streams, so they survive a +// restart. +func (p *prog) openInternalLogFiles() { + isRouter := router.Name() != "" + _, journalBudget := logBudgets(isRouter) + openInternalLogFile(p.internalLogWriter, absHomeDir(logFileName), debugLogBudget(&p.cfg.Service, isRouter), "internal") + openInternalLogFile(p.internalJournalWriter, absHomeDir(journalLogFileName), journalBudget, "journal") +} - // Fall back to in-memory buffer. - lw.mu.Lock() - lwReader := bytes.NewReader(lw.buf.Bytes()) - lwSize := lw.buf.Len() - lw.mu.Unlock() - // Warn log content. - wlw.mu.Lock() - wlwReader := bytes.NewReader(wlw.buf.Bytes()) - wlwSize := wlw.buf.Len() - wlw.mu.Unlock() - reader := io.MultiReader(lwReader, bytes.NewReader([]byte(logWriterLogEndMarker)), wlwReader) - lr := &logReader{r: io.NopCloser(reader)} - lr.size = int64(lwSize + wlwSize) - if lr.size == 0 { - return nil, errors.New("internal log is empty") - } - return lr, nil - } - if p.cfg.Service.LogPath == "" { - return &logReader{r: io.NopCloser(strings.NewReader(""))}, nil - } - f, err := os.Open(normalizeLogFilePath(p.cfg.Service.LogPath)) - if err != nil { - return nil, err - } - lr := &logReader{r: f} - if st, err := f.Stat(); err == nil { - lr.size = st.Size() - } else { - return nil, fmt.Errorf("f.Stat: %w", err) - } - if lr.size == 0 { - return nil, errors.New("log file is empty") +// openInternalLogFile persists one internal stream. A stream whose file cannot +// open keeps its lines in memory. Only these files get a prune: ctrld owns +// their names, while log_path can name a file in a directory that holds the +// files of other programs. +func openInternalLogFile(lw *logWriter, path string, budget logBudget, name string) { + pruneNumberedBackups(path, budget.backups) + if err := lw.setLogFile(path, budget); err != nil { + mainLog.Load().Warn().Err(err).Msgf("could not enable persistent %s logging", name) + return } - return lr, nil + mainLog.Load().Notice().Msgf("%s log file: %s", name, path) } -// logReaderFromFiles builds a logReader that concatenates the backup file -// (if it exists), the current log file, and the in-memory warn log buffer. -func (p *prog) logReaderFromFiles(current, backup string, wlw *logWriter) (*logReader, error) { - var rcs []io.ReadCloser - var totalSize int64 +// internalWriters returns the debug stream and the journal stream. Both stay +// nil until initInternalLogging opens them, and outside cd mode. +func (p *prog) internalWriters() (debugWriter, journalWriter *logWriter) { + p.mu.Lock() + defer p.mu.Unlock() + return p.internalLogWriter, p.internalJournalWriter +} - closeAll := func() { - for _, rc := range rcs { - rc.Close() +// openLogFiles returns the files that this run writes, debug first. A stream +// that keeps its lines in memory only has no file, and log_path belongs to +// this list although no internal writer owns it. +func (p *prog) openLogFiles() []*rotatingFile { + debugWriter, journalWriter := p.internalWriters() + files := make([]*rotatingFile, 0, 3) + for _, writer := range []*logWriter{debugWriter, journalWriter} { + if writer == nil { + continue } - } - - // Read backup file first (older entries). - if backup != "" { - if bf, err := os.Open(backup); err == nil { - if st, err := bf.Stat(); err == nil { - totalSize += st.Size() - } - rcs = append(rcs, bf) + if rf := writer.rotating(); rf != nil { + files = append(files, rf) } } - - // Read current file. - cf, err := os.Open(current) - if err != nil { - closeAll() - return nil, fmt.Errorf("opening current log file: %w", err) - } - if st, err := cf.Stat(); err == nil { - totalSize += st.Size() - } - rcs = append(rcs, cf) - - // Append warn log content from memory. - wlw.mu.Lock() - warnData := make([]byte, wlw.buf.Len()) - copy(warnData, wlw.buf.Bytes()) - wlw.mu.Unlock() - - if len(warnData) > 0 { - rcs = append(rcs, io.NopCloser(bytes.NewReader([]byte(logWriterLogEndMarker)))) - rcs = append(rcs, io.NopCloser(bytes.NewReader(warnData))) - totalSize += int64(len(logWriterLogEndMarker) + len(warnData)) + if rf := logPathFile.Load(); rf != nil { + files = append(files, rf) } + return files +} - if totalSize == 0 { - closeAll() - return nil, errors.New("internal log is empty") +// closeInternalLogs closes both internal files, so a stop leaves no open +// handle behind. +func (p *prog) closeInternalLogs() { + debugWriter, journalWriter := p.internalWriters() + for _, writer := range []*logWriter{debugWriter, journalWriter} { + if writer == nil { + continue + } + writer.closeLogFile() } +} - readers := make([]io.Reader, len(rcs)) - closers := make([]io.Closer, len(rcs)) - for i, rc := range rcs { - readers[i] = rc - closers[i] = rc +// needInternalLogging reports whether prog needs to run internal logging. +func (p *prog) needInternalLogging() bool { + // Do not run in silent mode: the user explicitly asked for no logging, so + // ctrld must not create or write the persisted internal log file (nor reset + // the global level back to debug). See https://github.com/Control-D-Inc/ctrld/issues/320. + if silent { + return false } - combined := io.MultiReader(readers...) - lr := &logReader{ - r: &multiCloser{Reader: combined, closers: closers}, - size: totalSize, + // Do not run in non-cd mode. + if cdUID == "" { + return false } - return lr, nil -} - -// multiCloser wraps an io.Reader and closes multiple underlying closers. -type multiCloser struct { - io.Reader - closers []io.Closer -} - -func (mc *multiCloser) Close() error { - var firstErr error - for _, c := range mc.closers { - if err := c.Close(); err != nil && firstErr == nil { - firstErr = err - } + // Do not run if there's already log file. + if p.cfg.Service.LogPath != "" { + return false } - return firstErr + return true } diff --git a/cmd/cli/log_writer_silent_test.go b/cmd/cli/log_writer_silent_test.go index e40f7798..70eddb39 100644 --- a/cmd/cli/log_writer_silent_test.go +++ b/cmd/cli/log_writer_silent_test.go @@ -1,10 +1,14 @@ package cli import ( + "io" "os" "path/filepath" + "strings" "testing" + "github.com/rs/zerolog" + "github.com/Control-D-Inc/ctrld" ) @@ -59,8 +63,77 @@ func Test_initInternalLogging_silentCreatesNoFile(t *testing.T) { p := &prog{cfg: &ctrld.Config{}} p.initInternalLogging(nil) - logPath := filepath.Join(dir, logFileName) - if _, err := os.Stat(logPath); !os.IsNotExist(err) { - t.Fatalf("silent mode must not create %s (stat err = %v)", logPath, err) + for _, name := range []string{logFileName, journalLogFileName} { + path := filepath.Join(dir, name) + if _, err := os.Stat(path); !os.IsNotExist(err) { + t.Fatalf("silent mode must not create %s (stat err = %v)", path, err) + } + } +} + +// Test_initInternalLogging_createsJournalFile drives a cd mode run and asserts +// that the journal file holds the retained lines only, while the debug file +// holds every line. +func Test_initInternalLogging_createsJournalFile(t *testing.T) { + origSilent, origCdUID, origHomedir, origVerbose := silent, cdUID, homedir, verbose + origMainLog, origProxyLog := mainLog.Load(), ctrld.ProxyLogger.Load() + origLevel := zerolog.GlobalLevel() + t.Cleanup(func() { + silent, cdUID, homedir, verbose = origSilent, origCdUID, origHomedir, origVerbose + mainLog.Store(origMainLog) + ctrld.ProxyLogger.Store(origProxyLog) + zerolog.SetGlobalLevel(origLevel) + }) + if origMainLog == nil { + discard := zerolog.New(io.Discard) + mainLog.Store(&discard) + } + + dir := t.TempDir() + homedir = dir + cdUID = "test-uid" + silent = false + verbose = 0 + + p := &prog{cfg: &ctrld.Config{}} + p.initInternalLogging(nil) + t.Cleanup(func() { + p.internalLogWriter.closeLogFile() + p.internalJournalWriter.closeLogFile() + }) + + debugPath := filepath.Join(dir, logFileName) + journalPath := filepath.Join(dir, journalLogFileName) + for _, path := range []string{debugPath, journalPath} { + if _, err := os.Stat(path); err != nil { + t.Fatalf("stat %s: %v", path, err) + } + } + + const ( + warnMsg = "retained warn line" + markedMsg = "retained marked info line" + plainMsg = "dropped plain info line" + ) + logger := mainLog.Load() + logger.Warn().Msg(warnMsg) + journal(logger.Info()).Msg(markedMsg) + logger.Info().Msg(plainMsg) + + journalText := rotatingFileTestContent(t, journalPath) + for _, msg := range []string{warnMsg, markedMsg} { + if !strings.Contains(journalText, msg) { + t.Fatalf("journal file misses %q: %s", msg, journalText) + } + } + if strings.Contains(journalText, plainMsg) { + t.Fatalf("journal file kept %q: %s", plainMsg, journalText) + } + + debugText := rotatingFileTestContent(t, debugPath) + for _, msg := range []string{warnMsg, markedMsg, plainMsg} { + if !strings.Contains(debugText, msg) { + t.Fatalf("debug file misses %q: %s", msg, debugText) + } } } diff --git a/cmd/cli/log_writer_test.go b/cmd/cli/log_writer_test.go index f6d1f949..24b268bb 100644 --- a/cmd/cli/log_writer_test.go +++ b/cmd/cli/log_writer_test.go @@ -1,11 +1,17 @@ package cli import ( + "encoding/json" + "errors" "os" "path/filepath" + "slices" "strings" "sync" "testing" + "time" + + "github.com/rs/zerolog" ) func Test_logWriter_Write(t *testing.T) { @@ -86,14 +92,23 @@ func Test_logWriter_MarkerInitEnd(t *testing.T) { } } -func Test_logWriter_SetLogFile(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "test.log") +// testLogWriterBudget keeps the rotation tests small: four lines fill the +// current file. +var testLogWriterBudget = logBudget{maxSize: 4 * rotatingFileTestLineSize, backups: 2} + +func newTestFileLogWriter(t *testing.T, budget logBudget) (*logWriter, string) { + t.Helper() + path := filepath.Join(t.TempDir(), "test.log") lw := newLogWriterWithSize(logWriterSize) - if err := lw.setLogFile(path); err != nil { + if err := lw.setLogFile(path, budget); err != nil { t.Fatalf("setLogFile: %v", err) } - defer lw.closeLogFile() + t.Cleanup(lw.closeLogFile) + return lw, path +} + +func Test_logWriter_SetLogFile(t *testing.T) { + lw, path := newTestFileLogWriter(t, testLogWriterBudget) msg := "hello file\n" lw.Write([]byte(msg)) @@ -113,69 +128,49 @@ func Test_logWriter_SetLogFile(t *testing.T) { } func Test_logWriter_FileRotation(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "test.log") - // Use a tiny max size to trigger rotation quickly. - lw := newLogWriterWithSize(logWriterSize) - if err := lw.setLogFile(path); err != nil { - t.Fatalf("setLogFile: %v", err) - } - defer lw.closeLogFile() + lw, path := newTestFileLogWriter(t, testLogWriterBudget) - // Write enough to exceed logFileMaxSize. - chunk := strings.Repeat("X", 1024) + "\n" - written := 0 - for written < logFileMaxSize+1024 { - lw.Write([]byte(chunk)) - written += len(chunk) + // Three rotations: each file takes four lines. + for seq := 1; seq <= 16; seq++ { + lw.Write(rotatingFileTestLine(1, seq)) } - // Backup file should exist. - backupPath := path + ".1" - if _, err := os.Stat(backupPath); os.IsNotExist(err) { - t.Fatal("expected backup file to exist after rotation") + for _, suffix := range []string{".1", ".2"} { + if _, err := os.Stat(path + suffix); err != nil { + t.Fatalf("stat %s: %v", path+suffix, err) + } + } + if _, err := os.Stat(path + ".3"); !os.IsNotExist(err) { + t.Fatalf("stat %s.3 = %v, want a not exist error", path, err) } - // Current file should be smaller than max (it was rotated). st, err := os.Stat(path) if err != nil { t.Fatalf("stat current: %v", err) } - if st.Size() > logFileMaxSize { - t.Fatalf("current file too large after rotation: %d", st.Size()) + if want := testLogWriterBudget.maxSize + rotatingFileTestLineSize; st.Size() > want { + t.Fatalf("current file holds %d bytes after rotation, want at most %d", st.Size(), want) } } -func Test_logWriter_FilePaths(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "test.log") +func Test_logWriter_RotatingPaths(t *testing.T) { lw := newLogWriterWithSize(logWriterSize) - - // No file configured. - c, b := lw.logFilePaths() - if c != "" || b != "" { - t.Fatalf("expected empty paths, got %q %q", c, b) + if lw.rotating() != nil { + t.Fatal("a writer without a file has a rotating file, want none") } - if err := lw.setLogFile(path); err != nil { - t.Fatalf("setLogFile: %v", err) + lw, path := newTestFileLogWriter(t, testLogWriterBudget) + if got, want := lw.rotating().paths(), []string{path}; !slices.Equal(got, want) { + t.Fatalf("paths = %v, want %v", got, want) } - defer lw.closeLogFile() - // Current exists, no backup yet. - c, b = lw.logFilePaths() - if c != path { - t.Fatalf("current: got %q, want %q", c, path) - } - if b != "" { - t.Fatalf("backup should be empty, got %q", b) + // Two rotations, so both backups exist. + for seq := 1; seq <= 9; seq++ { + lw.Write(rotatingFileTestLine(1, seq)) } - - // Create a backup file manually. - os.WriteFile(path+".1", []byte("old"), 0600) - _, b = lw.logFilePaths() - if b != path+".1" { - t.Fatalf("backup: got %q, want %q", b, path+".1") + want := []string{path + ".2", path + ".1", path} + if got := lw.rotating().paths(); !slices.Equal(got, want) { + t.Fatalf("paths = %v, want %v", got, want) } } @@ -185,7 +180,7 @@ func Test_logWriter_FileAppendOnRestart(t *testing.T) { // Simulate first run. lw1 := newLogWriterWithSize(logWriterSize) - if err := lw1.setLogFile(path); err != nil { + if err := lw1.setLogFile(path, testLogWriterBudget); err != nil { t.Fatalf("setLogFile: %v", err) } lw1.Write([]byte("run1\n")) @@ -193,7 +188,7 @@ func Test_logWriter_FileAppendOnRestart(t *testing.T) { // Simulate second run (restart) — file should be appended. lw2 := newLogWriterWithSize(logWriterSize) - if err := lw2.setLogFile(path); err != nil { + if err := lw2.setLogFile(path, testLogWriterBudget); err != nil { t.Fatalf("setLogFile: %v", err) } lw2.Write([]byte("run2\n")) @@ -208,3 +203,169 @@ func Test_logWriter_FileAppendOnRestart(t *testing.T) { t.Fatalf("file: got %q, want %q", data, want) } } + +func Test_logWriter_EmitsLogRotated(t *testing.T) { + setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + lw, path := newTestFileLogWriter(t, testLogWriterBudget) + + // The logger writes back into the writer that rotates, which is the case + // that deadlocks when the writer still holds its lock. + buf := &syncBuffer{} + logger := zerolog.New(zerolog.MultiLevelWriter(lw, buf)) + old := mainLog.Load() + mainLog.Store(&logger) + t.Cleanup(func() { mainLog.Store(old) }) + + done := make(chan struct{}) + go func() { + defer close(done) + for seq := 1; seq <= 5; seq++ { + lw.Write(rotatingFileTestLine(1, seq)) + } + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("the writer did not return: the rotation event blocked on the writer lock") + } + + st, err := os.Stat(path + ".1") + if err != nil { + t.Fatalf("stat %s.1: %v", path, err) + } + + var rotatedLines []string + for _, line := range strings.Split(buf.String(), "\n") { + if strings.Contains(line, "Log rotated") { + rotatedLines = append(rotatedLines, line) + } + } + if len(rotatedLines) != 1 { + t.Fatalf("got %d Log rotated events, want 1: %q", len(rotatedLines), buf.String()) + } + if !strings.Contains(rotatedLines[0], string(journalMarker)) { + t.Fatalf("the event is not marked for the journal: %s", rotatedLines[0]) + } + var event struct { + File string `json:"file"` + BytesWritten int64 `json:"bytes_written"` + } + if err := json.Unmarshal([]byte(rotatedLines[0]), &event); err != nil { + t.Fatalf("unmarshal %s: %v", rotatedLines[0], err) + } + if event.File != path+".1" { + t.Fatalf("file = %q, want %q", event.File, path+".1") + } + if event.BytesWritten != st.Size() { + t.Fatalf("bytes_written = %d, want %d", event.BytesWritten, st.Size()) + } +} + +func Test_logWriter_NestedRotationStillEmitsEvent(t *testing.T) { + setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + debugLog, debugPath := newTestFileLogWriter(t, testLogWriterBudget) + journalLog, journalPath := newTestFileLogWriter(t, logBudget{maxSize: 2 * rotatingFileTestLineSize, backups: 1}) + + buf := &syncBuffer{} + logger := zerolog.New(zerolog.MultiLevelWriter(debugLog, journalLog, buf)) + old := mainLog.Load() + mainLog.Store(&logger) + t.Cleanup(func() { mainLog.Store(old) }) + + // Fill the journal, so the rotation event of the debug file is the line + // that rotates the journal. That nested rotation needs its own event. + journalLog.Write(rotatingFileTestLine(1, 1)) + journalLog.Write(rotatingFileTestLine(1, 2)) + for seq := 1; seq <= 5; seq++ { + debugLog.Write(rotatingFileTestLine(1, seq)) + } + + rotatedFiles := map[string]bool{} + for _, line := range strings.Split(buf.String(), "\n") { + if !strings.Contains(line, "Log rotated") { + continue + } + var event struct { + File string `json:"file"` + } + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatalf("parse %q: %v", line, err) + } + rotatedFiles[event.File] = true + } + for _, want := range []string{debugPath + ".1", journalPath + ".1"} { + if !rotatedFiles[want] { + t.Fatalf("no Log rotated event for %s; events: %v", want, rotatedFiles) + } + } + if len(rotatedFiles) != 2 { + t.Fatalf("got %d rotated files, want 2: %v", len(rotatedFiles), rotatedFiles) + } +} + +// rotationEventFields parses one log line into its raw fields, so a test can +// ask whether a field is there at all. +func rotationEventFields(t *testing.T, line string) map[string]json.RawMessage { + t.Helper() + fields := map[string]json.RawMessage{} + if err := json.Unmarshal([]byte(strings.TrimSpace(line)), &fields); err != nil { + t.Fatalf("unmarshal %q: %v", line, err) + } + return fields +} + +func Test_emitLogRotated_eventTimes(t *testing.T) { + setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + + t.Run("a file with no event names no time", func(t *testing.T) { + buf := captureMainLog(t) + + emitLogRotated(logRotation{file: "/tmp/ctrld.log.1", backups: 1}) + + fields := rotationEventFields(t, buf.String()) + for _, name := range []string{"first_event_at", "last_event_at"} { + if raw, ok := fields[name]; ok { + t.Errorf("%s = %s, want no field for a file with no event", name, raw) + } + } + }) + + t.Run("a file with events names both times", func(t *testing.T) { + buf := captureMainLog(t) + now := time.Now() + + emitLogRotated(logRotation{file: "/tmp/ctrld.log.1", backups: 1, firstWrite: now, lastWrite: now}) + + fields := rotationEventFields(t, buf.String()) + for _, name := range []string{"first_event_at", "last_event_at"} { + if _, ok := fields[name]; !ok { + t.Errorf("%s is missing: %v", name, fields) + } + } + }) +} + +func Test_emitLogRotated_reportsAFailedRotation(t *testing.T) { + setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + buf := captureMainLog(t) + + emitLogRotated(logRotation{file: "/tmp/ctrld.log.1", err: errors.New("rename refused")}) + + var event struct { + Level string `json:"level"` + Message string `json:"message"` + Error string `json:"error"` + } + if err := json.Unmarshal([]byte(strings.TrimSpace(buf.String())), &event); err != nil { + t.Fatalf("unmarshal %q: %v", buf.String(), err) + } + if event.Message != "Log rotation failed" { + t.Errorf("message = %q, want %q", event.Message, "Log rotation failed") + } + if event.Level != "warn" { + t.Errorf("level = %q, want %q", event.Level, "warn") + } + if event.Error != "rename refused" { + t.Errorf("error = %q, want %q", event.Error, "rename refused") + } +} diff --git a/cmd/cli/main.go b/cmd/cli/main.go index d372e8de..24f47ce5 100644 --- a/cmd/cli/main.go +++ b/cmd/cli/main.go @@ -143,30 +143,8 @@ func initInteractiveLogging() { // wrapper instead of calling this function directly. func initLoggingWithBackup(doBackup bool) []io.Writer { var writers []io.Writer - if logFilePath := normalizeLogFilePath(cfg.Service.LogPath); logFilePath != "" { - // Create parent directory if necessary. - if err := os.MkdirAll(filepath.Dir(logFilePath), 0750); err != nil { - mainLog.Load().Error().Msgf("failed to create log path: %v", err) - os.Exit(1) - } - - // Default open log file in append mode. - flags := os.O_CREATE | os.O_RDWR | os.O_APPEND - if doBackup { - // Backup old log file with .1 suffix. - if err := os.Rename(logFilePath, logFilePath+oldLogSuffix); err != nil && !os.IsNotExist(err) { - mainLog.Load().Error().Msgf("could not backup old log file: %v", err) - } else { - // Backup was created, set flags for truncating old log file. - flags = os.O_CREATE | os.O_RDWR - } - } - logFile, err := openLogFile(logFilePath, flags) - if err != nil { - mainLog.Load().Error().Msgf("failed to create log file: %v", err) - os.Exit(1) - } - writers = append(writers, logFile) + if rf := openLogPathWriter(doBackup); rf != nil { + writers = append(writers, rf) } writers = append(writers, consoleWriter) multi := zerolog.MultiLevelWriter(writers...) diff --git a/cmd/cli/network_transition_test.go b/cmd/cli/network_transition_test.go index 5db17793..eb68d7a8 100644 --- a/cmd/cli/network_transition_test.go +++ b/cmd/cli/network_transition_test.go @@ -250,11 +250,14 @@ func TestRecoveryDebouncePreservesTransitionID(t *testing.T) { func TestTransitionWarningsSurviveDebugRotation(t *testing.T) { sourceTestGlobals(t) + level := zerolog.GlobalLevel() + zerolog.SetGlobalLevel(zerolog.DebugLevel) + t.Cleanup(func() { zerolog.SetGlobalLevel(level) }) old := mainLog.Load() debug, warnings := newLogWriterWithSize(2048), newSmallLogWriter() // Runtime summaries occur after initialization, outside the preserved prefix. debug.Write([]byte(logWriterInitEndMarker)) - logger := zerolog.New(zerolog.MultiLevelWriter(debug, &zerolog.FilteredLevelWriter{Writer: zerolog.LevelWriterAdapter{Writer: warnings}, Level: zerolog.WarnLevel})) + logger := zerolog.New(zerolog.MultiLevelWriter(debug, newJournalLevelWriter(warnings))) mainLog.Store(&logger) t.Cleanup(func() { mainLog.Store(old) }) ctrld.SetDefaultLocalIPv4(net.ParseIP("192.0.2.10")) @@ -272,6 +275,8 @@ func TestTransitionWarningsSurviveDebugRotation(t *testing.T) { wg.Wait() d.end("completed") (&recoveryDiagnostic{transitionID: 8, generation: 10, started: time.Now()}).end("completed") + journal(logger.Info()).Str("interface", "en0").Msg("Network snapshot") + logger.Info().Msg("plain info line") for range 100 { logger.Debug().Msg(strings.Repeat("noise", 100)) } @@ -290,6 +295,12 @@ func TestTransitionWarningsSurviveDebugRotation(t *testing.T) { if strings.Contains(text, "private.customer") || strings.Contains(text, "secret payload") { t.Fatalf("raw payload or normal debug elevated: %s", text) } + if strings.Count(text, "Network snapshot") != 1 { + t.Fatalf("expected one retained marked info line: %s", text) + } + if strings.Contains(text, "plain info line") { + t.Fatalf("unmarked info line must not be retained: %s", text) + } } func TestRecoveryWaitLogsFirstFailure(t *testing.T) { diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 60913dc2..487ca314 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -137,7 +137,9 @@ type prog struct { queryFromSelfMap sync.Map initInternalLogWriterOnce sync.Once internalLogWriter *logWriter - internalWarnLogWriter *logWriter + internalJournalWriter *logWriter + querySampler errorSampler + lastNetworkState atomic.Pointer[netmon.State] internalLogSent time.Time runningIface string requiredMultiNICsConfig bool @@ -427,6 +429,10 @@ func (p *prog) runWait() { } p.mu.Unlock() + // The header names the mode, the listeners, and the upstreams, so the + // open files take the values of the config that now runs. + p.refreshLogHeader() + logger.Notice().Msg("reloading config successfully") select { @@ -764,6 +770,12 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { } } + // The new monitor starts with every upstream up, so the old one closes its + // open outages first. Without this the journal holds a down event that no + // up event ever follows. + if p.um != nil { + p.um.retire() + } p.um = newUpstreamMonitor(p.cfg) if !reload { @@ -908,6 +920,8 @@ func (p *prog) Stop(s service.Service) error { mainLog.Load().Debug().Msg("finish running onStopped functions") defer func() { mainLog.Load().Info().Msg("Service stopped") + // The files take the last line of this run before they close. + p.closeInternalLogs() }() if err := p.deAllocateIP(); err != nil { mainLog.Load().Error().Err(err).Msg("de-allocate ip failed") diff --git a/cmd/cli/query_error_sampler.go b/cmd/cli/query_error_sampler.go new file mode 100644 index 00000000..78d017b4 --- /dev/null +++ b/cmd/cli/query_error_sampler.go @@ -0,0 +1,166 @@ +package cli + +import ( + "sync" + "time" + + "github.com/rs/zerolog" +) + +// Classes of per-query errors. Each class keeps its own budget per upstream. +const ( + sampleClassResolveFailed = "resolve_failed" + sampleClassAllEndpointsFailed = "all_endpoints_failed" + sampleClassSendResponseFailed = "send_response_failed" +) + +const ( + // querySampleLimit is the number of lines that stay at error level in one window. + querySampleLimit = 5 + // querySampleWindow is the length of a window. It starts at the first error of the window. + querySampleWindow = 60 * time.Second +) + +// sampleKey identifies one budget. +type sampleKey struct { + class string + upstream string +} + +// sampleWindow counts the events of one budget between its start and its end. +type sampleWindow struct { + start time.Time + count int + suppressed int + timer *time.Timer +} + +// ended reports whether the window closed at or before now. +func (w *sampleWindow) ended(now time.Time) bool { + return !now.Before(w.start.Add(querySampleWindow)) +} + +// sampleSummary holds what one closed window reports. +type sampleSummary struct { + key sampleKey + count int + suppressed int +} + +// errorSampler bounds the noise of a repeated per-query error. It lowers the +// level of a line, never its presence, so the debug stream stays complete. A +// run with -v holds every stream at info, and the lowered lines fall below it. +// The zero value is ready to use. +type errorSampler struct { + mu sync.Mutex + windows map[sampleKey]*sampleWindow + now func() time.Time + after func(time.Duration, func()) *time.Timer +} + +// event returns the event a call site logs with. The first querySampleLimit +// events of a window come back at error level, the later ones at debug level. +// The caller adds its own fields and the message. +func (s *errorSampler) event(class, upstream string) *zerolog.Event { + key := sampleKey{class: class, upstream: upstream} + + s.mu.Lock() + now := s.clock() + var summaries []sampleSummary + window := s.windows[key] + if window != nil && window.ended(now) { + summaries = appendWindowSummary(summaries, key, window) + s.dropWindow(key, window) + window = nil + } + if window == nil { + window = s.startWindow(key, now) + } + window.count++ + suppressed := window.count > querySampleLimit + if suppressed { + window.suppressed++ + } + s.mu.Unlock() + + s.logSummaries(summaries) + + if suppressed { + return mainLog.Load().Debug() + } + return mainLog.Load().Error() +} + +// closeExpired closes every window that ended at or before now. The window +// timer calls it, so an idle window reports without a new event. +func (s *errorSampler) closeExpired(now time.Time) { + s.mu.Lock() + var summaries []sampleSummary + for key, window := range s.windows { + if !window.ended(now) { + continue + } + summaries = appendWindowSummary(summaries, key, window) + s.dropWindow(key, window) + } + s.mu.Unlock() + + s.logSummaries(summaries) +} + +// startWindow opens a window and arms the timer that closes it. The caller holds the lock. +func (s *errorSampler) startWindow(key sampleKey, start time.Time) *sampleWindow { + if s.windows == nil { + s.windows = make(map[sampleKey]*sampleWindow) + } + window := &sampleWindow{start: start} + s.windows[key] = window + window.timer = s.afterFunc()(querySampleWindow, func() { s.closeExpired(s.clock()) }) + return window +} + +// dropWindow forgets a window and stops its timer. The caller holds the lock. +func (s *errorSampler) dropWindow(key sampleKey, window *sampleWindow) { + if window.timer != nil { + window.timer.Stop() + } + delete(s.windows, key) +} + +// logSummaries reports the closed windows. The caller released the lock first, +// because the logger writes back through code that can take it again. +func (s *errorSampler) logSummaries(summaries []sampleSummary) { + for _, summary := range summaries { + mainLog.Load().Error(). + Str("class", summary.key.class). + Str("upstream", summary.key.upstream). + Int("count", summary.count). + Int("suppressed", summary.suppressed). + Int("window_s", int(querySampleWindow/time.Second)). + Msg("Per-query errors sampled") + } +} + +// clock reads the time of the sampler. A test sets its own clock, so a window +// closes without a wait. +func (s *errorSampler) clock() time.Time { + if s.now != nil { + return s.now() + } + return time.Now() +} + +func (s *errorSampler) afterFunc() func(time.Duration, func()) *time.Timer { + if s.after != nil { + return s.after + } + return time.AfterFunc +} + +// appendWindowSummary keeps the windows that suppressed at least one line. +func appendWindowSummary(summaries []sampleSummary, key sampleKey, window *sampleWindow) []sampleSummary { + if window.suppressed == 0 { + return summaries + } + return append(summaries, sampleSummary{key: key, count: window.count, suppressed: window.suppressed}) +} diff --git a/cmd/cli/query_error_sampler_test.go b/cmd/cli/query_error_sampler_test.go new file mode 100644 index 00000000..57eaa32d --- /dev/null +++ b/cmd/cli/query_error_sampler_test.go @@ -0,0 +1,260 @@ +package cli + +import ( + "encoding/json" + "strings" + "testing" + "time" + + "github.com/rs/zerolog" +) + +// samplerTestMessage is the message the fake call site writes on every sampled event. +const samplerTestMessage = "failed to resolve query" + +// samplerSummaryMessage is the message of the window summary. +const samplerSummaryMessage = "Per-query errors sampled" + +// captureDebugMainLog sends mainLog into a buffer and lowers the global level, +// so the test also sees the lines that the code under test drops to debug. It +// swaps two globals, so no test of this package runs in parallel. +func captureDebugMainLog(t *testing.T) *syncBuffer { + t.Helper() + level := zerolog.GlobalLevel() + zerolog.SetGlobalLevel(zerolog.DebugLevel) + t.Cleanup(func() { zerolog.SetGlobalLevel(level) }) + return captureMainLog(t) +} + +// jsonLogEvents parses the captured lines as JSON events, in order. An empty +// message takes every line. +func jsonLogEvents(t *testing.T, logs *syncBuffer, message string) []map[string]any { + t.Helper() + var events []map[string]any + for _, line := range strings.Split(logs.String(), "\n") { + if strings.TrimSpace(line) == "" { + continue + } + event := map[string]any{} + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatalf("log line is not JSON: %q: %v", line, err) + } + if message != "" && event["message"] != message { + continue + } + events = append(events, event) + } + return events +} + +// samplerHarness drives an errorSampler on a clock and a timer queue the test owns. +type samplerHarness struct { + sampler *errorSampler + logs *syncBuffer + now time.Time + pending []func() +} + +func newSamplerHarness(t *testing.T) *samplerHarness { + t.Helper() + h := &samplerHarness{ + logs: captureDebugMainLog(t), + now: time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC), + } + h.sampler = &errorSampler{ + now: func() time.Time { return h.now }, + after: func(_ time.Duration, fn func()) *time.Timer { + h.pending = append(h.pending, fn) + return nil + }, + } + return h +} + +// emit plays a call site: it takes an event from the sampler and adds its own fields. +func (h *samplerHarness) emit(class, upstream string) { + h.sampler.event(class, upstream).Str("class", class).Str("upstream", upstream).Msg(samplerTestMessage) +} + +func (h *samplerHarness) advance(d time.Duration) { + h.now = h.now.Add(d) +} + +func (h *samplerHarness) events(t *testing.T) []map[string]any { + t.Helper() + return jsonLogEvents(t, h.logs, "") +} + +func countSampled(events []map[string]any, level, upstream string) int { + count := 0 + for _, event := range events { + if event["message"] != samplerTestMessage { + continue + } + if event["level"] != level || event["upstream"] != upstream { + continue + } + count++ + } + return count +} + +func samplerSummaries(events []map[string]any) []map[string]any { + var summaries []map[string]any + for _, event := range events { + if event["message"] == samplerSummaryMessage { + summaries = append(summaries, event) + } + } + return summaries +} + +func wantField(t *testing.T, event map[string]any, field string, want any) { + t.Helper() + if got := event[field]; got != want { + t.Fatalf("field %q: got %v, want %v", field, got, want) + } +} + +func Test_errorSamplerLimitsWindowAndSummarizes(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < querySampleLimit+2; i++ { + h.emit(sampleClassResolveFailed, "upstream.0") + } + + events := h.events(t) + if got := countSampled(events, "error", "upstream.0"); got != querySampleLimit { + t.Fatalf("error lines: got %d, want %d", got, querySampleLimit) + } + if got := countSampled(events, "debug", "upstream.0"); got != 2 { + t.Fatalf("debug lines: got %d, want 2", got) + } + if got := len(samplerSummaries(events)); got != 0 { + t.Fatalf("summaries before the window closed: got %d, want 0", got) + } + + h.advance(querySampleWindow) + h.emit(sampleClassResolveFailed, "upstream.0") + + events = h.events(t) + summaries := samplerSummaries(events) + if len(summaries) != 1 { + t.Fatalf("summaries after the window closed: got %d, want 1", len(summaries)) + } + summary := summaries[0] + wantField(t, summary, "level", "error") + wantField(t, summary, "class", sampleClassResolveFailed) + wantField(t, summary, "upstream", "upstream.0") + wantField(t, summary, "count", float64(7)) + wantField(t, summary, "suppressed", float64(2)) + wantField(t, summary, "window_s", float64(60)) + + last := events[len(events)-1] + wantField(t, last, "message", samplerTestMessage) + wantField(t, last, "level", "error") +} + +func Test_errorSamplerKeepsBudgetPerUpstream(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < querySampleLimit+1; i++ { + h.emit(sampleClassResolveFailed, "upstream.0") + } + for i := 0; i < querySampleLimit; i++ { + h.emit(sampleClassResolveFailed, "upstream.1") + } + + events := h.events(t) + if got := countSampled(events, "error", "upstream.1"); got != querySampleLimit { + t.Fatalf("error lines for the second upstream: got %d, want %d", got, querySampleLimit) + } + if got := countSampled(events, "debug", "upstream.1"); got != 0 { + t.Fatalf("debug lines for the second upstream: got %d, want 0", got) + } + if got := countSampled(events, "debug", "upstream.0"); got != 1 { + t.Fatalf("debug lines for the first upstream: got %d, want 1", got) + } +} + +func Test_errorSamplerKeepsQuietWindowSilent(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < 3; i++ { + h.emit(sampleClassAllEndpointsFailed, "") + } + + h.advance(querySampleWindow) + h.sampler.closeExpired(h.now) + + if got := len(samplerSummaries(h.events(t))); got != 0 { + t.Fatalf("summaries for a window without suppressed lines: got %d, want 0", got) + } +} + +func Test_errorSamplerCloseExpiredSummarizesOnce(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < querySampleLimit+2; i++ { + h.emit(sampleClassSendResponseFailed, "") + } + sampled := len(h.events(t)) + + h.advance(querySampleWindow) + h.sampler.closeExpired(h.now) + + events := h.events(t) + if len(events) != sampled+1 { + t.Fatalf("lines after closeExpired: got %d, want %d", len(events), sampled+1) + } + summaries := samplerSummaries(events) + if len(summaries) != 1 { + t.Fatalf("summaries after closeExpired: got %d, want 1", len(summaries)) + } + wantField(t, summaries[0], "class", sampleClassSendResponseFailed) + wantField(t, summaries[0], "suppressed", float64(2)) + + h.sampler.closeExpired(h.now.Add(querySampleWindow)) + + if got := len(h.events(t)); got != sampled+1 { + t.Fatalf("lines after the second closeExpired: got %d, want %d", got, sampled+1) + } +} + +func Test_errorSamplerTimerClosesIdleWindow(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < querySampleLimit+1; i++ { + h.emit(sampleClassResolveFailed, "upstream.0") + } + if len(h.pending) != 1 { + t.Fatalf("window timers: got %d, want 1", len(h.pending)) + } + + h.advance(querySampleWindow) + h.pending[0]() + + summaries := samplerSummaries(h.events(t)) + if len(summaries) != 1 { + t.Fatalf("summaries after the timer fired: got %d, want 1", len(summaries)) + } + wantField(t, summaries[0], "suppressed", float64(1)) + wantField(t, summaries[0], "count", float64(6)) +} + +func Test_errorSamplerZeroValueWorks(t *testing.T) { + buf := captureDebugMainLog(t) + + var sampler errorSampler + // The zero value arms a real timer. Close its window after the assertions, + // so no late summary line reaches another test. + t.Cleanup(func() { sampler.closeExpired(time.Now().Add(querySampleWindow)) }) + sampler.event(sampleClassResolveFailed, "upstream.0").Msg(samplerTestMessage) + + if got := strings.Count(buf.String(), samplerTestMessage); got != 1 { + t.Fatalf("lines from the zero value sampler: got %d, want 1", got) + } + if !strings.Contains(buf.String(), `"level":"error"`) { + t.Fatalf("first line of a window is not at error level: %s", buf.String()) + } +} diff --git a/cmd/cli/service_others.go b/cmd/cli/service_others.go index 954b2287..17004080 100644 --- a/cmd/cli/service_others.go +++ b/cmd/cli/service_others.go @@ -4,14 +4,17 @@ package cli import ( "os" + "syscall" ) func hasElevatedPrivilege() (bool, error) { return os.Geteuid() == 0, nil } +// openLogFile opens a log file. It refuses a symlink, because ctrld often +// writes its logs as root into a directory that other users can write. func openLogFile(path string, flags int) (*os.File, error) { - return os.OpenFile(path, flags, os.FileMode(0o600)) + return os.OpenFile(path, flags|syscall.O_NOFOLLOW, os.FileMode(0o600)) } // hasLocalDnsServerRunning reports whether we are on Windows and having Dns server running. diff --git a/cmd/cli/upstream_monitor.go b/cmd/cli/upstream_monitor.go index 3eeb24b5..432d6110 100644 --- a/cmd/cli/upstream_monitor.go +++ b/cmd/cli/upstream_monitor.go @@ -1,6 +1,7 @@ package cli import ( + "strings" "sync" "time" @@ -17,6 +18,12 @@ const ( // the backoff so an unroutable endpoint is still re-probed periodically and // recovers once the route returns. checkUpstreamUnreachableBackoffMax = 60 * time.Second + // upstreamDownDelay is the time that the failures of an upstream must last + // before the monitor marks it down. + upstreamDownDelay = 10 * time.Second + // upstreamRecoveredDelay is the time that a reset holds back the failure + // counts of an upstream. + upstreamRecoveredDelay = 1 * time.Second ) // unreachableRecoveryBackoff returns the retry interval for the given streak of @@ -33,28 +40,55 @@ func unreachableRecoveryBackoff(streak int) time.Duration { return d } +// upstreamDownState remembers when an upstream went down and how many +// failures put it there, so the up event can report the outage. +type upstreamDownState struct { + since time.Time + failures uint64 +} + // upstreamMonitor performs monitoring upstreams health. type upstreamMonitor struct { cfg *ctrld.Config - mu sync.RWMutex + mu sync.RWMutex + // down holds one entry for each upstream that is down now. + down map[string]upstreamDownState checking map[string]bool - down map[string]bool failureReq map[string]uint64 recovered map[string]bool // failureTimerActive tracks if a timer is already running for a given upstream. failureTimerActive map[string]bool + // retired is set when a reload replaced this monitor. A timer that was + // armed before the reload must not open an outage that nothing closes. + retired bool + + // now is a seam, so a test can set the reported down time. + now func() time.Time + // after arms a one-shot timer, so a test fires the check itself instead of + // waiting for it. nil means time.AfterFunc. + after func(time.Duration, func()) +} + +// afterFunc returns the timer seam. A real timer drops its handle, because +// nothing stops these checks. +func (um *upstreamMonitor) afterFunc() func(time.Duration, func()) { + if um.after != nil { + return um.after + } + return func(d time.Duration, fn func()) { time.AfterFunc(d, fn) } } func newUpstreamMonitor(cfg *ctrld.Config) *upstreamMonitor { um := &upstreamMonitor{ cfg: cfg, checking: make(map[string]bool), - down: make(map[string]bool), + down: make(map[string]upstreamDownState), failureReq: make(map[string]uint64), recovered: make(map[string]bool), failureTimerActive: make(map[string]bool), + now: time.Now, } for n := range cfg.Upstream { upstream := upstreamPrefix + n @@ -71,6 +105,9 @@ func (um *upstreamMonitor) increaseFailureCount(upstream string) { um.mu.Lock() defer um.mu.Unlock() + if um.retired { + return + } if um.recovered[upstream] { mainLog.Load().Debug().Msgf("upstream %q is recovered, skipping failure count increase", upstream) return @@ -82,29 +119,115 @@ func (um *upstreamMonitor) increaseFailureCount(upstream string) { // Log the updated failure count. mainLog.Load().Debug().Msgf("upstream %q failure count updated to %d", upstream, failedCount) - // If this is the first failure and no timer is running, start a 10-second timer. + // If this is the first failure and no timer is running, start the timer. if failedCount == 1 && !um.failureTimerActive[upstream] { um.failureTimerActive[upstream] = true - go func(upstream string) { - time.Sleep(10 * time.Second) - um.mu.Lock() - defer um.mu.Unlock() - // If no success occurred during the 10-second window (i.e. counter remains > 0) - // and the upstream is not in a recovered state, mark it as down. - if um.failureReq[upstream] > 0 && !um.recovered[upstream] { - um.down[upstream] = true - mainLog.Load().Warn().Msgf("upstream %q marked as down after 10 seconds (failure count: %d)", upstream, um.failureReq[upstream]) - } - // Reset the timer flag so that a new timer can be spawned if needed. - um.failureTimerActive[upstream] = false - }(upstream) + um.afterFunc()(upstreamDownDelay, func() { um.markDownAfterDelay(upstream) }) } // If the failure count quickly reaches the threshold, mark the upstream as down immediately. if failedCount >= maxFailureRequest { - um.down[upstream] = true - mainLog.Load().Warn().Msgf("upstream %q marked as down immediately (failure count: %d)", upstream, failedCount) + um.markDown(upstream, failedCount, "immediate") + } +} + +// markDownAfterDelay marks an upstream down when its failures did not clear +// while the timer ran. A success during that time leaves the count at zero. +func (um *upstreamMonitor) markDownAfterDelay(upstream string) { + um.mu.Lock() + defer um.mu.Unlock() + if um.retired { + return + } + if um.failureReq[upstream] > 0 && !um.recovered[upstream] { + um.markDown(upstream, um.failureReq[upstream], "timer") + } + // The next failure of this upstream can arm a new timer. + um.failureTimerActive[upstream] = false +} + +// markDown marks an upstream as down and logs the transition once. The caller holds the lock. +func (um *upstreamMonitor) markDown(upstream string, failures uint64, reason string) { + if um.retired { + return + } + if _, down := um.down[upstream]; down { + return + } + um.down[upstream] = upstreamDownState{since: um.now(), failures: failures} + journal(mainLog.Load().Warn()). + Str("upstream", journalUpstreamName(upstream)). + Str("state", "down"). + Uint64("failure_count", failures). + Str("reason", reason). + Msg("Upstream state changed") +} + +// markUp marks an upstream as up and logs the transition once. The caller holds the lock. +func (um *upstreamMonitor) markUp(upstream, reason string) { + state, down := um.down[upstream] + if !down { + return + } + delete(um.down, upstream) + journal(mainLog.Load().Info()). + Str("upstream", journalUpstreamName(upstream)). + Str("state", "up"). + Str("reason", reason). + Int64("down_for_ms", um.now().Sub(state.since).Milliseconds()). + Uint64("failure_count", state.failures). + Msg("Upstream state changed") +} + +// journalUpstreamName bounds the upstream name that a journal event carries. +// ctrld generates upstream.os, upstream., and upstream.internal_; every +// other key is operator text that can hold a token, so it reads as +// upstream.custom. +func journalUpstreamName(upstream string) string { + if upstream == upstreamOS || generatedUpstreamKey(strings.TrimPrefix(upstream, upstreamPrefix)) { + return upstream + } + return upstreamPrefix + "custom" +} + +// generatedUpstreamKey reports whether key is a number, or internal_ and a +// number, the two shapes that ctrld writes itself. +func generatedUpstreamKey(key string) bool { + digits := strings.TrimPrefix(key, internalDomainUpstreamPrefix) + if digits == "" { + return false + } + for _, r := range digits { + if r < '0' || r > '9' { + return false + } + } + return true +} + +// retire closes the outage of every upstream that is still down and stops +// every later state change. A reload builds a new monitor, so the journal ends +// the outages of the old one, and the old timers must not open new ones. +func (um *upstreamMonitor) retire() { + um.mu.Lock() + defer um.mu.Unlock() + for upstream := range um.down { + um.markUp(upstream, "reload") } + um.retired = true + clear(um.failureReq) + clear(um.failureTimerActive) +} + +// noteSuccess records an answer from an upstream. It does not set the recovered +// flag, because that flag stops failure counts for one second and an answer +// must not hide the failures that follow it. +func (um *upstreamMonitor) noteSuccess(upstream string) { + um.mu.Lock() + defer um.mu.Unlock() + + um.failureReq[upstream] = 0 + um.markUp(upstream, "recovered") } // isDown reports whether the given upstream is being marked as down. @@ -112,23 +235,26 @@ func (um *upstreamMonitor) isDown(upstream string) bool { um.mu.Lock() defer um.mu.Unlock() - return um.down[upstream] + _, down := um.down[upstream] + return down } // reset marks an upstream as up and set failed queries counter to zero. func (um *upstreamMonitor) reset(upstream string) { um.mu.Lock() um.failureReq[upstream] = 0 - um.down[upstream] = false + um.markUp(upstream, "recovered") um.recovered[upstream] = true um.mu.Unlock() - go func() { - // debounce the recovery to avoid incrementing failure counts already in flight - time.Sleep(1 * time.Second) - um.mu.Lock() - um.recovered[upstream] = false - um.mu.Unlock() - }() + // The flag holds back the failure counts that are already in flight. + um.afterFunc()(upstreamRecoveredDelay, func() { um.clearRecovered(upstream) }) +} + +// clearRecovered lets the failures of an upstream count again. +func (um *upstreamMonitor) clearRecovered(upstream string) { + um.mu.Lock() + defer um.mu.Unlock() + um.recovered[upstream] = false } // countHealthy returns the number of upstreams in the provided map that are considered healthy. @@ -136,7 +262,7 @@ func (um *upstreamMonitor) countHealthy(upstreams []string) int { var count int um.mu.RLock() for _, upstream := range upstreams { - if !um.down[upstream] { + if _, down := um.down[upstream]; !down { count++ } } diff --git a/cmd/cli/upstream_monitor_test.go b/cmd/cli/upstream_monitor_test.go new file mode 100644 index 00000000..c45b4e25 --- /dev/null +++ b/cmd/cli/upstream_monitor_test.go @@ -0,0 +1,231 @@ +package cli + +import ( + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// upstreamStateMessage is the message of every upstream transition event. +const upstreamStateMessage = "Upstream state changed" + +// downMonitor returns a monitor on a clock the test owns, with the upstream down. +func downMonitor(t *testing.T, upstream string) (*upstreamMonitor, *syncBuffer, *time.Time) { + t.Helper() + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}) + now := time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + um.now = func() time.Time { return now } + for i := 0; i < maxFailureRequest; i++ { + um.increaseFailureCount(upstream) + } + if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 1 { + t.Fatalf("state events after the failures: got %d, want 1", got) + } + return um, logs, &now +} + +func Test_upstreamMonitorLogsOneDownEvent(t *testing.T) { + const upstream = upstreamPrefix + "0" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}) + + for i := 0; i < 633; i++ { + um.increaseFailureCount(upstream) + } + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 1 { + t.Fatalf("state events: got %d, want 1", len(events)) + } + wantField(t, events[0], "level", "warn") + wantField(t, events[0], "journal", true) + wantField(t, events[0], "upstream", upstream) + wantField(t, events[0], "state", "down") + wantField(t, events[0], "reason", "immediate") + wantField(t, events[0], "failure_count", float64(maxFailureRequest)) +} + +func Test_upstreamMonitorTimerLogsDownEvent(t *testing.T) { + const upstream = upstreamPrefix + "0" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}) + var armed []func() + um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + + um.increaseFailureCount(upstream) + if len(armed) != 1 { + t.Fatalf("armed timers: got %d, want 1", len(armed)) + } + armed[0]() + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 1 { + t.Fatalf("state events: got %d, want 1", len(events)) + } + wantField(t, events[0], "state", "down") + wantField(t, events[0], "reason", "timer") + wantField(t, events[0], "failure_count", float64(1)) +} + +func Test_upstreamMonitorResetLogsOneUpEvent(t *testing.T) { + const upstream = upstreamPrefix + "0" + um, logs, now := downMonitor(t, upstream) + + *now = now.Add(1500 * time.Millisecond) + um.reset(upstream) + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 2 { + t.Fatalf("state events: got %d, want 2", len(events)) + } + up := events[1] + wantField(t, up, "level", "info") + wantField(t, up, "journal", true) + wantField(t, up, "upstream", upstream) + wantField(t, up, "state", "up") + wantField(t, up, "reason", "recovered") + wantField(t, up, "down_for_ms", float64(1500)) + wantField(t, up, "failure_count", float64(maxFailureRequest)) +} + +func Test_upstreamMonitorRetireLogsTheOpenOutage(t *testing.T) { + const upstream = upstreamPrefix + "0" + um, logs, now := downMonitor(t, upstream) + + *now = now.Add(3 * time.Second) + um.retire() + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 2 { + t.Fatalf("state events: got %d, want 2", len(events)) + } + up := events[1] + wantField(t, up, "upstream", upstream) + wantField(t, up, "state", "up") + wantField(t, up, "reason", "reload") + wantField(t, up, "down_for_ms", float64(3000)) + + um.retire() + + if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 2 { + t.Fatalf("state events after the second retire: got %d, want 2", got) + } +} + +func Test_upstreamMonitorNoteSuccessLogsOneUpEvent(t *testing.T) { + const upstream = upstreamPrefix + "0" + um, logs, now := downMonitor(t, upstream) + + *now = now.Add(2 * time.Second) + um.noteSuccess(upstream) + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 2 { + t.Fatalf("state events: got %d, want 2", len(events)) + } + up := events[1] + wantField(t, up, "state", "up") + wantField(t, up, "reason", "recovered") + wantField(t, up, "down_for_ms", float64(2000)) + wantField(t, up, "failure_count", float64(maxFailureRequest)) + if got := um.failureReq[upstream]; got != 0 { + t.Fatalf("failure count after the answer: got %d, want 0", got) + } +} + +func Test_upstreamMonitorNoteSuccessOnUpUpstreamIsQuiet(t *testing.T) { + const upstream = upstreamPrefix + "0" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}) + + um.noteSuccess(upstream) + um.noteSuccess(upstream) + + if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 0 { + t.Fatalf("state events: got %d, want 0", got) + } +} + +// Test_upstreamMonitorRetiredTimerStaysSilent fires the down timer that a +// failure armed before the reload retired the monitor. A retired monitor +// owns no outage, so the timer must add no event. +func Test_upstreamMonitorRetiredTimerStaysSilent(t *testing.T) { + const upstream = upstreamPrefix + "0" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}) + var armed []func() + um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + + um.increaseFailureCount(upstream) + um.retire() + for _, fire := range armed { + fire() + } + + if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 0 { + t.Fatalf("state events from the retired monitor: got %d, want 0: %s", got, logs.String()) + } + if um.isDown(upstream) { + t.Fatal("the retired monitor marked the upstream down") + } +} + +// Test_upstreamMonitorRetiredAfterImmediateDownStaysSilent retires a monitor +// that already reported an outage. The armed timer and a late failure must +// add no event after the closing up event. +func Test_upstreamMonitorRetiredAfterImmediateDownStaysSilent(t *testing.T) { + const upstream = upstreamPrefix + "0" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}) + var armed []func() + um.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + + for i := 0; i < maxFailureRequest; i++ { + um.increaseFailureCount(upstream) + } + um.retire() + before := jsonLogEvents(t, logs, upstreamStateMessage) + if len(before) != 2 { + t.Fatalf("state events after the retire: got %d, want down and up", len(before)) + } + for _, fire := range armed { + fire() + } + um.increaseFailureCount(upstream) + + if got := len(jsonLogEvents(t, logs, upstreamStateMessage)); got != 2 { + t.Fatalf("state events after the retired timer fired: got %d, want 2: %s", got, logs.String()) + } + if um.isDown(upstream) { + t.Fatal("the retired monitor marked the upstream down again") + } +} + +// Test_upstreamMonitorKeepsACustomNameOutOfTheJournal marks an upstream with +// an operator-chosen key down. The key can hold a token, so the event names +// the upstream by a bounded name. +func Test_upstreamMonitorKeepsACustomNameOutOfTheJournal(t *testing.T) { + const upstream = upstreamPrefix + "dns.controld.com/org-v1-SECRET0123" + logs := captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}) + + for i := 0; i < maxFailureRequest; i++ { + um.increaseFailureCount(upstream) + } + um.noteSuccess(upstream) + + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 2 { + t.Fatalf("state events: got %d, want down and up", len(events)) + } + for _, event := range events { + wantField(t, event, "upstream", upstreamPrefix+"custom") + } + for _, generated := range []string{upstreamOS, upstreamPrefix + "0", upstreamPrefix + "internal_3"} { + if got := journalUpstreamName(generated); got != generated { + t.Fatalf("journalUpstreamName(%q) = %q, want the name itself", generated, got) + } + } +} diff --git a/config.go b/config.go index fd94675e..90363259 100644 --- a/config.go +++ b/config.go @@ -218,6 +218,8 @@ func (c *Config) FirstUpstream() *UpstreamConfig { type ServiceConfig struct { LogLevel string `mapstructure:"log_level" toml:"log_level,omitempty"` LogPath string `mapstructure:"log_path" toml:"log_path,omitempty"` + LogMaxSizeMB int `mapstructure:"log_max_size_mb" toml:"log_max_size_mb,omitempty" validate:"omitempty,gte=1,lte=1024"` + LogMaxBackups *int `mapstructure:"log_max_backups" toml:"log_max_backups,omitempty" validate:"omitempty,gte=0,lte=64"` CacheEnable bool `mapstructure:"cache_enable" toml:"cache_enable,omitempty"` CacheSize int `mapstructure:"cache_size" toml:"cache_size,omitempty"` CacheTTLOverride int `mapstructure:"cache_ttl_override" toml:"cache_ttl_override,omitempty"` diff --git a/config_test.go b/config_test.go index cd392d50..bb2c7b28 100644 --- a/config_test.go +++ b/config_test.go @@ -169,6 +169,72 @@ discover_ptr = false require.False(t, *cfg.Service.DiscoverPtr) } +func TestLogMaxKeys(t *testing.T) { + tests := []struct { + name string + serviceKeys string + wantSize int + wantBackups *int + }{ + {"size and backups set", "log_max_size_mb = 20\nlog_max_backups = 1", 20, intPtr(1)}, + {"size zero, backups absent", "log_max_size_mb = 0", 0, nil}, + {"backups zero", "log_max_backups = 0", 0, intPtr(0)}, + {"the largest values the keys take", "log_max_size_mb = 1024\nlog_max_backups = 64", 1024, intPtr(64)}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := configWithServiceKeys(t, tc.serviceKeys) + validate := validator.New() + require.NoError(t, ctrld.ValidateConfig(validate, cfg)) + require.Equal(t, tc.wantSize, cfg.Service.LogMaxSizeMB) + if tc.wantBackups == nil { + require.Nil(t, cfg.Service.LogMaxBackups) + return + } + require.NotNil(t, cfg.Service.LogMaxBackups) + require.Equal(t, *tc.wantBackups, *cfg.Service.LogMaxBackups) + }) + } +} + +func TestLogMaxKeysOutOfRange(t *testing.T) { + tests := []struct { + name string + serviceKeys string + wantField string + }{ + {"negative size", "log_max_size_mb = -1", "LogMaxSizeMB"}, + {"negative backups", "log_max_backups = -1", "LogMaxBackups"}, + {"size above the limit", "log_max_size_mb = 1025", "LogMaxSizeMB"}, + {"backups above the limit", "log_max_backups = 65", "LogMaxBackups"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := configWithServiceKeys(t, tc.serviceKeys) + validate := validator.New() + err := ctrld.ValidateConfig(validate, cfg) + require.ErrorContains(t, err, tc.wantField) + }) + } +} + +func configWithServiceKeys(t *testing.T, serviceKeys string) *ctrld.Config { + t.Helper() + v := viper.NewWithOptions(viper.KeyDelimiter("::")) + ctrld.InitConfig(v, "test_log_max_keys") + v.SetConfigType("toml") + require.NoError(t, v.ReadConfig(strings.NewReader("[service]\n"+serviceKeys+"\n"))) + var cfg ctrld.Config + require.NoError(t, v.Unmarshal(&cfg)) + return &cfg +} + +func intPtr(n int) *int { + return &n +} + func defaultConfig(t *testing.T) *ctrld.Config { v := viper.New() ctrld.InitConfig(v, "test_load_default_config") diff --git a/docs/config.md b/docs/config.md index 8f21a4c2..17beb7c4 100644 --- a/docs/config.md +++ b/docs/config.md @@ -121,10 +121,28 @@ Logging level you wish to enable. ### log_path Relative or absolute path of the log file. +Point `log_path` at a file that no other program rotates. ctrld renames `.1` to `.N` and uploads these files. A lower `log_max_backups` leaves the older numbered `log_path` files in place, but ctrld removes the extra numbered files of its internal logs. + - Type: string - Required: no - Default: "" +### log_max_size_mb +Maximum size of the debug log file in MB. When the file reaches this size, ctrld rotates it. This limit also applies to the `log_path` file. + +- Type: integer +- Required: no +- Valid values: 1 to 1024 +- Default: 10 on desktop platforms, 5 on routers + +### log_max_backups +Number of rotated debug log files that ctrld keeps. If the value is 0, ctrld keeps no rotated file. This number also applies to the `log_path` file. + +- Type: integer +- Required: no +- Valid values: 0 to 64 +- Default: 4 on desktop platforms, 1 on routers + ### cache_enable When `cache_enable = true`, all resolved DNS query responses will be cached for duration of the upstream record TTLs. diff --git a/docs/internal-domains.md b/docs/internal-domains.md index 44caad62..1630b8de 100644 --- a/docs/internal-domains.md +++ b/docs/internal-domains.md @@ -153,7 +153,10 @@ removed domain leaves no rule and no `internal_*` upstream behind. Generated `internal_*` upstreams carry the organization's private resolver addresses, so anything naming one stays at debug, and the level above it carries only a classification (`timeout`, `refused`, `unreachable`, `source_unavailable`, -`error`). That covers every consumer, not just query handling: +`error`). The per-query error sampler applies to the classification line as +well. After 5 lines in one minute for the same upstream and class, the next +classification lines go to debug, and one `Per-query errors sampled` line gives +the count. That covers every consumer, not just query handling: - the bootstrap-IP line in `setupUpstream`; - the transport error from a failed query, which names the endpoint; diff --git a/docs/network-recovery-diagnostics.md b/docs/network-recovery-diagnostics.md index e7c18c68..856c679d 100644 --- a/docs/network-recovery-diagnostics.md +++ b/docs/network-recovery-diagnostics.md @@ -1,9 +1,12 @@ # Network-recovery diagnostics -Use continuous `ctrld log tail` capture during a reproduction. Internal logs rotate by size. -A later `log send` can contain recent debug events and older warnings. -Match timestamps and IDs. The last retained error does not necessarily describe the current network. -Issue #604 tracks upload-time device and version headers separately. +The retained stream is the journal, the file `ctrld-journal.log` in the ctrld home directory. +It keeps every warning and every error, plus every event marked `journal=true`. +The journal is on disk, so it keeps these events across a restart and a self-upgrade. +Its budget is 2 MB with 2 backup files on a desktop, and 1 MB with 1 backup file on a router. +Every `ctrld log send` upload holds the whole journal, so a reproduction does not need a continuous `ctrld log tail` capture. +Match timestamps and IDs. The last retained error does not describe the current network in every case. +Each upload starts with a header line that names the build, the intercept mode, and the network at send time. ## Transitions and recovery @@ -16,10 +19,10 @@ For minor callbacks and late major callbacks in that epoch, ctrld reads current This read does not change host DNS. A failed read defers source writes, but reconciliation continues. `Removed stale resolver source` is a warning only when ctrld removes an invalid address. -The warning gives the reason and survives debug rotation. +The warning gives the reason. Every warning enters the journal, so it stays on disk after a debug rotation and after a restart. `Recovery begin` and `Recovery end` connect the transition to a `recovery_generation`, reason, outcome, and elapsed time. Only the first upstream failure in each recovery produces a bounded summary. -If a recovery encounters a failure, its final outcome also enters the warning stream. +If a recovery encounters a failure, its final outcome also enters the journal. Normal successful recovery stays at debug level. These summaries exclude upstream URLs and queried names. @@ -43,9 +46,9 @@ The cancellation event alone does not prove that interface DNS remains removed. A changed unsuccessful condition produces a warning. The condition consists of the stage, error code, outcome, and target. Equal results stay at debug level. The next changed condition includes the preceding repeat count. -The first restored receipt also produces a warning-stream event. This event prevents an older failure from appearing current. +The first restored receipt also produces a warning, which enters the journal. This event prevents an older failure from appearing current. Normal receipts and shutdown cancellation stay at debug level. -The warning buffer is finite. These events do not guarantee indefinite retention. +The journal budget is finite and the journal rotates by size. It does not keep these events without a limit. The outcomes have these meanings: diff --git a/docs/runtime-internal-logging.md b/docs/runtime-internal-logging.md index 982632cb..8a52cadc 100644 --- a/docs/runtime-internal-logging.md +++ b/docs/runtime-internal-logging.md @@ -1,46 +1,94 @@ -# Runtime Internal Logging - -When no logging is configured (i.e., `log_path` is not set), ctrld automatically enables an internal logging system. This system stores logs in memory to provide troubleshooting information when problems occur. +# Runtime internal logging ## Purpose -The runtime internal logging system is designed primarily for **ctrld developers**, not end users. It captures detailed diagnostic information that can be useful for troubleshooting issues when they arise, especially in production environments where explicit logging may not be configured. +ctrld writes its own log when the configuration sets no log file. The log gives Control D support and ctrld developers the evidence they need after a problem. Two of the three streams are files on disk. They keep their content across a restart and a self-upgrade. -## When It's Enabled +## When it is on -Internal logging is automatically enabled when: +Internal logging is on when these three conditions are true at the same time: -- ctrld is running in Control D mode (i.e., `--cd` flag is provided) -- No log file is configured (i.e., `log_path` is empty or not set) +- ctrld runs in Control D mode, with the `--cd` flag +- The configuration sets no `log_path` +- Silent mode is off -If a log file is explicitly configured via `log_path`, internal logging will **not** be enabled, as the configured log file serves the logging purpose. +When `log_path` is set, ctrld writes that file and starts no internal stream. The journal needs internal logging, so `log_path` mode has no journal. -## How It Works +## Streams and files -The internal logging system: +ctrld writes three streams. The debug file and the journal file live in the ctrld home directory, next to each other. -- Stores logs in **in-memory buffers** (not written to disk) -- Captures logs at **debug level** for normal operations and **warn level** for warnings -- Maintains separate buffers for normal logs and warning logs -- Automatically manages buffer size to prevent unbounded memory growth -- Preserves initialization logs even when buffers overflow +| Stream | Level filter | File | Desktop budget | Router budget | Survives a restart | +|---|---|---|---|---|---| +| Console | Notice and above. The `-v` flag adds the lower levels | none | none | none | No | +| Debug | Every level | `ctrld.log` | 10 MB, 4 backup files | 5 MB, 1 backup file | Yes | +| Journal | Warning and above, plus every event with `journal=true` | `ctrld-journal.log` | 2 MB, 2 backup files | 1 MB, 1 backup file | Yes | -## Configuration +The `-v` flag changes the level of every stream, the files included. Without the flag, ctrld sets the level to debug and holds the console at notice. With `-v` the level is info, so no debug line reaches the debug file. With `-vv` the level is debug again, and the console shows every line. -**Important**: The `log_level` configuration option does **not** affect the internal logging system. Internal logging always operates at debug level for normal logs and warn level for warnings, regardless of the `log_level` setting in the configuration file. +A backup file carries a number suffix. On a desktop the debug backup files are `ctrld.log.1` to `ctrld.log.4`, and `ctrld.log.1` holds the newest rotated content. The journal backup files are `ctrld-journal.log.1` and `ctrld-journal.log.2`. -The `log_level` setting only affects: -- Console output (when running interactively) -- File-based logging (when `log_path` is configured) +Rotation is by size. A write that crosses the size limit rotates the file first. ctrld deletes the file beyond the backup count. -## Accessing Internal Logs +Each stream also keeps a memory buffer: 5 MB for the debug stream and 1 MB for the journal. These buffers serve `ctrld log tail` and the fallback path when a file cannot open. The files hold the history. -Internal logs can be accessed through the control server API endpoints. This functionality is intended for developers and support personnel who need to diagnose issues. +A restart deletes the memory buffers and keeps the files. The new process appends a header line, which marks the restart point inside each file. -## Notes +## Header and rotation events + +Every debug file and every journal file starts with one JSON line, the header. ctrld writes the header when it opens a file at start and after every rotation. On an existing file at start, ctrld appends the header. + +The message of the line is `Log header`. The line carries these fields: + +- Build: `version` and `commit` +- Host: `os`, `arch`, `pid`, and `start_time` +- Mode: `intercept_mode`, `listeners`, `upstream_count`, and `upstream_types` +- Identity: `resolver_uid` +- Files: `log_files` +- Network: `network` with `default_route_interface`, `have_v4`, `have_v6`, and `interfaces` + +The header holds no provisioning token, no upstream endpoint URL, and no query name. `upstream_types` holds the type words, for example `doh` or `os`. `listeners` holds the listen addresses. + +After each rotation ctrld logs one `Log rotated` event at info level, with the fields `file`, `bytes_written`, `first_event_at`, `last_event_at`, and `backups`. This event is a journal event, so it lands in the debug stream and in the journal. + +## Sampling of per-query errors + +A broken network produces one error line per query. ctrld bounds this flood for each error class and each upstream. The first 5 lines in a 60 s window stay at error level. Later lines in the same window drop to debug level, so the debug stream stays complete. A run with `-v` holds every stream at info, and the debug stream then loses these lines. + +When a window closes, ctrld logs one `Per-query errors sampled` summary at error level. The summary carries `class`, `upstream`, `count`, `suppressed`, and `window_s`. ctrld logs the summary only when the window suppressed at least one line. + +An upstream logs one `Upstream state changed` event per transition. The down event is at warn level and carries `state` `down`, `failure_count`, and `reason`. The up event is at info level and carries `state` `up`, `down_for_ms`, and `failure_count`. Both are journal events. + +## Configuration keys -- Internal logging is **not** a replacement for proper log file configuration in production environments -- For production deployments, it is recommended to configure `log_path` to enable persistent file-based logging -- Internal logs are stored in memory and will be lost if the process terminates unexpectedly -- The internal logging system is automatically disabled when explicit logging is configured +The `log_level` key does not change the internal streams. The debug file keeps every level, and the journal keeps the warnings, the errors, and the journal events. The `log_level` key changes the console output and the `log_path` file. The `-v` flag changes every stream, as the section "Streams and files" explains. + +Two keys in the `[service]` section size the debug stream: + +- The key `log_max_size_mb` sets the size limit of `ctrld.log` in MB. The default is 10 on a desktop and 5 on a router. +- The key `log_max_backups` sets the number of backup files. The default is 4 on a desktop and 1 on a router. The value 0 keeps no backup file. + +Both keys also apply to the `log_path` file, which rotates the same way. The journal budget is fixed per platform. + +## Uploads + +`ctrld log send` uploads the log to Control D. The body holds these parts, in this order: + +- One header line rendered at send time, with the field `trigger` set to `send` +- The newest 10 MB of the debug files, oldest file first, cut at a line boundary +- The marker `=== LOG_END ===` +- Every journal file, oldest first + +The `network` object of the send-time header comes from a fresh read of the interfaces and the default route. `ctrld log send --full` sends every debug file in place of the newest 10 MB. The reported size equals the number of bytes uploaded. + +`ctrld log view` prints the same composition on the terminal and takes the same `--full` flag. ctrld accepts one upload per minute. The body is plain text without gzip. + +In `log_path` mode the body holds the send-time header, then the `log_path` file and its backup files, oldest first. This body has no marker and no journal. + +`ctrld log tail` follows the live debug stream and prints each new line, journal events included. + +## Notes +- Internal logging is not a replacement for a configured `log_path` file in production. +- The journal is small on purpose. It keeps the events that describe a problem, not the events of each query. +- Support reads the upload without access to the host. The header line names the build, the mode, and the network, so support does not have to ask for them. From 8ae09ac98e4598bff28d7d13f3fd9707baf698df Mon Sep 17 00:00:00 2001 From: Anthony Wong Date: Fri, 18 Sep 2026 13:57:27 +0000 Subject: [PATCH 10/29] feat(cli): put the network state of the host in the log journal The journal records the network state of the host and each change of that state. The support team reads one file to see the interfaces, the routes, the resolvers, and the recoveries around an incident. The same file shows the health of the query path. A change that touches only AirDrop or virtual interfaces runs no DNS work. --- cmd/cli/change_logger.go | 205 +++++++ cmd/cli/change_logger_test.go | 198 +++++++ cmd/cli/dns64.go | 40 +- cmd/cli/dns64_test.go | 58 ++ cmd/cli/dns_config.go | 360 ++++++++++++ cmd/cli/dns_config_darwin.go | 22 + cmd/cli/dns_config_others.go | 9 + cmd/cli/dns_config_test.go | 598 +++++++++++++++++++ cmd/cli/dns_intercept_darwin.go | 218 ++++++- cmd/cli/dns_intercept_darwin_test.go | 537 ++++++++++++++++- cmd/cli/dns_intercept_settle.go | 13 +- cmd/cli/dns_intercept_settle_test.go | 7 +- cmd/cli/dns_intercept_windows.go | 2 +- cmd/cli/dns_proxy.go | 283 ++++++--- cmd/cli/dns_target_darwin.go | 31 +- cmd/cli/dns_target_darwin_test.go | 105 +++- cmd/cli/intercept_target.go | 15 + cmd/cli/log_header.go | 135 +---- cmd/cli/log_header_test.go | 126 ++-- cmd/cli/log_journal.go | 8 +- cmd/cli/log_journal_test.go | 1 + cmd/cli/log_writer_silent_test.go | 2 + cmd/cli/net_darwin.go | 268 ++++++++- cmd/cli/net_darwin_test.go | 315 ++++++++++ cmd/cli/net_linux.go | 131 ++++- cmd/cli/net_linux_test.go | 201 +++++++ cmd/cli/net_others.go | 15 + cmd/cli/net_windows.go | 213 ++++++- cmd/cli/net_windows_test.go | 141 +++++ cmd/cli/network_delta.go | 301 ++++++++++ cmd/cli/network_delta_test.go | 488 ++++++++++++++++ cmd/cli/network_events.go | 167 ++++++ cmd/cli/network_events_test.go | 804 ++++++++++++++++++++++++++ cmd/cli/network_snapshot.go | 301 ++++++++++ cmd/cli/network_snapshot_prog.go | 226 ++++++++ cmd/cli/network_snapshot_prog_test.go | 348 +++++++++++ cmd/cli/network_snapshot_test.go | 557 ++++++++++++++++++ cmd/cli/network_transition.go | 36 +- cmd/cli/network_transition_test.go | 7 +- cmd/cli/pf_state.go | 82 +++ cmd/cli/pf_state_darwin.go | 119 ++++ cmd/cli/pf_state_test.go | 137 +++++ cmd/cli/prog.go | 163 ++++-- cmd/cli/query_error_sampler.go | 12 + cmd/cli/query_health.go | 284 +++++++++ cmd/cli/query_health_prog_test.go | 459 +++++++++++++++ cmd/cli/query_health_test.go | 424 ++++++++++++++ cmd/cli/recovery_events_test.go | 349 +++++++++++ cmd/cli/recovery_state.go | 4 +- cmd/cli/recovery_state_test.go | 2 +- cmd/cli/route_darwin.go | 158 +++++ cmd/cli/route_darwin_test.go | 291 ++++++++++ cmd/cli/route_others.go | 10 + cmd/cli/tunnel_owner.go | 35 ++ cmd/cli/tunnel_owner_test.go | 83 +++ cmd/cli/upstream_monitor.go | 38 ++ cmd/cli/upstream_monitor_test.go | 55 ++ cmd/cli/vpn_dns.go | 12 +- cmd/cli/vpn_dns_test.go | 51 ++ docs/dns-intercept-mode.md | 10 +- docs/network-recovery-diagnostics.md | 138 ++++- docs/pf-dns-intercept.md | 7 + docs/runtime-internal-logging.md | 17 +- log.go | 9 + net.go | 24 +- net_ipv6_test.go | 37 ++ resolver.go | 161 +++++- resolver_test.go | 233 ++++++++ 68 files changed, 10456 insertions(+), 440 deletions(-) create mode 100644 cmd/cli/change_logger.go create mode 100644 cmd/cli/change_logger_test.go create mode 100644 cmd/cli/dns_config.go create mode 100644 cmd/cli/dns_config_darwin.go create mode 100644 cmd/cli/dns_config_others.go create mode 100644 cmd/cli/dns_config_test.go create mode 100644 cmd/cli/intercept_target.go create mode 100644 cmd/cli/net_darwin_test.go create mode 100644 cmd/cli/net_linux_test.go create mode 100644 cmd/cli/network_delta.go create mode 100644 cmd/cli/network_delta_test.go create mode 100644 cmd/cli/network_events.go create mode 100644 cmd/cli/network_events_test.go create mode 100644 cmd/cli/network_snapshot.go create mode 100644 cmd/cli/network_snapshot_prog.go create mode 100644 cmd/cli/network_snapshot_prog_test.go create mode 100644 cmd/cli/network_snapshot_test.go create mode 100644 cmd/cli/pf_state.go create mode 100644 cmd/cli/pf_state_darwin.go create mode 100644 cmd/cli/pf_state_test.go create mode 100644 cmd/cli/query_health.go create mode 100644 cmd/cli/query_health_prog_test.go create mode 100644 cmd/cli/query_health_test.go create mode 100644 cmd/cli/recovery_events_test.go create mode 100644 cmd/cli/route_darwin.go create mode 100644 cmd/cli/route_darwin_test.go create mode 100644 cmd/cli/route_others.go create mode 100644 cmd/cli/tunnel_owner.go create mode 100644 cmd/cli/tunnel_owner_test.go create mode 100644 net_ipv6_test.go diff --git a/cmd/cli/change_logger.go b/cmd/cli/change_logger.go new file mode 100644 index 00000000..52601d8c --- /dev/null +++ b/cmd/cli/change_logger.go @@ -0,0 +1,205 @@ +package cli + +import ( + "maps" + "slices" + "sync" + "tailscale.com/net/netmon" + "time" +) + +// noiseSummaryInterval bounds a storm of noise deltas to one journal line, +// because AirDrop and virtual adapters can churn for hours. +const noiseSummaryInterval = 10 * time.Minute + +// wakeReportInterval merges the reports of the wake sources. netmon polls for +// a time jump every 15 s, so one wake can reach ctrld over half a minute. +const wakeReportInterval = 30 * time.Second + +// repeatLogger holds the last value of each named line, so a line that +// repeats every few seconds reaches the log on change only. +type repeatLogger struct { + mu sync.Mutex + last map[string]string + repeats map[string]uint64 +} + +// changed reports whether the value of a key differs from the value before it. +// The repeat count tells how many lines the caller held back, so the next line +// it writes can carry them. +func (r *repeatLogger) changed(key, value string) (changed bool, repeats uint64) { + r.mu.Lock() + defer r.mu.Unlock() + if r.last == nil { + r.last = make(map[string]string) + r.repeats = make(map[string]uint64) + } + if previous, seen := r.last[key]; seen && previous == value { + r.repeats[key]++ + return false, r.repeats[key] + } + repeats = r.repeats[key] + r.last[key] = value + r.repeats[key] = 0 + return true, repeats +} + +// wakeHoldInterval is the time that a wake report without a gap waits for the +// detector. The detector ticks every two seconds, so it reports well inside +// this hold when it runs. +const wakeHoldInterval = 8 * time.Second + +// wakeReport is one wake as a source saw it. +type wakeReport struct { + source string + gap time.Duration + state *netmon.State +} + +// wakeReporter keeps one report per wake. netmon measures no gap, so its +// report waits for the detector, which measures one. When no detector report +// comes, the held report goes out as it is. +type wakeReporter struct { + mu sync.Mutex + last time.Time + held *wakeReport + // after arms the hold timer. nil means time.AfterFunc, and a test fires + // the hold itself. + after func(time.Duration, func()) +} + +// note takes one report of a wake and hands the report that the journal gets +// to emit. The first source of a window reports. A report with a gap replaces +// a held report without one. Every other report of the window is the same +// wake and drops. +func (w *wakeReporter) note(now time.Time, report wakeReport, emit func(wakeReport)) { + w.mu.Lock() + defer w.mu.Unlock() + gapKnown := report.gap > 0 + if !w.last.IsZero() && now.Sub(w.last) < wakeReportInterval { + if w.held != nil && gapKnown { + w.held = nil + emit(report) + } + return + } + w.last = now + if gapKnown { + w.held = nil + emit(report) + return + } + held := &wakeReport{source: report.source, gap: report.gap, state: report.state} + w.held = held + w.afterFunc()(wakeHoldInterval, func() { w.release(held, emit) }) +} + +// release emits a held report when no source with a gap replaced it. +func (w *wakeReporter) release(held *wakeReport, emit func(wakeReport)) { + w.mu.Lock() + defer w.mu.Unlock() + if w.held != held { + return + } + w.held = nil + emit(*held) +} + +func (w *wakeReporter) afterFunc() func(time.Duration, func()) { + if w.after != nil { + return w.after + } + return func(d time.Duration, fn func()) { time.AfterFunc(d, fn) } +} + +// noiseSummary describes the noise deltas that one journal line covers. +type noiseSummary struct { + Count int + First time.Time + Last time.Time + Interfaces []string +} + +// noiseCoalescer counts the noise deltas of a storm and reports when the +// journal needs a line for them. The count, the first time, and the names +// cover the open window; last covers the storm, so a quiet period ends it. +type noiseCoalescer struct { + mu sync.Mutex + count int + first time.Time + last time.Time + emitted time.Time + names map[string]struct{} +} + +// add records one noise delta and reports whether to log a summary now. The +// summary covers the deltas since the last one, so a storm of any length +// keeps to one line per interval. +func (c *noiseCoalescer) add(interfaces []string, now time.Time) (summary noiseSummary, emit bool) { + c.mu.Lock() + defer c.mu.Unlock() + newRun := c.stormEnded(now) + if newRun { + c.clearWindow() + } + c.record(interfaces, now) + if !newRun && now.Sub(c.emitted) < noiseSummaryInterval { + return noiseSummary{}, false + } + summary = c.summary() + c.emitted = now + c.clearWindow() + return summary, true +} + +// flush reports the open window and closes it. A storm ends with a delta that +// the daemon acts on, so the last window of the storm reaches the journal +// there instead of waiting for the next storm. +func (c *noiseCoalescer) flush(now time.Time) (summary noiseSummary, emit bool) { + c.mu.Lock() + defer c.mu.Unlock() + if c.count == 0 { + return noiseSummary{}, false + } + summary = c.summary() + c.emitted = now + c.clearWindow() + return summary, true +} + +// stormEnded is true when the deltas stopped for longer than the interval, so +// a new storm does not inherit the count of the storm before it. +func (c *noiseCoalescer) stormEnded(now time.Time) bool { + return c.last.IsZero() || now.Sub(c.last) > noiseSummaryInterval +} + +func (c *noiseCoalescer) record(interfaces []string, now time.Time) { + if c.count == 0 { + c.first = now + } + c.count++ + c.last = now + if c.names == nil { + c.names = make(map[string]struct{}) + } + for _, name := range interfaces { + c.names[name] = struct{}{} + } +} + +// summary sorts the names, because a log line that keeps its order reads as +// one value across a capture. +func (c *noiseCoalescer) summary() noiseSummary { + return noiseSummary{ + Count: c.count, + First: c.first, + Last: c.last, + Interfaces: slices.Sorted(maps.Keys(c.names)), + } +} + +func (c *noiseCoalescer) clearWindow() { + c.count = 0 + c.first = time.Time{} + c.names = nil +} diff --git a/cmd/cli/change_logger_test.go b/cmd/cli/change_logger_test.go new file mode 100644 index 00000000..4292c4c9 --- /dev/null +++ b/cmd/cli/change_logger_test.go @@ -0,0 +1,198 @@ +package cli + +import ( + "slices" + "testing" + "time" +) + +func Test_repeatLogger(t *testing.T) { + var repeats repeatLogger + for _, step := range []struct { + name string + key string + value string + wantChanged bool + wantRepeats uint64 + }{ + {"the first value of a key is a change", "pf anchor", "intact", true, 0}, + {"the same value repeats", "pf anchor", "intact", false, 1}, + {"the same value repeats again", "pf anchor", "intact", false, 2}, + {"a new value reports the run before it", "pf anchor", "missing", true, 2}, + {"the new value repeats", "pf anchor", "missing", false, 1}, + {"the value before is a change again", "pf anchor", "intact", true, 1}, + {"a second key starts on its own", "tunnels", "utun0", true, 0}, + {"the second key repeats", "tunnels", "utun0", false, 1}, + {"the first key keeps its own count", "pf anchor", "intact", false, 1}, + } { + t.Run(step.name, func(t *testing.T) { + changed, count := repeats.changed(step.key, step.value) + if changed != step.wantChanged || count != step.wantRepeats { + t.Errorf("changed(%q, %q) = %v, %d, want %v, %d", + step.key, step.value, changed, count, step.wantChanged, step.wantRepeats) + } + }) + } +} + +func Test_wakeReporter(t *testing.T) { + base := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + var wake wakeReporter + var armed []func() + wake.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + var emitted []wakeReport + emit := func(report wakeReport) { emitted = append(emitted, report) } + fireHeld := func() { + for _, fn := range armed { + fn() + } + armed = nil + } + + // netmon reports first and knows no gap, so its report waits. + wake.note(base, wakeReport{source: "netmon"}, emit) + if len(emitted) != 0 || len(armed) != 1 { + t.Fatalf("a report without a gap must wait: emitted %d, timers %d", len(emitted), len(armed)) + } + // The detector reports inside the hold with the gap, so it replaces the + // held report. + wake.note(base.Add(2*time.Second), wakeReport{source: "detector", gap: 90 * time.Second}, emit) + fireHeld() + if len(emitted) != 1 || emitted[0].source != "detector" { + t.Fatalf("emitted %+v, want the detector report alone", emitted) + } + // Every other report of this wake is the same wake. + wake.note(base.Add(10*time.Second), wakeReport{source: "netmon"}, emit) + wake.note(base.Add(12*time.Second), wakeReport{source: "detector", gap: 91 * time.Second}, emit) + if len(emitted) != 1 { + t.Fatalf("emitted %d reports inside the window, want 1", len(emitted)) + } + + // A wake that only netmon sees reports without a gap when the hold ends. + wake.note(base.Add(40*time.Second), wakeReport{source: "netmon"}, emit) + if len(emitted) != 1 || len(armed) != 1 { + t.Fatalf("a new wake must arm one hold: emitted %d, timers %d", len(emitted), len(armed)) + } + fireHeld() + if len(emitted) != 2 || emitted[1].source != "netmon" || emitted[1].gap != 0 { + t.Fatalf("emitted %+v, want the netmon report after the hold", emitted) + } + // A gap that arrives after the hold is too late: one event per wake. + wake.note(base.Add(50*time.Second), wakeReport{source: "detector", gap: 92 * time.Second}, emit) + if len(emitted) != 2 { + t.Fatalf("emitted %d reports, want no second report of one wake", len(emitted)) + } + + // The detector first reports at once, and netmon adds nothing. + wake.note(base.Add(80*time.Second), wakeReport{source: "detector", gap: 93 * time.Second}, emit) + wake.note(base.Add(82*time.Second), wakeReport{source: "netmon"}, emit) + fireHeld() + if len(emitted) != 3 || emitted[2].source != "detector" || len(armed) != 0 { + t.Fatalf("emitted %+v with %d timers, want one detector report and no hold", emitted, len(armed)) + } +} + +func Test_noiseCoalescer(t *testing.T) { + base := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + + t.Run("a storm reports once every ten minutes", func(t *testing.T) { + const deltas = 100 + const cadence = 15 * time.Second + var noise noiseCoalescer + var got []noiseSummary + for i := 0; i < deltas; i++ { + interfaces := []string{"awdl0"} + if i%2 == 1 { + interfaces = []string{"llw0", "awdl0"} + } + summary, emit := noise.add(interfaces, base.Add(time.Duration(i)*cadence)) + if !emit { + continue + } + got = append(got, summary) + } + want := []noiseSummary{{ + Count: 1, + First: base, + Last: base, + Interfaces: []string{"awdl0"}, + }, { + Count: 40, + First: base.Add(cadence), + Last: base.Add(10 * time.Minute), + Interfaces: []string{"awdl0", "llw0"}, + }, { + Count: 40, + First: base.Add(10*time.Minute + cadence), + Last: base.Add(20 * time.Minute), + Interfaces: []string{"awdl0", "llw0"}, + }} + assertSummaries(t, got, want) + }) + + t.Run("a flush reports the open window", func(t *testing.T) { + var noise noiseCoalescer + if _, emit := noise.flush(base); emit { + t.Fatal("an empty coalescer reported a summary") + } + for _, offset := range []time.Duration{0, time.Minute, 2 * time.Minute} { + noise.add([]string{"awdl0"}, base.Add(offset)) + } + summary, emit := noise.flush(base.Add(3 * time.Minute)) + if !emit { + t.Fatal("the flush reported no summary for an open window") + } + assertSummaries(t, []noiseSummary{summary}, []noiseSummary{{ + Count: 2, + First: base.Add(time.Minute), + Last: base.Add(2 * time.Minute), + Interfaces: []string{"awdl0"}, + }}) + if _, emit = noise.flush(base.Add(4 * time.Minute)); emit { + t.Fatal("the flush reported the same window twice") + } + }) + + t.Run("a gap longer than the interval starts a new run", func(t *testing.T) { + var noise noiseCoalescer + var got []noiseSummary + for _, offset := range []time.Duration{0, time.Minute, 2 * time.Minute, 17 * time.Minute} { + summary, emit := noise.add([]string{"awdl0"}, base.Add(offset)) + if !emit { + continue + } + got = append(got, summary) + } + want := []noiseSummary{{ + Count: 1, + First: base, + Last: base, + Interfaces: []string{"awdl0"}, + }, { + Count: 1, + First: base.Add(17 * time.Minute), + Last: base.Add(17 * time.Minute), + Interfaces: []string{"awdl0"}, + }} + assertSummaries(t, got, want) + }) +} + +func assertSummaries(t *testing.T, got, want []noiseSummary) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("got %d summaries, want %d: %+v", len(got), len(want), got) + } + for i := range want { + if !sameSummary(got[i], want[i]) { + t.Errorf("summary %d = %+v, want %+v", i, got[i], want[i]) + } + } +} + +func sameSummary(got, want noiseSummary) bool { + return got.Count == want.Count && + got.First.Equal(want.First) && + got.Last.Equal(want.Last) && + slices.Equal(got.Interfaces, want.Interfaces) +} diff --git a/cmd/cli/dns64.go b/cmd/cli/dns64.go index 352a1b1d..b8d39899 100644 --- a/cmd/cli/dns64.go +++ b/cmd/cli/dns64.go @@ -293,6 +293,15 @@ func (p *prog) activeDNS64Prefix() (netip.Prefix, bool) { return p.dns64.prefix, p.dns64.prefix.IsValid() } +// dns64PrefixSnapshot reports the stored NAT64 prefix and whether synthesis +// runs. It reads the state only, because a report of the network must not +// reclassify it or start a discovery query. +func (p *prog) dns64PrefixSnapshot() (netip.Prefix, bool) { + p.dns64.mu.Lock() + defer p.dns64.mu.Unlock() + return p.dns64.prefix, p.dns64.active +} + func dns64CacheVariant(prefix netip.Prefix) string { return "dns64:" + prefix.String() } @@ -398,13 +407,40 @@ func (p *prog) discoverNAT64Prefix(generation uint64) { } prefix, ok := nat64PrefixFromAnswer(answer) if !ok { - mainLog.Load().Debug().Msg("dns64: no NAT64 prefix present (not a DNS64 network)") + p.logNAT64Result("") return } if !p.storeDiscoveredNAT64Prefix(generation, prefix) { return } - mainLog.Load().Info().Msgf("dns64: discovered NAT64 prefix %s; enabling AAAA synthesis for IPv6-only network without CLAT", prefix) + p.logNAT64Result(prefix.String()) +} + +// logNAT64Result reports one discovery result. An empty prefix means that the +// network has none. Both results report a change of the network, so both go to +// the journal at info. +func (p *prog) logNAT64Result(prefix string) { + if !p.noteNAT64Result(prefix) { + return + } + if prefix == "" { + journal(mainLog.Load().Info()).Msg("dns64: no NAT64 prefix present (not a DNS64 network)") + return + } + journal(mainLog.Load().Info()).Msgf("dns64: discovered NAT64 prefix %s; enabling AAAA synthesis for IPv6-only network without CLAT", prefix) +} + +// noteNAT64Result reports whether this NAT64 result differs from the one that +// the journal already holds. An empty prefix means no prefix. Discovery runs +// every recheck window, and the journal keeps the changes only. +func (p *prog) noteNAT64Result(prefix string) bool { + p.lastNAT64Mu.Lock() + defer p.lastNAT64Mu.Unlock() + if p.lastNAT64Logged == prefix { + return false + } + p.lastNAT64Logged = prefix + return true } // maybeDNS64 applies DNS64 synthesis to an already-filtered answer when the diff --git a/cmd/cli/dns64_test.go b/cmd/cli/dns64_test.go index 292450e8..3985aeaa 100644 --- a/cmd/cli/dns64_test.go +++ b/cmd/cli/dns64_test.go @@ -364,3 +364,61 @@ func TestMaybeDNS64DropsStaleInFlightPrefix(t *testing.T) { t.Fatal("in-flight synthesis used a prefix invalidated by a network change") } } + +// TestDNS64PrefixSnapshotReadsTheStoredStateOnly covers the read that a report +// of the network uses. The active read reclassifies the network and can start +// a discovery query, and a report must change nothing. +func TestDNS64PrefixSnapshotReadsTheStoredStateOnly(t *testing.T) { + classReads := 0 + old := dns64NetworkClassFn + dns64NetworkClassFn = func() (bool, bool, error) { + classReads++ + return false, false, nil + } + t.Cleanup(func() { dns64NetworkClassFn = old }) + + p := &prog{} + p.dns64.prefix = netip.MustParsePrefix("64:ff9b::/96") + p.dns64.active = true + + prefix, active := p.dns64PrefixSnapshot() + + if !active || prefix != p.dns64.prefix { + t.Fatalf("snapshot = %v, %v, want %v, true", prefix, active, p.dns64.prefix) + } + if classReads != 0 { + t.Fatalf("the snapshot classified the network %d times, want 0", classReads) + } + if p.dns64.discovering { + t.Fatal("the snapshot started a NAT64 prefix discovery") + } +} + +// TestDNS64ResultLinesReachTheJournalAtInfo covers the level of the two NAT64 +// result lines. Both report a change, and a debug line never leaves a host +// that runs at the normal level, so the journal lost the result of a network +// that has no prefix. +func TestDNS64ResultLinesReachTheJournalAtInfo(t *testing.T) { + logs := captureDebugMainLog(t) + p := &prog{} + + // The first result of a run reports nothing when the network has no + // prefix, so the prefix has to arrive before it goes away. + p.logNAT64Result("64:ff9b::/96") + p.logNAT64Result("64:ff9b::/96") + p.logNAT64Result("") + + absent := jsonLogEvents(t, logs, "dns64: no NAT64 prefix present (not a DNS64 network)") + if len(absent) != 1 { + t.Fatalf("got %d no-prefix lines, want 1", len(absent)) + } + wantField(t, absent[0], "level", "info") + wantField(t, absent[0], "journal", true) + + found := jsonLogEvents(t, logs, "dns64: discovered NAT64 prefix 64:ff9b::/96; enabling AAAA synthesis for IPv6-only network without CLAT") + if len(found) != 1 { + t.Fatalf("got %d discovered lines for two identical results, want 1", len(found)) + } + wantField(t, found[0], "level", "info") + wantField(t, found[0], "journal", true) +} diff --git a/cmd/cli/dns_config.go b/cmd/cli/dns_config.go new file mode 100644 index 00000000..22c44ef7 --- /dev/null +++ b/cmd/cli/dns_config.go @@ -0,0 +1,360 @@ +package cli + +import ( + "bufio" + "errors" + "io" + "slices" + "strconv" + "strings" + "sync" + "time" +) + +const ( + // dnsConfigFastInterval is the poll period that follows a network event. + dnsConfigFastInterval = 15 * time.Second + + // dnsConfigSlowInterval is the poll period of a quiet host. + dnsConfigSlowInterval = 5 * time.Minute + + // dnsConfigFastWindow is how long the fast period lasts. A resolver change + // that follows a network event arrives late, after the OS settles. + dnsConfigFastWindow = 10 * time.Minute +) + +// scutilTablePrefix starts a resolver table. macOS prints the table for scoped +// queries after the main one, with the same line. +const scutilTablePrefix = "DNS configuration" + +// scutilResolverPrefix starts one resolver block inside a table. +const scutilResolverPrefix = "resolver #" + +// scutilDNSOutputLimit bounds the text of one read. A resolver table of a host +// is a few kilobytes, so a larger output does not come from a table. +const scutilDNSOutputLimit = 1 << 20 + +// errSCUtilDNSTooLarge ends a read that passed the bound. +var errSCUtilDNSTooLarge = errors.New("scutil --dns output is too large") + +// dnsResolverEntry is one resolver of the DNS configuration of the host. +type dnsResolverEntry struct { + Order int + Nameservers []string + IfIndex int + Interface string + Flags string + SearchDomains []string + Reachable string + // Scoped marks a resolver of the table for scoped queries, which serves one + // interface. A second service keeps its own resolver there while the + // unscoped table stays the same. + Scoped bool + // Action tells what the diff saw: added, changed, or removed. A resolver + // without nameservers looks the same whether it came or went, so the event + // needs the word. + Action string +} + +// The actions of a changed resolver. +const ( + resolverActionAdded = "added" + resolverActionChanged = "changed" + resolverActionRemoved = "removed" +) + +// parseSCUtilDNS reads the resolver tables of "scutil --dns": the unscoped +// table that answers a query of the host, then the table for scoped queries +// with one resolver per interface. Both tables count, because a second +// service can change its resolver while the unscoped table stays the same. +// +// A read that ends early or passes the bound gives an error and no table. A +// part of a table looks like resolvers that went away, and the caller must +// keep the table of the read before instead. +func parseSCUtilDNS(r io.Reader) ([]dnsResolverEntry, error) { + var entries []dnsResolverEntry + scoped := false + var entry *dnsResolverEntry + closeEntry := func() { + if entry == nil { + return + } + entry.Scoped = scoped + entries = append(entries, *entry) + entry = nil + } + read := 0 + scanner := bufio.NewScanner(io.LimitReader(r, scutilDNSOutputLimit+1)) + for scanner.Scan() { + read += len(scanner.Bytes()) + 1 + if read > scutilDNSOutputLimit { + return nil, errSCUtilDNSTooLarge + } + line := strings.TrimSpace(scanner.Text()) + switch { + case strings.HasPrefix(line, scutilTablePrefix): + closeEntry() + scoped = strings.Contains(line, "scoped") + case strings.HasPrefix(line, scutilResolverPrefix): + closeEntry() + entry = &dnsResolverEntry{} + case entry != nil: + applySCUtilField(entry, line) + } + } + if err := scanner.Err(); err != nil { + return nil, err + } + closeEntry() + return entries, nil +} + +func applySCUtilField(entry *dnsResolverEntry, line string) { + name, value, ok := strings.Cut(line, ":") + if !ok { + return + } + name, value = strings.TrimSpace(name), strings.TrimSpace(value) + switch { + case strings.HasPrefix(name, "nameserver["): + entry.Nameservers = append(entry.Nameservers, value) + case strings.HasPrefix(name, "search domain["): + entry.SearchDomains = append(entry.SearchDomains, value) + case name == "if_index": + entry.IfIndex, entry.Interface = parseSCUtilIfIndex(value) + case name == "flags": + entry.Flags = value + case name == "reach": + entry.Reachable = scutilBracketText(value) + case name == "order": + entry.Order, _ = strconv.Atoi(value) + } +} + +// parseSCUtilIfIndex reads an "if_index" value of the form "14 (en0)". +func parseSCUtilIfIndex(value string) (int, string) { + number, name, _ := strings.Cut(value, " ") + index, _ := strconv.Atoi(number) + return index, scutilBracketText(name) +} + +// scutilBracketText returns the text in brackets, where scutil puts the +// readable form of a number. +func scutilBracketText(value string) string { + start := strings.Index(value, "(") + end := strings.LastIndex(value, ")") + if start < 0 || end < start { + return "" + } + return value[start+1 : end] +} + +// resolverKey identifies one resolver across two reads. An unscoped resolver +// has an order, and several can share it, so the count of earlier entries +// with that order completes the key. A scoped resolver has no order line and +// serves one interface, so its interface index is its identity. The two +// tables are two key spaces. +type resolverKey struct { + scoped bool + order int + ifIndex int + occurrence int +} + +func resolverKeys(entries []dnsResolverEntry) []resolverKey { + counts := make(map[resolverKey]int, len(entries)) + keys := make([]resolverKey, len(entries)) + for i, entry := range entries { + base := resolverKey{scoped: entry.Scoped, order: entry.Order} + if entry.Scoped { + base = resolverKey{scoped: true, ifIndex: entry.IfIndex} + } + key := base + key.occurrence = counts[base] + keys[i] = key + counts[base]++ + } + return keys +} + +func equalResolverEntry(a, b dnsResolverEntry) bool { + return a.IfIndex == b.IfIndex && + a.Interface == b.Interface && + a.Flags == b.Flags && + a.Reachable == b.Reachable && + slices.Equal(a.Nameservers, b.Nameservers) && + slices.Equal(a.SearchDomains, b.SearchDomains) +} + +// diffResolverTables returns the resolvers that differ between two reads. A +// resolver that vanished comes back without nameservers, so the journal shows +// that the host dropped it. +func diffResolverTables(before, after []dnsResolverEntry) []dnsResolverEntry { + beforeKeys, afterKeys := resolverKeys(before), resolverKeys(after) + previous := make(map[resolverKey]dnsResolverEntry, len(before)) + for i, key := range beforeKeys { + previous[key] = before[i] + } + current := make(map[resolverKey]struct{}, len(after)) + for _, key := range afterKeys { + current[key] = struct{}{} + } + var changed []dnsResolverEntry + for i, key := range afterKeys { + known, ok := previous[key] + if ok && equalResolverEntry(known, after[i]) { + continue + } + entry := after[i] + entry.Action = resolverActionChanged + if !ok { + entry.Action = resolverActionAdded + } + changed = append(changed, entry) + } + for i, key := range beforeKeys { + if _, ok := current[key]; ok { + continue + } + vanished := before[i] + vanished.Nameservers = nil + vanished.Action = resolverActionRemoved + changed = append(changed, vanished) + } + return changed +} + +// dnsConfigPoller reads the DNS configuration of the host on a timer and +// reports the resolvers that changed since the read before. +type dnsConfigPoller struct { + run func() ([]dnsResolverEntry, error) + now func() time.Time + mu sync.Mutex + last []dnsResolverEntry + lastActivity time.Time + started bool + readFailed bool + // kick wakes a loop that waits on the slow delay. One pending signal is + // enough, so a storm of events does not queue polls. + kick chan struct{} +} + +func newDNSConfigPoller(run func() ([]dnsResolverEntry, error)) *dnsConfigPoller { + return &dnsConfigPoller{run: run, now: time.Now, kick: make(chan struct{}, 1)} +} + +// noteActivity keeps the poll fast for a while after the network moved, and +// wakes a loop that waits on the slow delay. +func (p *dnsConfigPoller) noteActivity(now time.Time) { + p.markActivity(now) + select { + case p.kick <- struct{}{}: + default: + } +} + +// markActivity records the time of the last network event. +func (p *dnsConfigPoller) markActivity(now time.Time) { + p.mu.Lock() + defer p.mu.Unlock() + p.lastActivity = now +} + +func (p *dnsConfigPoller) nextDelay(now time.Time) time.Duration { + p.mu.Lock() + defer p.mu.Unlock() + if p.lastActivity.IsZero() || now.Sub(p.lastActivity) >= dnsConfigFastWindow { + return dnsConfigSlowInterval + } + return dnsConfigFastInterval +} + +// pollOnce returns the resolvers that changed. The first table is the baseline +// of the process and reports no change. A failed read keeps the table of the +// read before, so a short failure does not report the whole table twice. The +// first failure and the read that ends it log once, so a host whose read +// never works says so. +func (p *dnsConfigPoller) pollOnce(now time.Time) []dnsResolverEntry { + table, err := p.run() + p.mu.Lock() + defer p.mu.Unlock() + if err != nil { + if !p.readFailed { + mainLog.Load().Warn().Err(err).Msg("DNS configuration read failed; the journal keeps the last table") + } + p.readFailed = true + return nil + } + if p.readFailed { + mainLog.Load().Info().Msg("DNS configuration read works again") + p.readFailed = false + } + if !p.started { + p.started = true + p.last = table + return nil + } + changed := diffResolverTables(p.last, table) + p.last = table + return changed +} + +// dnsConfigDelayFn lets a test shorten the wait between two polls. +var dnsConfigDelayFn = (*dnsConfigPoller).nextDelay + +// startLoop runs the poll loop and returns a channel that closes when the loop +// returns. A caller joins that channel before it reads the log. +func (p *dnsConfigPoller) startLoop(stop <-chan struct{}, emit func([]dnsResolverEntry)) <-chan struct{} { + done := make(chan struct{}) + go func() { + defer close(done) + p.loop(stop, emit) + }() + return done +} + +// loop polls until stop closes. The read starts a subprocess on macOS, so the +// delay follows the last activity and a quiet host pays little. The start of a +// run counts as activity, because the table of the host settles after the +// daemon comes up, and the read at entry is the baseline of the process. +func (p *dnsConfigPoller) loop(stop <-chan struct{}, emit func([]dnsResolverEntry)) { + now := p.now() + p.markActivity(now) + p.pollOnce(now) + delay := dnsConfigDelayFn(p, now) + deadline := now.Add(delay) + timer := time.NewTimer(delay) + defer timer.Stop() + for { + select { + case <-stop: + return + case <-p.kick: + // noteActivity set the time just now, so the delay is the fast one. + // A kick only moves a deadline closer; a storm of kicks must not + // push the read back again and again. + now := p.now() + wanted := now.Add(dnsConfigDelayFn(p, now)) + if !wanted.Before(deadline) { + continue + } + if !timer.Stop() { + select { + case <-timer.C: + default: + } + } + timer.Reset(wanted.Sub(now)) + deadline = wanted + continue + case <-timer.C: + } + now := p.now() + if changed := p.pollOnce(now); len(changed) > 0 { + emit(changed) + } + delay := dnsConfigDelayFn(p, now) + deadline = now.Add(delay) + timer.Reset(delay) + } +} diff --git a/cmd/cli/dns_config_darwin.go b/cmd/cli/dns_config_darwin.go new file mode 100644 index 00000000..60d0c78e --- /dev/null +++ b/cmd/cli/dns_config_darwin.go @@ -0,0 +1,22 @@ +package cli + +import ( + "bytes" + "context" + "os/exec" + "time" +) + +// scutilDNSTimeout bounds the read. scutil talks to the configuration daemon, +// and that daemon stalls while the network moves. +const scutilDNSTimeout = 5 * time.Second + +func runSCUtilDNS() ([]dnsResolverEntry, error) { + ctx, cancel := context.WithTimeout(context.Background(), scutilDNSTimeout) + defer cancel() + out, err := exec.CommandContext(ctx, "scutil", "--dns").Output() + if err != nil { + return nil, err + } + return parseSCUtilDNS(bytes.NewReader(out)) +} diff --git a/cmd/cli/dns_config_others.go b/cmd/cli/dns_config_others.go new file mode 100644 index 00000000..9905dd51 --- /dev/null +++ b/cmd/cli/dns_config_others.go @@ -0,0 +1,9 @@ +//go:build !darwin + +package cli + +// runSCUtilDNS has no source of a resolver table outside macOS, so the poller +// finds no change and logs nothing. +func runSCUtilDNS() ([]dnsResolverEntry, error) { + return nil, nil +} diff --git a/cmd/cli/dns_config_test.go b/cmd/cli/dns_config_test.go new file mode 100644 index 00000000..b389ad0f --- /dev/null +++ b/cmd/cli/dns_config_test.go @@ -0,0 +1,598 @@ +package cli + +import ( + "errors" + "fmt" + "reflect" + "strings" + "sync" + "sync/atomic" + "testing" + "time" +) + +const scutilDNSFixture = `DNS configuration + +resolver #1 + search domain[0] : corp.example + nameserver[0] : 192.168.50.200 + nameserver[1] : 192.168.50.1 + if_index : 16 (en0) + flags : Request A records, Request AAAA records + reach : 0x00020002 (Reachable,Directly Reachable Address) + order : 200000 + +resolver #2 + nameserver[0] : 100.100.100.100 + if_index : 28 (utun4) + flags : Supplemental, Request A records, Request AAAA records + reach : 0x00000003 (Reachable,Transient Connection) + order : 101600 + +resolver #3 + domain : local + options : mdns + timeout : 5 + flags : Request A records + reach : 0x00000000 (Not Reachable) + order : 300000 + +DNS configuration (for scoped queries) + +resolver #1 + nameserver[0] : 192.168.50.200 + if_index : 16 (en0) + flags : Scoped, Request A records + reach : 0x00020002 (Reachable,Directly Reachable Address) +` + +const scutilDNSScopedOnlyFixture = `DNS configuration + +DNS configuration (for scoped queries) + +resolver #1 + nameserver[0] : 192.168.50.200 + if_index : 16 (en0) + flags : Scoped, Request A records + reach : 0x00020002 (Reachable,Directly Reachable Address) +` + +var errStubResolverRun = errors.New("scutil failed") + +func wifiResolverEntry() dnsResolverEntry { + return dnsResolverEntry{ + Order: 200000, + Nameservers: []string{"192.168.50.200", "192.168.50.1"}, + IfIndex: 16, + Interface: "en0", + Flags: "Request A records, Request AAAA records", + SearchDomains: []string{"corp.example"}, + Reachable: "Reachable,Directly Reachable Address", + } +} + +func vpnResolverEntry() dnsResolverEntry { + return dnsResolverEntry{ + Order: 101600, + Nameservers: []string{"100.100.100.100"}, + IfIndex: 28, + Interface: "utun4", + Flags: "Supplemental, Request A records, Request AAAA records", + Reachable: "Reachable,Transient Connection", + } +} + +func mdnsResolverEntry() dnsResolverEntry { + return dnsResolverEntry{ + Order: 300000, + Flags: "Request A records", + Reachable: "Not Reachable", + } +} + +func Test_parseSCUtilDNS(t *testing.T) { + tests := []struct { + name string + input string + want []dnsResolverEntry + }{ + { + name: "both tables", + input: scutilDNSFixture, + want: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry(), mdnsResolverEntry(), { + Nameservers: []string{"192.168.50.200"}, + IfIndex: 16, + Interface: "en0", + Flags: "Scoped, Request A records", + Reachable: "Reachable,Directly Reachable Address", + Scoped: true, + }}, + }, + { + name: "scoped table when the first is empty", + input: scutilDNSScopedOnlyFixture, + want: []dnsResolverEntry{{ + Nameservers: []string{"192.168.50.200"}, + IfIndex: 16, + Interface: "en0", + Flags: "Scoped, Request A records", + Reachable: "Reachable,Directly Reachable Address", + Scoped: true, + }}, + }, + { + name: "no output", + input: "", + want: nil, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, err := parseSCUtilDNS(strings.NewReader(tc.input)) + if err != nil { + t.Fatalf("parseSCUtilDNS() error = %v", err) + } + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("parseSCUtilDNS() = %+v, want %+v", got, tc.want) + } + }) + } +} + +func Test_diffResolverTables(t *testing.T) { + changedWiFi := wifiResolverEntry() + changedWiFi.Nameservers = []string{"10.0.0.1"} + changedWiFi.Action = resolverActionChanged + vanishedVPN := vpnResolverEntry() + vanishedVPN.Nameservers = nil + vanishedVPN.Action = resolverActionRemoved + appearedVPN := vpnResolverEntry() + appearedVPN.Action = resolverActionAdded + + tests := []struct { + name string + before []dnsResolverEntry + after []dnsResolverEntry + want []dnsResolverEntry + }{ + { + name: "no change", + before: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + after: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + want: nil, + }, + { + name: "one resolver changed", + before: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + after: []dnsResolverEntry{changedWiFi, vpnResolverEntry()}, + want: []dnsResolverEntry{changedWiFi}, + }, + { + name: "one resolver vanished", + before: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + after: []dnsResolverEntry{wifiResolverEntry()}, + want: []dnsResolverEntry{vanishedVPN}, + }, + { + name: "one resolver appeared", + before: []dnsResolverEntry{wifiResolverEntry()}, + after: []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()}, + want: []dnsResolverEntry{appearedVPN}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := diffResolverTables(tc.before, tc.after) + if !reflect.DeepEqual(got, tc.want) { + t.Fatalf("diffResolverTables() = %+v, want %+v", got, tc.want) + } + }) + } +} + +func Test_diffResolverTablesKeepsTheInput(t *testing.T) { + before := []dnsResolverEntry{vpnResolverEntry()} + diffResolverTables(before, nil) + if !reflect.DeepEqual(before[0], vpnResolverEntry()) { + t.Fatalf("before entry = %+v, want %+v", before[0], vpnResolverEntry()) + } +} + +// resolverRunResult is one answer of the stub runner. +type resolverRunResult struct { + table []dnsResolverEntry + err error +} + +// stubResolverRunner answers one result per call and repeats the last one. +type stubResolverRunner struct { + mu sync.Mutex + results []resolverRunResult + calls int +} + +func (s *stubResolverRunner) run() ([]dnsResolverEntry, error) { + s.mu.Lock() + defer s.mu.Unlock() + result := s.results[min(s.calls, len(s.results)-1)] + s.calls++ + return result.table, result.err +} + +func Test_dnsConfigPollerNextDelay(t *testing.T) { + base := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + tests := []struct { + name string + activity bool + elapsed time.Duration + want time.Duration + }{ + {name: "nine minutes after activity", activity: true, elapsed: 9 * time.Minute, want: dnsConfigFastInterval}, + {name: "eleven minutes after activity", activity: true, elapsed: 11 * time.Minute, want: dnsConfigSlowInterval}, + {name: "no activity yet", elapsed: time.Minute, want: dnsConfigSlowInterval}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + poller := newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return nil, nil }) + if tc.activity { + poller.noteActivity(base) + } + if got := poller.nextDelay(base.Add(tc.elapsed)); got != tc.want { + t.Fatalf("nextDelay() = %s, want %s", got, tc.want) + } + }) + } +} + +func Test_dnsConfigPollerPollOnce(t *testing.T) { + now := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + baseline := []dnsResolverEntry{wifiResolverEntry(), vpnResolverEntry()} + changedWiFi := wifiResolverEntry() + changedWiFi.Nameservers = []string{"10.0.0.1"} + changedWiFi.Action = resolverActionChanged + runner := &stubResolverRunner{results: []resolverRunResult{ + {table: baseline}, + {err: errStubResolverRun}, + {table: baseline}, + {table: []dnsResolverEntry{changedWiFi, vpnResolverEntry()}}, + }} + poller := newDNSConfigPoller(runner.run) + + if got := poller.pollOnce(now); got != nil { + t.Fatalf("baseline poll = %+v, want nil", got) + } + if got := poller.pollOnce(now.Add(time.Minute)); got != nil { + t.Fatalf("failed poll = %+v, want nil", got) + } + if got := poller.pollOnce(now.Add(2 * time.Minute)); got != nil { + t.Fatalf("poll after a failure = %+v, want nil", got) + } + want := []dnsResolverEntry{changedWiFi} + if got := poller.pollOnce(now.Add(3 * time.Minute)); !reflect.DeepEqual(got, want) { + t.Fatalf("changed poll = %+v, want %+v", got, want) + } +} + +func Test_dnsConfigPollerLoop(t *testing.T) { + previous := dnsConfigDelayFn + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { return time.Millisecond } + t.Cleanup(func() { dnsConfigDelayFn = previous }) + + changedWiFi := wifiResolverEntry() + changedWiFi.Nameservers = []string{"10.0.0.1"} + changedWiFi.Action = resolverActionChanged + runner := &stubResolverRunner{results: []resolverRunResult{ + {table: []dnsResolverEntry{wifiResolverEntry()}}, + {table: []dnsResolverEntry{changedWiFi}}, + }} + poller := newDNSConfigPoller(runner.run) + + emitted := make(chan []dnsResolverEntry, 4) + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + poller.loop(stop, func(changed []dnsResolverEntry) { emitted <- changed }) + }() + + select { + case got := <-emitted: + want := []dnsResolverEntry{changedWiFi} + if !reflect.DeepEqual(got, want) { + t.Fatalf("emitted = %+v, want %+v", got, want) + } + case <-time.After(5 * time.Second): + t.Fatal("the loop emitted no change") + } + + close(stop) + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("the loop did not stop") + } +} + +// Test_dnsConfigPollerLoopWakesOnActivity proves that a network event does not +// wait for a sleeping timer. The delay is one hour until the event arrives, so +// a loop without the kick would poll after the test deadline. +func Test_dnsConfigPollerLoopWakesOnActivity(t *testing.T) { + var kicked atomic.Bool + parked := make(chan struct{}, 1) + previous := dnsConfigDelayFn + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { + if kicked.Load() { + return time.Millisecond + } + select { + case parked <- struct{}{}: + default: + } + return time.Hour + } + t.Cleanup(func() { dnsConfigDelayFn = previous }) + + changedWiFi := wifiResolverEntry() + changedWiFi.Nameservers = []string{"10.0.0.1"} + changedWiFi.Action = resolverActionChanged + runner := &stubResolverRunner{results: []resolverRunResult{ + {table: []dnsResolverEntry{wifiResolverEntry()}}, + {table: []dnsResolverEntry{changedWiFi}}, + }} + poller := newDNSConfigPoller(runner.run) + + emitted := make(chan []dnsResolverEntry, 4) + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + poller.loop(stop, func(changed []dnsResolverEntry) { emitted <- changed }) + }() + t.Cleanup(func() { + close(stop) + <-done + }) + + // The kick matters only once the loop waits on the long delay. + select { + case <-parked: + case <-time.After(5 * time.Second): + t.Fatal("the loop did not park on the long delay") + } + kicked.Store(true) + poller.noteActivity(time.Now()) + + select { + case got := <-emitted: + want := []dnsResolverEntry{changedWiFi} + if !reflect.DeepEqual(got, want) { + t.Fatalf("emitted = %+v, want %+v", got, want) + } + case <-time.After(5 * time.Second): + t.Fatal("the loop polled no table after the activity") + } +} + +// Test_dnsConfigPollerLoopTakesABaseline proves that the loop reads the table +// once at entry. Without that read the first change after the start reports +// the whole table. +func Test_dnsConfigPollerLoopTakesABaseline(t *testing.T) { + previous := dnsConfigDelayFn + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { return time.Hour } + t.Cleanup(func() { dnsConfigDelayFn = previous }) + + runner := &stubResolverRunner{results: []resolverRunResult{{table: []dnsResolverEntry{wifiResolverEntry()}}}} + poller := newDNSConfigPoller(runner.run) + + stop := make(chan struct{}) + done := make(chan struct{}) + go func() { + defer close(done) + poller.loop(stop, func([]dnsResolverEntry) { t.Error("the baseline read reported a change") }) + }() + + deadline := time.Now().Add(5 * time.Second) + for time.Now().Before(deadline) { + runner.mu.Lock() + calls := runner.calls + runner.mu.Unlock() + if calls > 0 { + break + } + time.Sleep(time.Millisecond) + } + close(stop) + <-done + + runner.mu.Lock() + defer runner.mu.Unlock() + if runner.calls != 1 { + t.Fatalf("reads at the loop entry: got %d, want 1", runner.calls) + } + if !poller.started { + t.Fatal("the loop took no baseline") + } +} + +// oversizedSCUtilDNS is a table with one line that no scanner buffer holds. A +// stalled or hostile process produces output of this shape. +func oversizedSCUtilDNS() string { + return "DNS configuration\n\nresolver #1\n nameserver[0] : " + strings.Repeat("9", 70*1024) + "\n" +} + +// Test_parseSCUtilDNSRejectsAnOversizedLine proves that a line the scanner +// cannot hold ends the read with an error. A part of a table must not pass for +// the table of the host. +func Test_parseSCUtilDNSRejectsAnOversizedLine(t *testing.T) { + got, err := parseSCUtilDNS(strings.NewReader(oversizedSCUtilDNS())) + if err == nil { + t.Fatal("parseSCUtilDNS accepted a 70 KiB line") + } + if got != nil { + t.Fatalf("parseSCUtilDNS() = %+v, want nil", got) + } +} + +// Test_parseSCUtilDNSRejectsAnOversizedOutput proves that output past the +// bound ends the read with an error instead of a table cut in the middle. +func Test_parseSCUtilDNSRejectsAnOversizedOutput(t *testing.T) { + block := "resolver #1\n nameserver[0] : 192.168.50.200\n order : 200000\n" + repeats := scutilDNSOutputLimit/len(block) + 2 + got, err := parseSCUtilDNS(strings.NewReader("DNS configuration\n\n" + strings.Repeat(block, repeats))) + if err == nil { + t.Fatal("parseSCUtilDNS accepted output past the bound") + } + if got != nil { + t.Fatalf("parseSCUtilDNS() = %+v, want nil", got) + } +} + +// Test_dnsConfigPollerKeepsTheTableOnAnOversizedRead proves that a read the +// parser rejects reports nothing and keeps the table of the read before. +func Test_dnsConfigPollerKeepsTheTableOnAnOversizedRead(t *testing.T) { + now := time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC) + outputs := []string{scutilDNSFixture, oversizedSCUtilDNS(), scutilDNSFixture} + calls := 0 + poller := newDNSConfigPoller(func() ([]dnsResolverEntry, error) { + output := outputs[min(calls, len(outputs)-1)] + calls++ + return parseSCUtilDNS(strings.NewReader(output)) + }) + + if got := poller.pollOnce(now); got != nil { + t.Fatalf("baseline poll = %+v, want nil", got) + } + if got := poller.pollOnce(now.Add(time.Minute)); got != nil { + t.Fatalf("oversized poll = %+v, want nil", got) + } + if got := poller.pollOnce(now.Add(2 * time.Minute)); got != nil { + t.Fatalf("poll after the oversized read = %+v, want nil", got) + } +} + +// scutilTwoTables renders a host with one unscoped resolver and one scoped +// resolver, the shape of a Mac with two active services. +func scutilTwoTables(unscoped, scoped string) string { + return "DNS configuration\n\nresolver #1\n nameserver[0] : " + unscoped + "\n if_index : 14 (en0)\n" + + " flags : Request A records\n reach : 0x00020002 (Reachable,Directly Reachable Address)\n\n" + + "DNS configuration (for scoped queries)\n\nresolver #1\n nameserver[0] : " + scoped + "\n if_index : 15 (en1)\n" + + " flags : Scoped, Request A records\n reach : 0x00020002 (Reachable,Directly Reachable Address)\n" +} + +// Test_diffResolverTablesSeesAScopedChange changes the scoped resolver of a +// second service while the unscoped resolver stays. The diff must report the +// scoped change, because the two tables are not copies of each other. +func Test_diffResolverTablesSeesAScopedChange(t *testing.T) { + before, err := parseSCUtilDNS(strings.NewReader(scutilTwoTables("192.168.1.1", "10.0.0.1"))) + if err != nil { + t.Fatal(err) + } + after, err := parseSCUtilDNS(strings.NewReader(scutilTwoTables("192.168.1.1", "9.9.9.9"))) + if err != nil { + t.Fatal(err) + } + changed := diffResolverTables(before, after) + if len(changed) != 1 || len(changed[0].Nameservers) != 1 || changed[0].Nameservers[0] != "9.9.9.9" { + t.Fatalf("changed resolvers = %+v, want the scoped resolver with 9.9.9.9", changed) + } + if !changed[0].Scoped || before[0].Scoped || len(before) != 2 { + t.Fatalf("scope marks: changed %v, before %+v; want the scoped resolver marked and the unscoped one not", changed[0].Scoped, before) + } +} + +// Test_dnsConfigPollerPollsUnderContinuousActivity kicks the poller faster +// than its fast interval. A kick must not push the read back again and again, +// or a host in a storm of transitions never reports a resolver change. +func Test_dnsConfigPollerPollsUnderContinuousActivity(t *testing.T) { + var reads atomic.Int64 + changed := wifiResolverEntry() + changed.Nameservers = []string{"9.9.9.9"} + p := newDNSConfigPoller(func() ([]dnsResolverEntry, error) { + if reads.Add(1) == 1 { + return []dnsResolverEntry{wifiResolverEntry()}, nil + } + return []dnsResolverEntry{changed}, nil + }) + origDelay := dnsConfigDelayFn + t.Cleanup(func() { dnsConfigDelayFn = origDelay }) + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { return 50 * time.Millisecond } + + var emits atomic.Int64 + stop := make(chan struct{}) + done := p.startLoop(stop, func([]dnsResolverEntry) { emits.Add(1) }) + deadline := time.Now().Add(500 * time.Millisecond) + for time.Now().Before(deadline) { + p.noteActivity(time.Now()) + time.Sleep(5 * time.Millisecond) + } + close(stop) + <-done + + if emits.Load() == 0 { + t.Fatalf("no resolver event during 500 ms of activity (reads %d)", reads.Load()) + } +} + +// scutilScopedTable renders the scoped table the way macOS prints it: one +// resolver per interface and no order line. +func scutilScopedTable(resolvers ...[2]string) string { + var b strings.Builder + b.WriteString("DNS configuration\n\nresolver #1\n nameserver[0] : 100.100.100.100\n if_index : 28 (utun4)\n flags : Request A records, Request AAAA records\n reach : 0x00000003 (Reachable,Transient Connection)\n order : 200000\n\n") + b.WriteString("DNS configuration (for scoped queries)\n\n") + for i, resolver := range resolvers { + fmt.Fprintf(&b, "resolver #%d\n nameserver[0] : %s\n if_index : %s\n flags : Scoped, Request A records\n reach : 0x00020002 (Reachable,Directly Reachable Address)\n\n", i+1, resolver[1], resolver[0]) + } + return b.String() +} + +// Test_diffResolverTablesKeysScopedResolversByInterface inserts a scoped +// resolver between two others and removes it again. The scoped table has no +// order line, so the interface is the identity, and the diff must name the +// resolver that came and went, not its neighbors. +func Test_diffResolverTablesKeysScopedResolversByInterface(t *testing.T) { + en0 := [2]string{"16 (en0)", "192.168.50.200"} + en7 := [2]string{"30 (en7)", "10.9.9.1"} + utun := [2]string{"28 (utun4)", "100.100.100.100"} + before, err := parseSCUtilDNS(strings.NewReader(scutilScopedTable(en0, utun))) + if err != nil { + t.Fatal(err) + } + after, err := parseSCUtilDNS(strings.NewReader(scutilScopedTable(en0, en7, utun))) + if err != nil { + t.Fatal(err) + } + + inserted := diffResolverTables(before, after) + if len(inserted) != 1 || inserted[0].Interface != "en7" || inserted[0].Nameservers == nil { + t.Fatalf("insert of en7 reported %+v, want en7 alone with its nameserver", inserted) + } + removed := diffResolverTables(after, before) + if len(removed) != 1 || removed[0].Interface != "en7" || removed[0].Nameservers != nil { + t.Fatalf("removal of en7 reported %+v, want en7 alone without nameservers", removed) + } +} + +// Test_diffResolverTablesNamesTheAction tells a resolver that came from one +// that went. An mDNS resolver has no nameservers, so without the action the +// two events read the same. +func Test_diffResolverTablesNamesTheAction(t *testing.T) { + mdns := "DNS configuration\n\nresolver #1\n domain : local\n options : mdns\n timeout : 5\n flags : Request A records, Request AAAA records\n reach : 0x00000000 (Not Reachable)\n order : 300000\n" + table, err := parseSCUtilDNS(strings.NewReader(mdns)) + if err != nil { + t.Fatal(err) + } + appeared := diffResolverTables(nil, table) + vanished := diffResolverTables(table, nil) + if len(appeared) != 1 || appeared[0].Action != resolverActionAdded { + t.Fatalf("appeared = %+v, want one resolver with action added", appeared) + } + if len(vanished) != 1 || vanished[0].Action != resolverActionRemoved { + t.Fatalf("vanished = %+v, want one resolver with action removed", vanished) + } + moved := wifiResolverEntry() + moved.Nameservers = []string{"9.9.9.9"} + changed := diffResolverTables([]dnsResolverEntry{wifiResolverEntry()}, []dnsResolverEntry{moved}) + if len(changed) != 1 || changed[0].Action != resolverActionChanged { + t.Fatalf("changed = %+v, want one resolver with action changed", changed) + } +} diff --git a/cmd/cli/dns_intercept_darwin.go b/cmd/cli/dns_intercept_darwin.go index ab724921..7336b84d 100644 --- a/cmd/cli/dns_intercept_darwin.go +++ b/cmd/cli/dns_intercept_darwin.go @@ -17,6 +17,8 @@ import ( "syscall" "time" + "github.com/rs/zerolog" + "github.com/Control-D-Inc/ctrld" ) @@ -221,8 +223,9 @@ func (p *prog) startDNSIntercept() error { } // Store the initial set of tunnel interfaces so we can detect changes later. + tunnels := p.activeTunnelInterfaces() p.mu.Lock() - p.lastTunnelIfaces = discoverTunnelInterfacesForReconcile() + p.lastTunnelIfaces = tunnels p.pendingTunnelIfaces = nil p.hasPendingTunnelIfaces = false p.mu.Unlock() @@ -723,14 +726,29 @@ func discoverTunnelInterfaces() []string { } } - if len(tunnels) > 0 { - mainLog.Load().Debug().Msgf("DNS intercept: discovered active tunnel interfaces: %v", tunnels) - } return tunnels } var discoverTunnelInterfacesForReconcile = discoverTunnelInterfaces +// tunnelInterfacesKey holds the last discovered tunnel set, so the discovery +// line reports a change and not every reconcile. +const tunnelInterfacesKey = "tunnel_interfaces" + +// activeTunnelInterfaces returns the live tunnel interfaces and names them on +// change. Every pf reconcile discovers them again, and the set is the same in +// almost every one of those reads. +func (p *prog) activeTunnelInterfaces() []string { + tunnels := discoverTunnelInterfacesForReconcile() + changed, repeats := p.repeats.changed(tunnelInterfacesKey, fmt.Sprintf("%v", tunnels)) + if !changed || len(tunnels) == 0 { + return tunnels + } + mainLog.Load().Debug().Uint64("repeats", repeats). + Msgf("DNS intercept: discovered active tunnel interfaces: %v", tunnels) + return tunnels +} + // dnsInterceptSupported reports whether DNS intercept mode is supported on this platform. func dnsInterceptSupported() bool { _, err := exec.LookPath("pfctl") @@ -787,7 +805,7 @@ func (p *prog) validateDNSIntercept() error { // // pf requires strict rule ordering: translation (rdr) BEFORE filtering (pass). func (p *prog) buildPFAnchorRules(vpnExemptions []vpnDNSExemption) string { - return p.buildPFAnchorRulesForTunnels(vpnExemptions, discoverTunnelInterfacesForReconcile()) + return p.buildPFAnchorRulesForTunnels(vpnExemptions, p.activeTunnelInterfaces()) } func (p *prog) buildPFAnchorRulesForTunnels(vpnExemptions []vpnDNSExemption, tunnelIfaces []string) string { @@ -1169,11 +1187,12 @@ var ( restorePFAnchorForReconcile = func(p *prog, reason string) pfAnchorCheckResult { return p.restorePFAnchor(reason) } + verifyPFStateFn = (*prog).verifyPFState pfShutdownStateRevokedForTest = func() {} ) func (p *prog) rebuildPFAnchorRules(vpnExemptions []vpnDNSExemption) ([]string, error) { - tunnelIfaces := append([]string(nil), discoverTunnelInterfacesForReconcile()...) + tunnelIfaces := append([]string(nil), p.activeTunnelInterfaces()...) rulesStr := p.buildPFAnchorRulesForTunnels(vpnExemptions, tunnelIfaces) if err := os.WriteFile(pfAnchorFile, []byte(rulesStr), 0644); err != nil { return nil, fmt.Errorf("write anchor file: %w", err) @@ -1224,7 +1243,7 @@ func (p *prog) restorePFAnchorWithTransportReset(reason string, resetTransports if resetTransports { p.resetUpstreamTransports() } - if !p.verifyPFState() { + if !verifyPFStateFn(p) { mainLog.Load().Error().Str("reason", reason).Msg("DNS intercept: rebuilt anchor failed post-load verification") return pfAnchorCheckFailed } @@ -1234,6 +1253,7 @@ func (p *prog) restorePFAnchorWithTransportReset(reason string, resetTransports p.commitPFReconcileState(tunnelIfaces) p.pfLastRestoreTime.Store(time.Now().UnixMilli()) mainLog.Load().Info().Str("reason", reason).Msg("DNS intercept: pf anchor restored successfully") + p.logPFAnchorList(pfAnchorReasonRestored, mainLog.Load().Info(), pfRuleDump{}) return pfAnchorCheckRestored } @@ -1284,7 +1304,8 @@ func (p *prog) checkTunnelInterfaceChanges() bool { return false } - current := append([]string(nil), discoverTunnelInterfacesForReconcile()...) + current := append([]string(nil), p.activeTunnelInterfaces()...) + p.noteTunnelInterfaceSet(current) p.mu.Lock() prev := append([]string(nil), p.lastTunnelIfaces...) @@ -1319,18 +1340,9 @@ func (p *prog) checkTunnelInterfaceChanges() bool { return true } - // Detect NEW tunnel interfaces (not just any change). - prevSet := make(map[string]bool, len(prev)) - for _, iface := range prev { - prevSet[iface] = true - } - hasNewTunnel := false - for _, iface := range current { - if !prevSet[iface] { - hasNewTunnel = true - mainLog.Load().Info().Msgf("DNS intercept: new tunnel interface detected: %s", iface) - break - } + added, _ := tunnelSetDiff(prev, current) + if len(added) > 0 { + mainLog.Load().Info().Msgf("DNS intercept: new tunnel interface detected: %s", added[0]) } if stabilizing { @@ -1338,14 +1350,13 @@ func (p *prog) checkTunnelInterfaceChanges() bool { return newObservation } - if hasNewTunnel { + if len(added) > 0 { // A new VPN tunnel appeared. The dedicated post-stabilization repair path // rebuilds the anchor if no successful exemption update applies it first. p.pfStartStabilization() return newObservation } - mainLog.Load().Info().Msgf("DNS intercept: tunnel interfaces changed (was %v, now %v) — rebuilding pf anchor rules", prev, current) result := p.reconcilePFAnchorForTunnelChange() if result != pfAnchorCheckRestored { // Keep the desired tunnel set pending. The next event, delayed check, or @@ -1355,6 +1366,70 @@ func (p *prog) checkTunnelInterfaceChanges() bool { return newObservation || result == pfAnchorCheckRestored } +// tunnelChangedMessage names the tunnel event of the journal. +const tunnelChangedMessage = "Tunnel interface changed" + +// noteTunnelInterfaceSet logs one event for each change of the tunnel set. The +// reconcile retries a set until it succeeds, and a set that returns to the one +// before it is news as well, so the diff runs against the set of the last +// event and not against the applied set. +func (p *prog) noteTunnelInterfaceSet(current []string) { + p.mu.Lock() + // Before the first event the applied set is the baseline, because no event + // named a set yet. + previous := p.lastLoggedTunnelIfaces + if previous == nil { + previous = p.lastTunnelIfaces + } + previous = append([]string(nil), previous...) + if stringSlicesEqual(previous, current) { + p.mu.Unlock() + return + } + p.lastLoggedTunnelIfaces = append([]string{}, current...) + p.mu.Unlock() + + added, removed := tunnelSetDiff(previous, current) + p.logTunnelInterfaceChange(added, removed) +} + +// logTunnelInterfaceChange names the tunnels that appeared and the ones that +// went away. The owner names the program behind a new tunnel, because its pf +// rules compete with the ctrld anchor for the same DNS packets. +func (p *prog) logTunnelInterfaceChange(added, removed []string) { + owner := "" + if len(added) > 0 { + owner = tunnelOwner(added[0]) + } + journal(mainLog.Load().Info()). + Strs("added", added). + Strs("removed", removed). + Str("owner", owner). + Msg(tunnelChangedMessage) +} + +// tunnelSetDiff reports the tunnels that appeared and the ones that went away, +// each in the order of the set it comes from. +func tunnelSetDiff(prev, current []string) (added, removed []string) { + before := make(map[string]bool, len(prev)) + for _, iface := range prev { + before[iface] = true + } + live := make(map[string]bool, len(current)) + for _, iface := range current { + live[iface] = true + if !before[iface] { + added = append(added, iface) + } + } + for _, iface := range prev { + if !live[iface] { + removed = append(removed, iface) + } + } + return added, removed +} + // stringSlicesEqual reports whether two string slices have the same elements in the same order. func stringSlicesEqual(a, b []string) bool { if len(a) != len(b) { @@ -1383,7 +1458,8 @@ func (p *prog) pfStartStabilization() { stableRequired = 45 * time.Second } - mainLog.Load().Info().Msgf("DNS intercept: VPN connecting — entering stabilization mode (waiting %s for pf to settle)", stableRequired) + journal(mainLog.Load().Info()).Msgf("DNS intercept: VPN connecting — entering stabilization mode (waiting %s for pf to settle)", stableRequired) + p.logPFAnchorList(pfAnchorReasonStabilizationStart, mainLog.Load().Info(), pfRuleDump{}) ctx, cancel := context.WithCancel(context.Background()) p.mu.Lock() @@ -1463,7 +1539,8 @@ func (p *prog) pfStabilizationLoopWithMaxWait(ctx context.Context, stableRequire // held still for the required window. The active loop retains ownership throughout: it // never re-enters stabilization recursively. func (p *prog) finishPFStabilization(stableRequired time.Duration) { - mainLog.Load().Info().Msgf("DNS intercept: pf stable for %s — reconciling anchor rules", stableRequired) + journal(mainLog.Load().Info()).Msgf("DNS intercept: pf stable for %s — reconciling anchor rules", stableRequired) + p.logPFAnchorList(pfAnchorReasonStabilizationEnd, mainLog.Load().Info(), pfRuleDump{}) result := p.reconcilePFAnchorAfterStabilization() if result != pfAnchorCheckRestored && result != pfAnchorCheckIntact { p.scheduleDelayedRechecks() @@ -1630,6 +1707,11 @@ func (p *prog) verifyInterceptAfterWake(gap time.Duration) { return } mainLog.Load().Info().Msgf("DNS intercept: host resumed after a %s gap - verifying interception is still translating", gap.Round(time.Second)) + p.noteHostWoke("detector", gap, freshNetworkState()) + // A resume moves the resolvers of the host, so the poll goes fast again. + if p.dnsConfig != nil { + p.dnsConfig.noteActivity(networkEventsNowFn()) + } repairs := 0 probed := 0 @@ -1702,7 +1784,7 @@ func (p *prog) waitBeforeWakeProbe(delay time.Duration) bool { // could not run does not count: nothing was changed, so the next attempt may try again. // The caller holds functional-probe ownership for the call. func (p *prog) runWakeProbeAttempt(attempt, repairs int) (restored, repaired bool) { - initializeOsResolver(true) + initializeOsResolver(true, osResolverReasonWakeProbe) result := probePFInterceptFn(p) if result.result == pfProbeIndeterminate { mainLog.Load().Debug().Msg("DNS intercept: post-wake probe indeterminate; repair deferred") @@ -1743,6 +1825,77 @@ var runPFAnchorCheckCommand = func(args ...string) ([]byte, error) { return exec.Command("pfctl", args...).CombinedOutput() } +const ( + // pfAnchorListMessage names the anchor state event of the journal. + pfAnchorListMessage = "PF anchor list changed" + + pfAnchorReasonStabilizationStart = "stabilization_start" + pfAnchorReasonStabilizationEnd = "stabilization_end" + pfAnchorReasonMissing = "missing" + pfAnchorReasonRestored = "restored" + + // pfAnchorStateKey holds the last anchor state, so an anchor that stays + // missing for a whole storm reaches the journal once. + pfAnchorStateKey = "pf_anchor_state" + + // pfAnchorWatchdogKey holds the last state the watchdog saw, so the healthy + // line reports the recovery and not every tick between two of them. + pfAnchorWatchdogKey = "pf_anchor_intact" + + pfAnchorIntactState = "pf anchor intact" + pfAnchorMissingState = "pf anchor missing" + + pfAnchorIntactMessage = "DNS intercept watchdog: " + pfAnchorIntactState +) + +// logPFAnchorList reports which anchors the running ruleset refers to. A capture +// of an outage has to tell whether ctrld or another program owned pf, and the +// anchor names are the only evidence of that. +func (p *prog) logPFAnchorList(reason string, level *zerolog.Event, dump pfRuleDump) { + anchors := p.pfAnchorNames(dump) + enabled, since := p.pfStatus() + if changed, _ := p.repeats.changed(pfAnchorStateKey, pfAnchorStateValue(reason, anchors, enabled)); !changed { + return + } + event := journal(level). + Str("reason", reason). + Bool("pf_enabled", enabled). + Str("pf_since", since) + if anchors == nil { + event.Bool("anchors_known", false).Msg(pfAnchorListMessage) + return + } + event.Strs("anchors", anchors).Msg(pfAnchorListMessage) +} + +// pfAnchorStateValue leaves the pf uptime out, because it grows between two +// reads of the same state. +func pfAnchorStateValue(reason string, anchors []string, enabled bool) string { + if anchors == nil { + return fmt.Sprintf("%s|%t|unknown", reason, enabled) + } + return fmt.Sprintf("%s|%t|%s", reason, enabled, strings.Join(anchors, ",")) +} + +// logPFAnchorIntact reports a healthy anchor. The watchdog finds it healthy on +// almost every tick for the life of the daemon, so only a change belongs in the +// log. +func (p *prog) logPFAnchorIntact() { + changed, repeats := p.repeats.changed(pfAnchorWatchdogKey, pfAnchorIntactState) + if !changed { + return + } + mainLog.Load().Debug().Uint64("repeats", repeats).Msg(pfAnchorIntactMessage) +} + +// notePFAnchorMissing reports the wiped anchor with the anchors that replaced +// it. The record of the missing state makes the next healthy check reach the +// log, so a capture holds both ends of the outage. +func (p *prog) notePFAnchorMissing(dump pfRuleDump) { + p.repeats.changed(pfAnchorWatchdogKey, pfAnchorMissingState) + p.logPFAnchorList(pfAnchorReasonMissing, mainLog.Load().Warn(), dump) +} + // ensurePFAnchorActive checks live PF state and returns an explicit outcome for // callers that must distinguish intact, restored, deferred, skipped, and failed. func (p *prog) ensurePFAnchorActive() pfAnchorCheckResult { @@ -1784,6 +1937,9 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu rdrAnchorRef := fmt.Sprintf("rdr-anchor \"%s\"", pfAnchorName) anchorRef := fmt.Sprintf("anchor \"%s\"", pfAnchorName) needsRestore := false + // The anchor event parses the rules this check reads, so it starts no + // second pair of pfctl processes. + var dump pfRuleDump // Check 1: anchor references in the main ruleset. natOut, err := runPFAnchorCheckCommand("-sn") @@ -1792,6 +1948,7 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu mainLog.Load().Warn().Err(err).Msg("DNS intercept watchdog: could not dump NAT rules") return pfAnchorCheckFailed } + dump.nat = natOut natStr := string(natOut) if !strings.Contains(natStr, rdrAnchorRef) { mainLog.Load().Warn().Msg("DNS intercept watchdog: rdr-anchor reference missing from running ruleset") @@ -1805,6 +1962,7 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu mainLog.Load().Warn().Err(err).Msg("DNS intercept watchdog: could not dump filter rules") return pfAnchorCheckFailed } + dump.rules = filterOut if !strings.Contains(string(filterOut), anchorRef) { mainLog.Load().Warn().Msg("DNS intercept watchdog: anchor reference missing from running filter rules") needsRestore = true @@ -1864,6 +2022,7 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu p.pfBackoffMultiplier.Add(1) mainLog.Load().Warn().Msgf("DNS intercept: rules wiped %s after restore — entering stabilization (backoff multiplier: %d)", elapsed, p.pfBackoffMultiplier.Load()) + p.notePFAnchorMissing(dump) p.pfStartStabilization() return pfAnchorCheckDeferred } @@ -1873,7 +2032,7 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu } if !needsRestore && !forceRebuild { - mainLog.Load().Debug().Msg("DNS intercept watchdog: pf anchor intact") + p.logPFAnchorIntact() return pfAnchorCheckIntact } @@ -1881,6 +2040,9 @@ func (p *prog) ensurePFAnchorActiveWithPolicy(allowRecentWipeDeferral, forceRebu if forceRebuild && !needsRestore { reason = "post-stabilization rebuild" } + if needsRestore { + p.notePFAnchorMissing(dump) + } return restorePFAnchorForReconcile(p, reason) } @@ -2007,7 +2169,7 @@ func (p *prog) scheduleDelayedRechecks() { // Refresh OS resolver — VPN may have finished DNS cleanup since the // immediate handler ran. This clears stale LAN nameservers (e.g., // a VPN's DNS IP (e.g., 10.255.255.3) lingering in scutil --dns). - ctrld.InitializeOsResolver(true) + ctrld.InitializeOsResolverWithReason(true, osResolverReasonDelayedRecheck) if p.vpnDNS != nil { p.vpnDNS.Refresh(true) } @@ -2145,7 +2307,7 @@ func (p *prog) exemptVPNDNSServers(exemptions []vpnDNSExemption) error { } defer p.pfEnsureRunning.Store(false) - tunnelIfaces := append([]string(nil), discoverTunnelInterfacesForReconcile()...) + tunnelIfaces := append([]string(nil), p.activeTunnelInterfaces()...) rulesStr := p.buildPFAnchorRulesForTunnels(exemptions, tunnelIfaces) if err := os.WriteFile(pfAnchorFile, []byte(rulesStr), 0644); err != nil { diff --git a/cmd/cli/dns_intercept_darwin_test.go b/cmd/cli/dns_intercept_darwin_test.go index 6c8b24e0..fa9b45b4 100644 --- a/cmd/cli/dns_intercept_darwin_test.go +++ b/cmd/cli/dns_intercept_darwin_test.go @@ -6,9 +6,12 @@ import ( "context" "errors" "fmt" + "net" "os" "path/filepath" + "slices" "strings" + "sync" "testing" "time" @@ -987,7 +990,7 @@ func TestFinishPFStabilizationRunsFunctionalVerification(t *testing.T) { "-a com.controld.ctrld -sn": "rdr on lo0", }) originalResolver := initializeOsResolver - initializeOsResolver = func(bool) []string { return nil } + initializeOsResolver = func(bool, string) []string { return nil } t.Cleanup(func() { initializeOsResolver = originalResolver }) probes, reloads := stubStabilizationProbe(t, []bool{false, true}, true) @@ -1092,7 +1095,7 @@ func stubWakeProbeSchedule(t *testing.T, attempts int) (refreshes *int) { pfWakeProbeOwnerWait = 10 * time.Millisecond calls := 0 - initializeOsResolver = func(bool) []string { + initializeOsResolver = func(bool, string) []string { calls++ return []string{"10.0.0.1:53"} } @@ -1275,7 +1278,7 @@ func TestWakeProbeStopsWhenServiceStops(t *testing.T) { pfWakeProbeDelays = []time.Duration{0, time.Hour} t.Cleanup(func() { pfWakeProbeDelays = originalDelays }) originalResolver := initializeOsResolver - initializeOsResolver = func(bool) []string { return []string{"10.0.0.1:53"} } + initializeOsResolver = func(bool, string) []string { return []string{"10.0.0.1:53"} } t.Cleanup(func() { initializeOsResolver = originalResolver }) probes, _ := stubStabilizationProbe(t, nil, false) // first probe fails, reload refuses @@ -1433,3 +1436,531 @@ func TestStartDNSInterceptSkipsBackgroundWorkWhenInstallFails(t *testing.T) { // wiring a moment to report rather than racing the test's end. time.Sleep(50 * time.Millisecond) } + +// ============================================================================= +// pf anchor state, tunnel, and wake event tests +// ============================================================================= + +// pfStateCommandsMu guards the answer of the pf and tunnel state reads, because +// a watchdog or stabilization goroutine can read while a test replaces it. +var ( + pfStateCommandsMu sync.Mutex + pfStateCommandFn func(string, ...string) ([]byte, error) +) + +// init keeps the whole test binary away from pfctl and ifconfig. A test that +// needs values stubs them with stubPFStateCommands. +func init() { + pfStateRunCommand = func(name string, args ...string) ([]byte, error) { + pfStateCommandsMu.Lock() + run := pfStateCommandFn + pfStateCommandsMu.Unlock() + if run == nil { + return nil, errors.New("pf state read is not stubbed") + } + return run(name, args...) + } +} + +// stubPFStateCommands answers the pf and tunnel state reads from fixture text, +// keyed by the whole command line. +func stubPFStateCommands(t *testing.T, outputs map[string]string) { + t.Helper() + stubPFStateCommandFn(t, func(name string, args ...string) ([]byte, error) { + key := strings.Join(append([]string{name}, args...), " ") + output, ok := outputs[key] + if !ok { + return nil, fmt.Errorf("no fixture for %q", key) + } + return []byte(output), nil + }) +} + +// stubPFStateCommandFn answers the pf and tunnel state reads from run. +func stubPFStateCommandFn(t *testing.T, run func(string, ...string) ([]byte, error)) { + t.Helper() + pfStateCommandsMu.Lock() + original := pfStateCommandFn + pfStateCommandFn = run + pfStateCommandsMu.Unlock() + t.Cleanup(func() { + pfStateCommandsMu.Lock() + pfStateCommandFn = original + pfStateCommandsMu.Unlock() + }) +} + +// pfIntactAnchorRules answers the anchor check of a healthy ruleset. +var pfIntactAnchorRules = map[string]string{ + "-sn": `rdr-anchor "com.controld.ctrld"`, + "-sr": `anchor "com.controld.ctrld"`, + "-a com.controld.ctrld -sr": "pass in quick on lo0", + "-a com.controld.ctrld -sn": "rdr on lo0", +} + +// pfStateFixtures answers the anchor list and status reads with a real ruleset. +var pfStateFixtures = map[string]string{ + "pfctl -sr": pfShowRules, + "pfctl -sn": pfShowNAT, + "pfctl -si": pfStatusEnabled, +} + +// waitForStabilizationExit waits for the stabilization loop to release pf. +func waitForStabilizationExit(t *testing.T, p *prog) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for p.pfStabilizing.Load() && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if p.pfStabilizing.Load() { + t.Fatal("the stabilization loop did not observe the closed stop channel") + } +} + +// Test_logPFAnchorListLogsStabilizationStartAndEnd covers the anchor list at the +// two stabilization events. A capture of a VPN connect has to name the anchors +// that were loaded before ctrld waited, and the ones that were loaded after. +func Test_logPFAnchorListLogsStabilizationStartAndEnd(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + stubPFAnchorCheckCommand(t, pfIntactAnchorRules) + originalResolver := initializeOsResolver + initializeOsResolver = func(bool, string) []string { return nil } + t.Cleanup(func() { initializeOsResolver = originalResolver }) + stubStabilizationProbe(t, []bool{true}, true) + + stopCh := make(chan struct{}) + close(stopCh) + p := &prog{dnsInterceptState: &pfState{}, stopCh: stopCh} + p.pfStartStabilization() + waitForStabilizationExit(t, p) + p.pfStabilizing.Store(true) + p.finishPFStabilization(time.Millisecond) + stopPFTestTimers(p) + + events := jsonLogEvents(t, logs, "PF anchor list changed") + if len(events) != 2 { + t.Fatalf("got %d anchor list events for one stabilization, want 2", len(events)) + } + wantField(t, events[0], "reason", "stabilization_start") + wantField(t, events[0], "journal", true) + wantField(t, events[0], "pf_enabled", true) + wantField(t, events[0], "pf_since", "0 days 02:11:05") + wantAnchors(t, events[0], []string{"com.apple/*", "com.controld.ctrld"}) + wantField(t, events[1], "reason", "stabilization_end") + wantField(t, events[1], "journal", true) + + for _, prefix := range []string{ + "DNS intercept: VPN connecting", + "DNS intercept: pf stable for", + } { + lines := eventsWithPrefix(t, logs, prefix) + if len(lines) != 1 { + t.Fatalf("got %d lines with prefix %q, want 1", len(lines), prefix) + } + wantField(t, lines[0], "journal", true) + } +} + +// Test_logPFAnchorListReportsAMissingAnchorOnce covers the watchdog state. The +// anchor stays missing for as long as another program holds pf, and one line per +// tick would fill the journal with the same fact. +func Test_logPFAnchorListReportsAMissingAnchorOnce(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + outputs := pfIntactAnchorRules + original := runPFAnchorCheckCommand + runPFAnchorCheckCommand = func(args ...string) ([]byte, error) { + return []byte(outputs[strings.Join(args, " ")]), nil + } + originalRestore := restorePFAnchorForReconcile + restorePFAnchorForReconcile = func(*prog, string) pfAnchorCheckResult { return pfAnchorCheckRestored } + t.Cleanup(func() { + runPFAnchorCheckCommand = original + restorePFAnchorForReconcile = originalRestore + }) + + p := &prog{dnsInterceptState: &pfState{}} + for range 3 { + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckIntact { + t.Fatalf("healthy check result = %v, want intact", result) + } + } + intact := eventsWithPrefix(t, logs, "DNS intercept watchdog: pf anchor intact") + if len(intact) != 1 { + t.Fatalf("got %d intact lines for three healthy checks, want 1", len(intact)) + } + wantField(t, intact[0], "repeats", float64(0)) + + outputs = map[string]string{} + for range 2 { + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckRestored { + t.Fatalf("wiped check result = %v, want restored", result) + } + } + missing := jsonLogEvents(t, logs, "PF anchor list changed") + if len(missing) != 1 { + t.Fatalf("got %d anchor list events for two wiped checks, want 1", len(missing)) + } + wantField(t, missing[0], "reason", "missing") + wantField(t, missing[0], "level", "warn") + wantField(t, missing[0], "journal", true) + + outputs = pfIntactAnchorRules + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckIntact { + t.Fatalf("recovered check result = %v, want intact", result) + } + intact = eventsWithPrefix(t, logs, "DNS intercept watchdog: pf anchor intact") + if len(intact) != 2 { + t.Fatalf("got %d intact lines after the anchor came back, want 2", len(intact)) + } + // The second wiped check repeated the state of the first one, and that held + // line is the count the recovered line carries. + wantField(t, intact[1], "repeats", float64(1)) +} + +// Test_logPFAnchorListReportsARestoredAnchor covers the restore report. The +// missing event alone leaves a capture without the anchors ctrld ended with. +func Test_logPFAnchorListReportsARestoredAnchor(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + originalReference := ensurePFAnchorReferenceForRestore + originalRebuild := rebuildPFAnchorRulesForReconcile + originalVerify := verifyPFStateFn + ensurePFAnchorReferenceForRestore = func(*prog) error { return nil } + rebuildPFAnchorRulesForReconcile = func(*prog, []vpnDNSExemption) ([]string, error) { return nil, nil } + verifyPFStateFn = func(*prog) bool { return true } + t.Cleanup(func() { + ensurePFAnchorReferenceForRestore = originalReference + rebuildPFAnchorRulesForReconcile = originalRebuild + verifyPFStateFn = originalVerify + }) + + p := &prog{dnsInterceptState: &pfState{}} + if result := p.restorePFAnchorWithTransportReset("test", false); result != pfAnchorCheckRestored { + t.Fatalf("restore result = %v, want restored", result) + } + + events := jsonLogEvents(t, logs, "PF anchor list changed") + if len(events) != 1 { + t.Fatalf("got %d anchor list events for one restore, want 1", len(events)) + } + wantField(t, events[0], "reason", "restored") + wantField(t, events[0], "level", "info") + wantField(t, events[0], "journal", true) + wantAnchors(t, events[0], []string{"com.apple/*", "com.controld.ctrld"}) +} + +// Test_logPFAnchorListMarksAnUnknownAnchorList covers a failed read. pfctl fails +// on a host that revoked the privileges of ctrld, and an empty list there reads +// as a ruleset that holds no anchor at all. +func Test_logPFAnchorListMarksAnUnknownAnchorList(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, nil) + + p := &prog{} + p.logPFAnchorList(pfAnchorReasonRestored, mainLog.Load().Info(), pfRuleDump{}) + + events := jsonLogEvents(t, logs, "PF anchor list changed") + if len(events) != 1 { + t.Fatalf("got %d anchor list events, want 1", len(events)) + } + wantField(t, events[0], "anchors_known", false) + if _, ok := events[0]["anchors"]; ok { + t.Fatalf("a failed read reported an anchor list: %v", events[0]) + } +} + +// TestTunnelInterfaceChangedNamesTheOwner covers the tunnel event. The #611 +// capture could not tell a VPN tunnel from a mesh client, and the owner is the +// field that names the program whose pf rules compete with the ctrld anchor. +func TestTunnelInterfaceChangedNamesTheOwner(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, map[string]string{"ifconfig -v utun9": ifconfigWindscribeTunnel}) + originalDiscover := discoverTunnelInterfacesForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return []string{"utun9"} } + t.Cleanup(func() { discoverTunnelInterfacesForReconcile = originalDiscover }) + + p := &prog{dnsInterceptState: &pfState{}, lastTunnelIfaces: []string{"utun7"}} + // Stabilization owns the rebuild, so the event is all this drives. + p.pfStabilizing.Store(true) + if !p.checkTunnelInterfaceChanges() { + t.Fatal("the tunnel set change was not observed") + } + + events := jsonLogEvents(t, logs, "Tunnel interface changed") + if len(events) != 1 { + t.Fatalf("got %d tunnel events for one set change, want 1", len(events)) + } + wantField(t, events[0], "owner", "Windscribe VPN") + wantField(t, events[0], "journal", true) + wantStrings(t, events[0], "added", []string{"utun9"}) + wantStrings(t, events[0], "removed", []string{"utun7"}) + + discovered := eventsWithPrefix(t, logs, "DNS intercept: discovered active tunnel interfaces") + if len(discovered) != 1 { + t.Fatalf("got %d discovery lines for one tunnel set, want 1", len(discovered)) + } + if p.checkTunnelInterfaceChanges() { + t.Fatal("the identical pending tunnel set was not coalesced") + } + discovered = eventsWithPrefix(t, logs, "DNS intercept: discovered active tunnel interfaces") + if len(discovered) != 1 { + t.Fatalf("got %d discovery lines for an unchanged tunnel set, want 1", len(discovered)) + } +} + +// TestVerifyInterceptAfterWakeReportsTheWake covers the detector source of the +// wake event. The 2026-09-14 capture has no wake line at all, and the resolver +// table after a resume is the state the investigation needs next. +func TestVerifyInterceptAfterWakeReportsTheWake(t *testing.T) { + logs := captureDebugMainLog(t) + stubWakeProbeSchedule(t, 4) + stubStabilizationProbe(t, []bool{true}, true) + now := time.Date(2026, 9, 14, 7, 14, 0, 0, time.UTC) + originalNow := networkEventsNowFn + networkEventsNowFn = func() time.Time { return now } + t.Cleanup(func() { networkEventsNowFn = originalNow }) + + p := &prog{ + dnsInterceptState: &pfState{}, + dnsConfig: newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return nil, nil }), + } + p.verifyInterceptAfterWake(18 * time.Second) + + events := jsonLogEvents(t, logs, hostWokeMessage) + if len(events) != 1 { + t.Fatalf("got %d Host woke events for one resume, want 1", len(events)) + } + wantField(t, events[0], "source", "detector") + wantField(t, events[0], "gap_ms", float64(18000)) + wantField(t, events[0], "journal", true) + if got := p.dnsConfig.nextDelay(now); got != dnsConfigFastInterval { + t.Fatalf("DNS configuration poll delay after a wake = %s, want %s", got, dnsConfigFastInterval) + } +} + +func wantAnchors(t *testing.T, event map[string]any, want []string) { + t.Helper() + wantStrings(t, event, "anchors", want) +} + +func wantStrings(t *testing.T, event map[string]any, field string, want []string) { + t.Helper() + values, ok := event[field].([]any) + if !ok { + t.Fatalf("field %q: got %v, want a list", field, event[field]) + } + if len(values) != len(want) { + t.Fatalf("field %q: got %v, want %v", field, values, want) + } + for i, value := range values { + if value != want[i] { + t.Fatalf("field %q: got %v, want %v", field, values, want) + } + } +} + +// TestIgnoredNetworkChangeJournalsTheInterfaceChange proves that an interface +// that appears or disappears on the ignored path enters the journal with its +// class and its action, so a hypervisor adapter storm is readable after the fact. +func TestIgnoredNetworkChangeJournalsTheInterfaceChange(t *testing.T) { + logs := captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + stubSnapshotVirtualSet(t, "bridge101") + originalDiscover := discoverTunnelInterfacesForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return nil } + t.Cleanup(func() { discoverTunnelInterfacesForReconcile = originalDiscover }) + + p := &prog{dnsInterceptState: &pfState{}, vpnDNS: newVPNDNSManager(nil)} + p.pfStabilizing.Store(true) + t.Cleanup(func() { + p.pfDelayedRecheckMu.Lock() + defer p.pfDelayedRecheckMu.Unlock() + for _, timer := range p.pfDelayedRecheckTimers { + if timer != nil { + timer.Stop() + } + } + }) + stateWith := func(names ...string) *netmon.State { + state := &netmon.State{Interface: map[string]netmon.Interface{}} + for _, name := range names { + state.Interface[name] = netmon.Interface{Interface: &net.Interface{Name: name, Flags: net.FlagUp}} + } + return state + } + now := time.Unix(1_000_000, 0) + + p.handleDNSInterceptIgnoredNetworkChange(&netmon.ChangeDelta{Old: stateWith("en0"), New: stateWith("en0", "bridge101")}, now) + p.handleDNSInterceptIgnoredNetworkChange(&netmon.ChangeDelta{Old: stateWith("en0", "bridge101"), New: stateWith("en0")}, now.Add(time.Second)) + + events := jsonLogEvents(t, logs, "DNS intercept: interface appeared/disappeared — starting interception probe monitor") + if len(events) != 2 { + t.Fatalf("interface events: got %d, want 2: %s", len(events), logs.String()) + } + for i, action := range []string{"added", "removed"} { + wantField(t, events[i], "journal", true) + wantField(t, events[i], "interface", "bridge101") + wantField(t, events[i], "class", "virtual") + wantField(t, events[i], "action", action) + } +} + +// TestTunnelInterfaceChangedLogsOncePerChange covers the retry path and the +// flap back. The reconcile keeps a new tunnel set pending until it succeeds, +// so every retry repeated the same event, and a set that returned to the one +// before it reported nothing at all. +func TestTunnelInterfaceChangedLogsOncePerChange(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + originalDiscover := discoverTunnelInterfacesForReconcile + originalRestore := restorePFAnchorForReconcile + discoverTunnelInterfacesForReconcile = func() []string { return []string{"utun7"} } + restorePFAnchorForReconcile = func(*prog, string) pfAnchorCheckResult { return pfAnchorCheckFailed } + t.Cleanup(func() { + discoverTunnelInterfacesForReconcile = originalDiscover + restorePFAnchorForReconcile = originalRestore + }) + + p := &prog{dnsInterceptState: &pfState{}, lastTunnelIfaces: []string{"utun7", "utun9"}} + for range 3 { + p.checkTunnelInterfaceChanges() + } + + events := jsonLogEvents(t, logs, tunnelChangedMessage) + if len(events) != 1 { + t.Fatalf("got %d tunnel events for three retries of one change, want 1", len(events)) + } + wantStrings(t, events[0], "removed", []string{"utun9"}) + + // The tunnel comes back before a reconcile applied its removal. + discoverTunnelInterfacesForReconcile = func() []string { return []string{"utun7", "utun9"} } + p.checkTunnelInterfaceChanges() + + events = jsonLogEvents(t, logs, tunnelChangedMessage) + if len(events) != 2 { + t.Fatalf("got %d tunnel events after the tunnel came back, want 2", len(events)) + } + wantStrings(t, events[1], "added", []string{"utun9"}) +} + +// TestPFAnchorWipeAfterRestoreReportsTheMissingAnchor covers the wipe that +// follows a restore within ten seconds. That path enters stabilization instead +// of a rebuild, and it left the journal without the wipe that caused it. +func TestPFAnchorWipeAfterRestoreReportsTheMissingAnchor(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + stubPFAnchorCheckCommand(t, map[string]string{"-sn": ""}) + stubStabilizationProbe(t, []bool{true}, true) + stopCh := make(chan struct{}) + close(stopCh) + + p := &prog{dnsInterceptState: &pfState{}, stopCh: stopCh} + p.pfLastRestoreTime.Store(time.Now().UnixMilli()) + + if result := p.ensurePFAnchorActive(); result != pfAnchorCheckDeferred { + t.Fatalf("wipe after restore result = %v, want deferred", result) + } + waitForStabilizationExit(t, p) + stopPFTestTimers(p) + + events := jsonLogEvents(t, logs, pfAnchorListMessage) + if len(events) != 2 { + t.Fatalf("got %d anchor list events for one wipe, want 2", len(events)) + } + wantField(t, events[0], "reason", pfAnchorReasonMissing) + wantField(t, events[0], "level", "warn") + wantField(t, events[1], "reason", pfAnchorReasonStabilizationStart) +} + +// TestPFAnchorListParsesTheRulesTheCallerHolds covers the anchor read of the +// watchdog. The watchdog already read both rule sets, and one more pair of +// pfctl processes on the network change path is work the host does not need. +func TestPFAnchorListParsesTheRulesTheCallerHolds(t *testing.T) { + logs := captureDebugMainLog(t) + var reads []string + stubPFStateCommandFn(t, func(name string, args ...string) ([]byte, error) { + key := strings.Join(append([]string{name}, args...), " ") + reads = append(reads, key) + output, ok := pfStateFixtures[key] + if !ok { + return nil, fmt.Errorf("no fixture for %q", key) + } + return []byte(output), nil + }) + + p := &prog{} + p.logPFAnchorList(pfAnchorReasonMissing, mainLog.Load().Warn(), pfRuleDump{ + rules: []byte(pfShowRules), + nat: []byte(pfShowNAT), + }) + + if want := []string{"pfctl -si"}; !slices.Equal(reads, want) { + t.Fatalf("pf state reads = %v, want %v", reads, want) + } + events := jsonLogEvents(t, logs, pfAnchorListMessage) + if len(events) != 1 { + t.Fatalf("got %d anchor list events, want 1", len(events)) + } + wantAnchors(t, events[0], []string{"com.apple/*", "com.controld.ctrld"}) + wantField(t, events[0], "pf_enabled", true) +} + +// TestPFAnchorListBacksOffOnExhaustedResources covers a failed anchor read. +// Every other pfctl read feeds the backoff, so the anchor read kept starting +// processes on a host that had none left. +func TestPFAnchorListBacksOffOnExhaustedResources(t *testing.T) { + captureDebugMainLog(t) + stubPFStateCommandFn(t, func(string, ...string) ([]byte, error) { + return nil, errors.New("fork/exec /sbin/pfctl: resource temporarily unavailable") + }) + + p := &prog{} + p.logPFAnchorList(pfAnchorReasonRestored, mainLog.Load().Info(), pfRuleDump{}) + + if !p.pfExecBackoffActive() { + t.Fatal("a failed anchor read did not start the pfctl backoff") + } +} + +// TestPFAnchorListMarksATruncatedRuleListUnknown covers an oversized read. A +// reader stops at a line it cannot hold, and the anchors it collected before +// that line read as a ruleset that lost the rest of them. +func TestPFAnchorListMarksATruncatedRuleListUnknown(t *testing.T) { + logs := captureDebugMainLog(t) + stubPFStateCommands(t, pfStateFixtures) + + p := &prog{} + p.logPFAnchorList(pfAnchorReasonRestored, mainLog.Load().Info(), pfRuleDump{ + rules: []byte(pfShowRules + strings.Repeat("a", 70<<10) + "\n"), + nat: []byte(pfShowNAT), + }) + + events := jsonLogEvents(t, logs, pfAnchorListMessage) + if len(events) != 1 { + t.Fatalf("got %d anchor list events, want 1", len(events)) + } + wantField(t, events[0], "anchors_known", false) + if _, ok := events[0]["anchors"]; ok { + t.Fatalf("a truncated read reported an anchor list: %v", events[0]) + } +} + +// TestPFStateOutputIsBounded covers the memory a state read can take. A pfctl +// that prints without end must not grow the daemon. +func TestPFStateOutputIsBounded(t *testing.T) { + out := &boundedBuffer{limit: 8} + if _, err := out.Write([]byte("12345")); err != nil { + t.Fatalf("write below the limit: %v", err) + } + if _, err := out.Write([]byte("67890")); err != nil { + t.Fatalf("write above the limit: %v", err) + } + if !out.exceeded { + t.Fatal("the writer did not report the dropped bytes") + } + if got := out.Len(); got > 8 { + t.Fatalf("the writer kept %d bytes, want at most 8", got) + } +} diff --git a/cmd/cli/dns_intercept_settle.go b/cmd/cli/dns_intercept_settle.go index 972a2abb..2af934da 100644 --- a/cmd/cli/dns_intercept_settle.go +++ b/cmd/cli/dns_intercept_settle.go @@ -2,11 +2,20 @@ package cli import "github.com/Control-D-Inc/ctrld" -var initializeOsResolver = ctrld.InitializeOsResolver +var initializeOsResolver = ctrld.InitializeOsResolverWithReason + +// Reasons that the journal reports when the resolver set changes. Each one +// names the code path that read the resolvers of the host. +const ( + osResolverReasonStart = "start" + osResolverReasonWakeProbe = "wake_probe" + osResolverReasonVPNSettle = "vpn_settle" + osResolverReasonDelayedRecheck = "delayed_recheck" +) func (p *prog) refreshDNSAfterVPNSettle(reason string) (routes, domainlessServers, exemptions int) { mainLog.Load().Info().Msgf("DNS intercept: refreshing OS/VPN DNS route state after VPN settle (%s)", reason) - ns := initializeOsResolver(true) + ns := initializeOsResolver(true, osResolverReasonVPNSettle) mainLog.Load().Debug().Msgf("DNS intercept: post-settle OS resolver nameservers: %v", ns) if p.vpnDNS == nil { diff --git a/cmd/cli/dns_intercept_settle_test.go b/cmd/cli/dns_intercept_settle_test.go index 36cc26f8..c2216d05 100644 --- a/cmd/cli/dns_intercept_settle_test.go +++ b/cmd/cli/dns_intercept_settle_test.go @@ -12,8 +12,10 @@ func TestRefreshDNSAfterVPNSettleRefreshesOSResolverAndVPNRoutes(t *testing.T) { defer func() { initializeOsResolver = oldInitialize }() var initialized []bool - initializeOsResolver = func(force bool) []string { + var reasons []string + initializeOsResolver = func(force bool, reason string) []string { initialized = append(initialized, force) + reasons = append(reasons, reason) return []string{"10.102.26.10:53"} } @@ -40,6 +42,9 @@ func TestRefreshDNSAfterVPNSettleRefreshesOSResolverAndVPNRoutes(t *testing.T) { if len(initialized) != 1 || !initialized[0] { t.Fatalf("expected forced OS resolver refresh once, got %v", initialized) } + if len(reasons) != 1 || reasons[0] != osResolverReasonVPNSettle { + t.Fatalf("OS resolver reasons = %v, want [%s]", reasons, osResolverReasonVPNSettle) + } if got := p.vpnDNS.UpstreamForDomain("jira.cc.bmwgroup.net."); len(got) != 1 || got[0] != "10.102.26.10" { t.Fatalf("expected refreshed VPN DNS route, got %v", got) } diff --git a/cmd/cli/dns_intercept_windows.go b/cmd/cli/dns_intercept_windows.go index 95738621..4dd87d25 100644 --- a/cmd/cli/dns_intercept_windows.go +++ b/cmd/cli/dns_intercept_windows.go @@ -2749,7 +2749,7 @@ func (p *prog) scheduleDelayedRechecks() { } // Refresh OS resolver — VPN may have finished DNS cleanup since the // immediate handler ran. - ctrld.InitializeOsResolver(true) + ctrld.InitializeOsResolverWithReason(true, osResolverReasonDelayedRecheck) if p.vpnDNS != nil { p.vpnDNS.Refresh(true) } diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index 3be5cfc5..bb6b0f58 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -264,6 +264,8 @@ func (p *prog) serveDNS(listenerNum string) error { } labelValues = append(labelValues, dns.TypeToString[q.Qtype]) labelValues = append(labelValues, dns.RcodeToString[answer.Rcode]) + // The grade of the window must not depend on the goroutine below. + p.health.countQuery() go func() { p.WithLabelValuesInc(statsQueriesCount, labelValues...) p.WithLabelValuesInc(statsClientQueriesCount, []string{ci.IP, ci.Mac, ci.Hostname}...) @@ -689,6 +691,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { } else { ctrld.Log(ctx, mainLog.Load().Debug(), "hit cached response") } + p.health.countCacheHit() res.answer = answer res.cached = true return res @@ -748,6 +751,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { if gotTransportFailure && p.vpnDNS.ShouldFailClosedAfterVPNDNSTransportFailure(domain, vpnServers) { ctrld.Log(ctx, mainLog.Load().Debug(), "All VPN DNS servers had transport failures for %s; returning SERVFAIL while retained VPN DNS state is active", domain) + p.countFailedClientQuery(upstreams) answer := new(dns.Msg) answer.SetRcode(req.msg, dns.RcodeServerFailure) return &proxyResponse{answer: answer} @@ -815,6 +819,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { if !gotDNSAnswer && gotTransportFailure && p.vpnDNS.ShouldFailClosedAfterVPNDNSTransportFailure(domain, dlServers) { ctrld.Log(ctx, mainLog.Load().Debug(), "All domain-less VPN DNS servers had transport failures for %s; returning SERVFAIL while retained VPN DNS state is active", domain) + p.countFailedClientQuery(upstreams) answer := new(dns.Msg) answer.SetRcode(req.msg, dns.RcodeServerFailure) return &proxyResponse{answer: answer} @@ -857,7 +862,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { // keep the address-bearing error at debug. if isInternalDomainUpstream(upstream) { ctrld.Log(ctx, mainLog.Load().Debug().Err(err), "failed to resolve query") - ctrld.Log(ctx, p.querySampler.event(sampleClassResolveFailed, upstream).Str("failure", internalDomainFailureReason(err)), + ctrld.Log(ctx, p.querySampler.event(sampleClassInternalDomain, upstream).Str("failure", internalDomainFailureReason(err)), "failed to resolve query using an Internal Domain resolver") } else { ctrld.Log(ctx, p.querySampler.event(sampleClassResolveFailed, upstream).Err(err), "failed to resolve query") @@ -899,6 +904,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { if answer == nil { if serveStaleCache && staleAnswer != nil { ctrld.Log(ctx, mainLog.Load().Debug(), "serving stale cached response") + p.health.countCacheHit() now := time.Now() setCachedAnswerTTL(staleAnswer, now, now.Add(staleTTL)) res.answer = staleAnswer @@ -1017,12 +1023,24 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { } } + p.countFailedClientQuery(upstreams) answer := new(dns.Msg) answer.SetRcode(req.msg, dns.RcodeServerFailure) res.answer = answer return res } +// countFailedClientQuery grades one client query that ended without an answer. +// A query that only Internal Domain resolvers serve stays out of the grade: an +// endpoint away from the organization network never reaches them, and that is +// not an outage of the query path. +func (p *prog) countFailedClientQuery(upstreams []string) { + if internalDomainExplicitUpstreams(upstreams) { + return + } + p.health.countFailedQuery() +} + func (p *prog) upstreamsAndUpstreamConfigForPtr(upstreams []string, upstreamConfigs []*ctrld.UpstreamConfig) ([]string, []*ctrld.UpstreamConfig) { if len(p.localUpstreams) > 0 { tmp := make([]string, 0, len(p.localUpstreams)+len(upstreams)) @@ -1678,12 +1696,66 @@ func (p *prog) handleNetworkChange(delta *netmon.ChangeDelta, isMajorChange bool // delta.Major flag after a time jump does not change the cache. p.networkSourceMu.Lock() currentState := networkChangeCurrentStateFn(delta) - if !networkSnapshotCurrent(delta, isMajorChange, currentState) { + current := networkSnapshotCurrent(delta, isMajorChange, currentState) + if delta.TimeJumped { + // A late callback still tells that the host woke, but the network of + // the wake is the one that netmon holds now. + wakeState := delta.New + if !current { + wakeState = currentState + } + p.noteHostWoke("netmon", 0, wakeState) + } + // Only a current callback becomes the baseline of the next diff. A late + // callback that a newer snapshot replaced would hide the changes between + // the callback before it and the callback after it. + before := p.deltaBeforeState(delta, currentState, current, isMajorChange) + changes := diffNetworkDelta(before, delta.New) + if current && hasAddOrRemove(changes) { + refreshInterfaceMeta() + } + // The noise test runs before any other work, because the point of the class + // is that a storm starts no pfctl and no scutil process. + if current && noiseDelta(before, delta.New, delta.TimeJumped, changes) { p.networkSourceMu.Unlock() - mainLog.Load().Debug().Msg("Network transition skipped: snapshot superseded") + p.noteNoiseDelta(changes) return } - transitionID := p.networkTransitionGen.Add(1) + // A superseded snapshot never becomes a transition, so it takes no ID and + // reports zero. + var transitionID uint64 + changed := false + changedInterface := "" + sourceSnapshotAvailable := false + before4, before6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + after4, after6 := before4, before6 + // The newer callback of this epoch carries the same interface changes, so + // this outcome reports at debug and stays out of the journal. + outcome := transitionOutcomeSnapshotSuperseded + defer func() { + stateBefore, stateAfter := networkStateOrEmpty(before), networkStateOrEmpty(delta.New) + networkTransitionEvent(outcome).Uint64("transition_id", transitionID). + Bool("is_major_change", isMajorChange).Bool("changed", changed). + Str("interface", changedInterface). + Strs("changed_interfaces", changedInterfaceNames(changes)). + Bool("time_jumped", delta.TimeJumped). + Str("default_route_before", stateBefore.DefaultRouteInterface). + Str("default_route_after", stateAfter.DefaultRouteInterface). + Bool("have_v4_before", stateBefore.HaveV4).Bool("have_v4_after", stateAfter.HaveV4). + Bool("have_v6_before", stateBefore.HaveV6).Bool("have_v6_after", stateAfter.HaveV6). + Str("source_ipv4_before", before4.String()).Str("source_ipv6_before", before6.String()). + Str("source_ipv4_after", after4.String()). + Str("source_ipv6_after", after6.String()). + Bool("source_snapshot_available", sourceSnapshotAvailable).Str("outcome", outcome). + Msg(networkTransitionMessage) + }() + if !current { + p.networkSourceMu.Unlock() + return + } + p.flushNoiseSummary() + outcome = transitionOutcomeIgnored + transitionID = p.networkTransitionGen.Add(1) sourceState := delta.New // A late major callback and reordered minor callbacks share one cache // epoch. For these cases, source validity comes from the OS, not ordering. @@ -1707,30 +1779,25 @@ func (p *prog) handleNetworkChange(delta *netmon.ChangeDelta, isMajorChange bool } } p.networkSourceState, p.networkSourceEpoch = sourceState, currentState - sourceSnapshotAvailable := sourceState != nil - before4, before6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + sourceSnapshotAvailable = sourceState != nil validateDefaultLocalIPsFromDelta(sourceState, transitionID) - after4, after6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + after4, after6 = ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + // The network of this callback belongs to the epoch that the lines above + // store. A store after the unlock can overwrite the network of a newer + // callback. The header render reads the hardware ports, so it runs after + // the unlock and starts no process under the lock. + p.lastNetworkState.Store(currentState) + // HasIPv6 reads the flag that this callback stores, so one monitor serves + // the whole process. + ctrld.SetIPv6Available(currentState.HaveV6) p.networkSourceMu.Unlock() - p.noteNetworkState(currentState) + p.refreshLogHeader() + describeInterfaceChanges(changes, interfaceMetaFor) + logInterfaceChanges(transitionID, changes) p.handleDNS64NetworkChange(delta, isMajorChange) validIfaces := networkChangeValidInterfacesFn() - changed := false - changedInterface := "" - outcome := "ignored" - defer func() { - mainLog.Load().Debug().Uint64("transition_id", transitionID). - Bool("is_major_change", isMajorChange).Bool("changed", changed). - Str("interface", changedInterface). - Str("default_route_before", delta.Old.DefaultRouteInterface). - Str("default_route_after", delta.New.DefaultRouteInterface). - Str("source_ipv4_before", before4.String()).Str("source_ipv6_before", before6.String()). - Str("source_ipv4_after", after4.String()). - Str("source_ipv6_after", after6.String()). - Bool("source_snapshot_available", sourceSnapshotAvailable).Str("outcome", outcome).Msg("Network transition") - }() activeInterfaceExists := false var changeIPs []netip.Prefix // Check each valid interface for changes @@ -1750,6 +1817,9 @@ func (p *prog) handleNetworkChange(delta *netmon.ChangeDelta, isMajorChange bool // The interface is new (was not present in the old state). usableNewIPs := filterUsableIPs(newIPs) if newIface.IsUp() && len(usableNewIPs) > 0 { + // The new interface is the active one, or the change ends as + // no_active_interface and nothing reconciles. + activeInterfaceExists = true changed = true changeIPs = usableNewIPs changedInterface = ifaceName @@ -1905,6 +1975,10 @@ func (p *prog) handleNetworkChange(delta *netmon.ChangeDelta, isMajorChange bool outcome = "accepted" networkChangeReconcileFn(p, transitionID) + p.logNetworkSnapshot("transition") + if p.dnsConfig != nil { + p.dnsConfig.noteActivity(networkEventsNowFn()) + } } // reconcileNetworkChange is the first DNS/PF mutation boundary after source updates. @@ -1963,32 +2037,16 @@ func (p *prog) handleDNSInterceptIgnoredNetworkChange(delta *netmon.ChangeDelta, // Spawn an async monitor that probes pf interception with backoff and forces // a full pf reload if broken. if delta.Old != nil { - interfaceChanged := false - var changedIface string - for ifaceName := range delta.Old.Interface { - if ifaceName == "lo0" { - continue - } - if _, exists := delta.New.Interface[ifaceName]; !exists { - interfaceChanged = true - changedIface = ifaceName - break - } - } - if !interfaceChanged { - for ifaceName := range delta.New.Interface { - if ifaceName == "lo0" { - continue - } - if _, exists := delta.Old.Interface[ifaceName]; !exists { - interfaceChanged = true - changedIface = ifaceName - break - } - } - } - if interfaceChanged { - mainLog.Load().Info().Str("interface", changedIface). + changedAction := "removed" + changedIface := interfaceOnlyIn(delta.Old.Interface, delta.New.Interface) + if changedIface == "" { + changedAction = "added" + changedIface = interfaceOnlyIn(delta.New.Interface, delta.Old.Interface) + } + if changedIface != "" { + journal(mainLog.Load().Info()).Str("interface", changedIface). + Str("class", interfaceMetaFor(changedIface).Class). + Str("action", changedAction). Msg("DNS intercept: interface appeared/disappeared — starting interception probe monitor") go p.pfInterceptMonitor() } @@ -2005,6 +2063,20 @@ func (p *prog) handleDNSInterceptIgnoredNetworkChange(delta *netmon.ChangeDelta, } } +// interfaceOnlyIn names one interface that have holds and lack does not. The +// loopback stays out, because it never appears and never goes away. +func interfaceOnlyIn(have, lack map[string]netmon.Interface) string { + for name := range have { + if name == "lo0" { + continue + } + if _, exists := lack[name]; !exists { + return name + } + } + return "" +} + // interfaceStatesEqual compares two interface states func interfaceStatesEqual(a, b *netmon.Interface) bool { if a == nil || b == nil { @@ -2133,6 +2205,10 @@ func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig, fail // leaves DoH transports in a stale state. const recoveryDebounceWindow = 500 * time.Millisecond +// recoveryResolverReason names this flow in the OS resolver events, so a +// reader tells a resolver read of the recovery from every other read. +const recoveryResolverReason = "recovery" + // debounceRecovery schedules a handleRecovery(NetworkChange) call after a debounce // window. If called again before the window expires, the timer is reset so that // recovery runs once with the final network state. All other state updates (IP, @@ -2166,6 +2242,48 @@ func (p *prog) debounceRecovery(transitionID uint64) { mainLog.Load().Debug().Msg("Recovery debounce: scheduled (500ms window)") } +// endRecovery closes one recovery pass. Support reads the end event and the +// snapshot that follows it to learn what the recovery left behind. +func (p *prog) endRecovery(diagnostic *recoveryDiagnostic, outcome, targetBefore string) { + // A superseded pass changed nothing that it owns any more. Its successor + // owns the loopback target and the bypass flag, so reading them here would + // report the state of the successor. + if outcome == recoveryOutcomeSuperseded { + diagnostic.interceptTargetAction = interceptTargetActionUnchanged + diagnostic.bypassActive = false + } else { + diagnostic.interceptTargetAction = interceptTargetAction(targetBefore, p.interceptTargetSnapshot()) + diagnostic.bypassActive = p.recoveryBypass.Load() + } + diagnostic.end(outcome) + p.logNetworkSnapshot("recovery_end") + // The resolver table of the host settles after the recovery, so the poll + // stays fast until it does. + if p.dnsConfig != nil { + p.dnsConfig.noteActivity(networkEventsNowFn()) + } +} + +// The actions that one recovery takes on the loopback DNS target. +const ( + interceptTargetActionUnchanged = "unchanged" + interceptTargetActionRemoved = "removed" + interceptTargetActionSet = "set" +) + +// interceptTargetAction names what one recovery did to the loopback DNS +// target. The target value alone does not say whether this pass wrote it. +func interceptTargetAction(before, after string) string { + switch { + case before == after: + return interceptTargetActionUnchanged + case after == "": + return interceptTargetActionRemoved + default: + return interceptTargetActionSet + } +} + // handleRecovery performs a unified recovery by removing DNS settings, // canceling existing recovery checks for network changes, but coalescing duplicate // upstream failure recoveries, waiting for recovery to complete (using a cancellable context without timeout), @@ -2190,9 +2308,11 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u return } diagnostic := recoveryDiagnostic{transitionID: transitionID, generation: gen, reason: reason, started: time.Now()} - diagnostic.event(mainLog.Load().Debug()).Bool("intercept", interceptRecovery).Msg("Recovery begin") - outcome := "superseded" - defer func() { diagnostic.end(outcome) }() + journal(diagnostic.event(mainLog.Load().Info())).Bool("intercept", interceptRecovery).Msg("Recovery begin") + p.logNetworkSnapshot("recovery_begin") + targetBefore := p.interceptTargetSnapshot() + outcome := recoveryOutcomeSuperseded + defer func() { p.endRecovery(&diagnostic, outcome, targetBefore) }() if reason == RecoveryReasonNetworkChange { mainLog.Load().Debug().Msg("Network change recovery now owns shared recovery state") } @@ -2217,15 +2337,19 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u // the OS/DHCP resolver. This handles captive portal authentication // without the overhead of filter teardown/rebuild. if interceptRecovery { - mainLog.Load().Info().Msg("DNS intercept recovery: enabling DHCP bypass (filters stay active)") + journal(mainLog.Load().Info()).Msg("DNS intercept recovery: enabling DHCP bypass (filters stay active)") // Reinitialize OS resolver to discover DHCP servers on the new network. mainLog.Load().Debug().Msg("DNS intercept recovery: discovering DHCP nameservers") - dhcpServers, systemNameservers := ctrld.InitializeOsResolverWithSystemNameservers(true) - if len(dhcpServers) == 0 { - mainLog.Load().Warn().Msg("DNS intercept recovery: no DHCP nameservers found") + // The effective list adds a synthetic public resolver when the network + // gave none, so the journal reports the discovered list instead. + resolverNameservers, systemNameservers := initializeOsResolverWithSystemNameserversFn(true, recoveryResolverReason) + diagnostic.dhcpServers = systemNameservers + if len(systemNameservers) == 0 { + journal(mainLog.Load().Warn()).Msg("DNS intercept recovery: no DHCP nameservers found") } else { - mainLog.Load().Info().Msgf("DNS intercept recovery: found DHCP nameservers: %v", dhcpServers) + journal(mainLog.Load().Info()).Strs("dhcp_servers", systemNameservers). + Msgf("DNS intercept recovery: found DHCP nameservers: %v", systemNameservers) } // If the new network provides no usable IPv4 DNS (e.g. IPv6-only @@ -2236,11 +2360,12 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u // Exempt DHCP nameservers from intercept filters so the OS resolver // can actually reach them on port 53. - if len(dhcpServers) > 0 { + // The OS resolver queries the effective list, so the exemptions follow it. + if len(resolverNameservers) > 0 { // Build exemptions without an Interface — DHCP servers are not VPN-specific, // so they only generate group-scoped pf rules (ctrld process only). - exemptions := make([]vpnDNSExemption, 0, len(dhcpServers)) - for _, s := range dhcpServers { + exemptions := make([]vpnDNSExemption, 0, len(resolverNameservers)) + for _, s := range resolverNameservers { host := s if h, _, err := net.SplitHostPort(s); err == nil { host = h @@ -2259,11 +2384,11 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u // For an OS failure, reinitialize OS resolver nameservers immediately. if reason == RecoveryReasonOSFailure { mainLog.Load().Debug().Msg("OS resolver failure detected; reinitializing OS resolver nameservers") - ns := ctrld.InitializeOsResolver(true) + ns := ctrld.InitializeOsResolverWithReason(true, recoveryResolverReason) if len(ns) == 0 { mainLog.Load().Warn().Msg("No nameservers found for OS resolver; using existing values") } else { - mainLog.Load().Info().Msgf("Reinitialized OS resolver with nameservers: %v", ns) + journal(mainLog.Load().Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns) } } } @@ -2272,19 +2397,25 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u upstreams := p.buildRecoveryUpstreams(reason) // Wait indefinitely until one of the upstreams recovers. - recovered, err := p.waitForUpstreamRecovery(recoveryCtx, upstreams, &diagnostic) + recovered, err := waitForUpstreamRecoveryFn(p, recoveryCtx, upstreams, &diagnostic) if err != nil { if p.recoveryOwnsState(gen) { - outcome = "canceled" + outcome = recoveryOutcomeCanceled } p.recoveryCanceledCleanup(gen) return } + diagnostic.recoveredUpstream = recovered if !p.recoveryOwnsState(gen) { mainLog.Load().Debug().Msgf("Recovery generation %d was superseded after upstream success; skipping stale completion", gen) return } - mainLog.Load().Info().Msgf("Upstream %q recovered; re-applying DNS settings", recovered) + // The reset ends the outage, so the down time is read before it. The name + // of the line is bounded, because an upstream key is operator text. + journalName := journalUpstreamName(recovered) + journal(mainLog.Load().Info()).Str("upstream", journalName). + Int64("down_for_ms", p.um.downFor(recovered).Milliseconds()). + Msgf("Upstream %q recovered; re-applying DNS settings", journalName) // Reset the upstream failure count and down state while this generation // still owns recovery completion. @@ -2298,11 +2429,11 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u // Reinitialize OS resolver for the recovered state. if reason == RecoveryReasonNetworkChange { - ns := ctrld.InitializeOsResolver(true) + ns := ctrld.InitializeOsResolverWithReason(true, recoveryResolverReason) if len(ns) == 0 { mainLog.Load().Warn().Msg("No nameservers found for OS resolver during network-change recovery; using existing values") } else { - mainLog.Load().Info().Msgf("Reinitialized OS resolver with nameservers: %v", ns) + journal(mainLog.Load().Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns) } } } else { @@ -2314,29 +2445,33 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u // on a healthy DHCP network. systemNameservers = systemNameserversForInterceptRetry() } else if reason == RecoveryReasonNetworkChange { - ns := ctrld.InitializeOsResolver(true) + ns := ctrld.InitializeOsResolverWithReason(true, recoveryResolverReason) if len(ns) == 0 { mainLog.Load().Warn().Msg("No nameservers found for OS resolver during network-change recovery; using existing values") } else { - mainLog.Load().Info().Msgf("Reinitialized OS resolver with nameservers: %v", ns) + journal(mainLog.Load().Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns) } } - // Apply our DNS settings back and log the interface state. + // Apply our DNS settings back. The snapshot that follows the end event + // reports the interfaces. p.setDNS(systemNameservers) - p.logInterfacesState() } if !p.completeRecovery(gen) { mainLog.Load().Debug().Msgf("Recovery generation %d was superseded during completion; preserving successor state", gen) return } - outcome = "completed" + outcome = recoveryOutcomeCompleted if interceptRecovery { - mainLog.Load().Info().Msg("DNS intercept recovery complete: disabling DHCP bypass, resuming normal flow") + journal(mainLog.Load().Info()).Msg("DNS intercept recovery complete: disabling DHCP bypass, resuming normal flow") } } +// waitForUpstreamRecoveryFn is the seam of the upstream probe. A test drives +// the whole recovery flow without a query to a real upstream. +var waitForUpstreamRecoveryFn = (*prog).waitForUpstreamRecovery + // waitForUpstreamRecovery checks the provided upstreams concurrently until one recovers. // It returns the name of the recovered upstream or an error if the check times out. func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string]*ctrld.UpstreamConfig, diagnostic *recoveryDiagnostic) (string, error) { @@ -2400,11 +2535,11 @@ func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string // we should try to reinit the OS resolver to ensure we can recover if name == upstreamOS && attempts%3 == 0 { mainLog.Load().Debug().Msgf("UpstreamOS check failed on attempt %d, reinitializing OS resolver", attempts) - ns := ctrld.InitializeOsResolver(true) + ns := ctrld.InitializeOsResolverWithReason(true, recoveryResolverReason) if len(ns) == 0 { mainLog.Load().Warn().Msg("No nameservers found for OS resolver; using existing values") } else { - mainLog.Load().Info().Msgf("Reinitialized OS resolver with nameservers: %v", ns) + journal(mainLog.Load().Info()).Msgf("Reinitialized OS resolver with nameservers: %v", ns) } } } diff --git a/cmd/cli/dns_target_darwin.go b/cmd/cli/dns_target_darwin.go index e39292cd..ca8b00eb 100644 --- a/cmd/cli/dns_target_darwin.go +++ b/cmd/cli/dns_target_darwin.go @@ -52,8 +52,7 @@ func (p *prog) loadInterceptDNSTargetStateLocked() { if err := json.Unmarshal(data, &st); err != nil || st.Service == "" || st.Value == "" { return } - p.interceptDNSTargetService = st.Service - p.interceptDNSTargetSetValue = st.Value + p.setInterceptDNSTargetLocked(st.Service, st.Value) mainLog.Load().Debug().Msgf("intercept DNS target: restored tracking of %s on %q from previous run", st.Value, st.Service) } @@ -177,10 +176,11 @@ func (p *prog) ensureInterceptDNSTarget(systemDiscovery []string) { mainLog.Load().Warn().Err(err).Msgf("intercept DNS target: could not set %s on %q", target, iface.Name) return } - p.interceptDNSTargetService = iface.Name - p.interceptDNSTargetSetValue = target + p.setInterceptDNSTargetLocked(iface.Name, target) p.persistInterceptDNSTargetStateLocked() - mainLog.Load().Warn().Msgf("intercept DNS target: service %q provides no usable IPv4 DNS; set %s so macOS can emit DNS queries (removed automatically when the network provides IPv4 DNS)", iface.Name, target) + journal(mainLog.Load().Warn()).Str("service", iface.Name).Str("target", target). + Str("reason", "dns_less_network"). + Msgf("intercept DNS target: service %q provides no usable IPv4 DNS; set %s so macOS can emit DNS queries (removed automatically when the network provides IPv4 DNS)", iface.Name, target) } // removeInterceptDNSTarget removes a previously set intercept DNS target, @@ -210,8 +210,12 @@ func (p *prog) removeInterceptDNSTargetLocked(reason string) { return } if !isInterceptDNSTargetOnly(cur, val) { - mainLog.Load().Debug().Msgf("intercept DNS target: %q DNS changed externally; not removing (%s)", svc, reason) p.clearInterceptDNSTargetStateLocked() + // ctrld owns the DNS of the service no longer, and a later outage + // report needs the moment that ownership ended. + journal(mainLog.Load().Info()).Str("service", svc).Str("target", val). + Str("reason", "external_change"). + Msgf("intercept DNS target: %q DNS changed externally; not removing (%s)", svc, reason) return } if saved := interceptSavedStaticNameserversFn(iface); len(saved) > 0 { @@ -224,11 +228,20 @@ func (p *prog) removeInterceptDNSTargetLocked(reason string) { return } p.clearInterceptDNSTargetStateLocked() - mainLog.Load().Info().Msgf("intercept DNS target: removed %s from %q (%s)", val, svc, reason) + journal(mainLog.Load().Info()).Str("service", svc).Str("target", val).Str("reason", reason). + Msgf("intercept DNS target: removed %s from %q (%s)", val, svc, reason) } func (p *prog) clearInterceptDNSTargetStateLocked() { - p.interceptDNSTargetService = "" - p.interceptDNSTargetSetValue = "" + p.setInterceptDNSTargetLocked("", "") p.persistInterceptDNSTargetStateLocked() } + +// setInterceptDNSTargetLocked stores the service that ctrld owns and the value +// it wrote, and publishes the value for the readers that take no lock. Callers +// must hold interceptDNSTargetMu. +func (p *prog) setInterceptDNSTargetLocked(service, value string) { + p.interceptDNSTargetService = service + p.interceptDNSTargetSetValue = value + p.publishInterceptTarget(value) +} diff --git a/cmd/cli/dns_target_darwin_test.go b/cmd/cli/dns_target_darwin_test.go index 6f37a80a..a04f785d 100644 --- a/cmd/cli/dns_target_darwin_test.go +++ b/cmd/cli/dns_target_darwin_test.go @@ -8,6 +8,7 @@ import ( "os" "path/filepath" "slices" + "strings" "testing" "github.com/Control-D-Inc/ctrld" @@ -125,8 +126,7 @@ func persistInterceptTargetForTest(t *testing.T, p *prog, service, value string) p.interceptDNSTargetMu.Lock() defer p.interceptDNSTargetMu.Unlock() p.interceptDNSTargetLoaded = true - p.interceptDNSTargetService = service - p.interceptDNSTargetSetValue = value + p.setInterceptDNSTargetLocked(service, value) p.persistInterceptDNSTargetStateLocked() } @@ -246,3 +246,104 @@ func TestRemoveInterceptDNSTargetRetainsStateOnFailure(t *testing.T) { }) } } + +// interceptTargetEvent returns the single event whose message starts with the +// given text. The two target events name the service and the value in their +// message, so no test can match a whole message. +func interceptTargetEvent(t *testing.T, logs *syncBuffer, prefix string) map[string]any { + t.Helper() + var found []map[string]any + for _, event := range jsonLogEvents(t, logs, "") { + message, _ := event["message"].(string) + if strings.HasPrefix(message, prefix) { + found = append(found, event) + } + } + if len(found) != 1 { + t.Fatalf("events with prefix %q: got %d, want 1", prefix, len(found)) + } + return found[0] +} + +func TestEnsureInterceptDNSTargetJournalsTheSetTarget(t *testing.T) { + logs := captureDebugMainLog(t) + newInterceptTargetHarness(t) + p := newInterceptTargetProg() + + p.ensureInterceptDNSTarget([]string{}) + + event := interceptTargetEvent(t, logs, "intercept DNS target: service ") + wantField(t, event, "journal", true) + wantField(t, event, "service", "Wi-Fi") + wantField(t, event, "target", "127.0.0.53") + wantField(t, event, "reason", "dns_less_network") +} + +func TestRemoveInterceptDNSTargetJournalsTheRemovedTarget(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Wi-Fi", "127.0.0.53") + h.dns["Wi-Fi"] = []string{"127.0.0.53"} + + p.removeInterceptDNSTarget("network has usable static IPv4 DNS") + + event := interceptTargetEvent(t, logs, "intercept DNS target: removed ") + wantField(t, event, "journal", true) + wantField(t, event, "service", "Wi-Fi") + wantField(t, event, "target", "127.0.0.53") + wantField(t, event, "reason", "network has usable static IPv4 DNS") +} + +// TestInterceptDNSTargetSnapshotFollowsTheWritesWithoutTheLock covers the read +// that a network report uses. The target mutex is held across networksetup +// calls that take seconds, so a report that takes it waits for them. +func TestInterceptDNSTargetSnapshotFollowsTheWritesWithoutTheLock(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + h.dns["Wi-Fi"] = nil + + p.ensureInterceptDNSTarget([]string{}) + + p.interceptDNSTargetMu.Lock() + written := p.interceptDNSTargetSetValue + got := p.interceptTargetSnapshot() + p.interceptDNSTargetMu.Unlock() + if written == "" { + t.Fatal("the pass wrote no target") + } + if got != written { + t.Fatalf("target snapshot = %q, want %q", got, written) + } + + p.removeInterceptDNSTarget("test") + + p.interceptDNSTargetMu.Lock() + got = p.interceptTargetSnapshot() + p.interceptDNSTargetMu.Unlock() + if got != "" { + t.Fatalf("target snapshot after the removal = %q, want an empty value", got) + } +} + +// TestRemoveInterceptDNSTargetJournalsAnExternalChange covers the branch that +// drops the ownership record. The user or another program owns the DNS of the +// service from that moment, and the journal held nothing about it. +func TestRemoveInterceptDNSTargetJournalsAnExternalChange(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Wi-Fi", "127.0.0.53") + h.dns["Wi-Fi"] = []string{"9.9.9.9"} + + p.removeInterceptDNSTarget("intercept shutdown") + + event := oneRecoveryEvent(t, logs, `intercept DNS target: "Wi-Fi" DNS changed externally; not removing (intercept shutdown)`) + wantField(t, event, "journal", true) + wantField(t, event, "service", "Wi-Fi") + wantField(t, event, "target", "127.0.0.53") + wantField(t, event, "reason", "external_change") + if p.interceptTargetSnapshot() != "" { + t.Fatal("the ownership record survived an external change") + } +} diff --git a/cmd/cli/intercept_target.go b/cmd/cli/intercept_target.go new file mode 100644 index 00000000..7fd3498c --- /dev/null +++ b/cmd/cli/intercept_target.go @@ -0,0 +1,15 @@ +package cli + +// publishInterceptTarget mirrors the loopback DNS target that ctrld wrote into +// a network service. Callers hold interceptDNSTargetMu. +func (p *prog) publishInterceptTarget(target string) { + p.interceptTargetMirror.Store(target) +} + +// interceptTargetSnapshot reads the mirrored loopback DNS target. It takes no +// lock, so a report of the network never waits on the networksetup calls that +// the target mutex covers. The value is empty while ctrld holds no service. +func (p *prog) interceptTargetSnapshot() string { + target, _ := p.interceptTargetMirror.Load().(string) + return target +} diff --git a/cmd/cli/log_header.go b/cmd/cli/log_header.go index f9a58f67..317da3d1 100644 --- a/cmd/cli/log_header.go +++ b/cmd/cli/log_header.go @@ -2,7 +2,6 @@ package cli import ( "bytes" - "cmp" "net" "net/netip" "os" @@ -13,8 +12,8 @@ import ( "time" "github.com/rs/zerolog" + "github.com/Control-D-Inc/ctrld" - "tailscale.com/net/netmon" ) // processStartTime is the time this process started. A reader compares it with @@ -42,21 +41,6 @@ const ( // macOS, and a header refresh follows every network change. var headerOSVersion = sync.OnceValue(osVersion) -// logHeaderInterface is one interface of the header network snapshot. -type logHeaderInterface struct { - Name string - Up bool - IPs []string -} - -// logHeaderNetwork is the minimal network snapshot of a header line. -type logHeaderNetwork struct { - DefaultRouteInterface string - HaveV4 bool - HaveV6 bool - Interfaces []logHeaderInterface -} - // logHeaderInput holds every value of a header line. The caller collects the // values, so the renderer stays free of locks and of system calls. type logHeaderInput struct { @@ -72,7 +56,7 @@ type logHeaderInput struct { UpstreamTypes []string ResolverUID string LogFiles []string - Network logHeaderNetwork + Network networkSnapshot Trigger string } @@ -95,7 +79,7 @@ func renderLogHeader(in logHeaderInput) []byte { Strs("upstream_types", in.UpstreamTypes). Str("resolver_uid", in.ResolverUID). Strs("log_files", in.LogFiles). - Dict("network", logHeaderNetworkDict(in.Network)) + Dict("network", snapshotDict(in.Network)) if in.Trigger != "" { event = event.Str("trigger", in.Trigger) } @@ -113,86 +97,13 @@ func logHeaderEvent(logger *zerolog.Logger) *zerolog.Event { return logger.Log().Str(zerolog.LevelFieldName, zerolog.LevelInfoValue) } -func logHeaderNetworkDict(network logHeaderNetwork) *zerolog.Event { - interfaces := zerolog.Arr() - for _, iface := range network.Interfaces { - interfaces = interfaces.Dict(zerolog.Dict(). - Str("name", iface.Name). - Bool("up", iface.Up). - Strs("ips", iface.IPs)) - } - return zerolog.Dict(). - Str("default_route_interface", network.DefaultRouteInterface). - Bool("have_v4", network.HaveV4). - Bool("have_v6", network.HaveV6). - Array("interfaces", interfaces) -} - -// logHeaderNetworkFromState makes the header snapshot from a netmon state. -func logHeaderNetworkFromState(st *netmon.State) logHeaderNetwork { - if st == nil { - return logHeaderNetwork{} - } - network := logHeaderNetwork{ - DefaultRouteInterface: st.DefaultRouteInterface, - HaveV4: st.HaveV4, - HaveV6: st.HaveV6, - } - for name, iface := range st.Interface { - network.Interfaces = append(network.Interfaces, logHeaderInterface{ - Name: name, - Up: iface.Interface != nil && iface.IsUp(), - IPs: prefixStrings(st.InterfaceIPs[name]), - }) - } - slices.SortFunc(network.Interfaces, func(a, b logHeaderInterface) int { - return cmp.Compare(a.Name, b.Name) - }) - return network -} - -// logHeaderNetworkFresh reads the network now. A header that no network -// callback reached yet, and the header of an upload, need the live picture. -func logHeaderNetworkFresh() logHeaderNetwork { - state, err := readNetworkSourceStateFn() - if err != nil { - state = nil - } - network := logHeaderNetworkFromState(state) - if route, err := netmon.DefaultRouteInterface(); err == nil { - network.DefaultRouteInterface = route - } - network.HaveV4, network.HaveV6 = globalAddressFamilies(state) - return network -} - -// globalAddressFamilies reports which families the host can reach. A loopback -// or a link-local address reaches no other host, so it proves nothing. -func globalAddressFamilies(st *netmon.State) (haveV4, haveV6 bool) { - if st == nil { - return false, false - } - for name, iface := range st.Interface { - if iface.Interface == nil || !iface.IsUp() { - continue - } - for _, prefix := range st.InterfaceIPs[name] { - addr := prefix.Addr().Unmap() - if !addr.IsGlobalUnicast() || addr.IsLoopback() || addr.IsLinkLocalUnicast() { - continue - } - if addr.Is4() { - haveV4 = true - continue - } - haveV6 = true - } - } - return haveV4, haveV6 -} - // logHeaderInput collects every header value of the running program. func (p *prog) logHeaderInput() logHeaderInput { + return p.logHeaderInputWith(p.networkSnapshot()) +} + +// logHeaderInputWith collects every header value around one network snapshot. +func (p *prog) logHeaderInputWith(network networkSnapshot) logHeaderInput { in := logHeaderInput{ Version: curVersion(), Commit: commit, @@ -204,7 +115,7 @@ func (p *prog) logHeaderInput() logHeaderInput { // every artifact, so the header keeps its first characters only. ResolverUID: redactToken(cdUID), LogFiles: p.logHeaderFiles(), - Network: p.logHeaderNetwork(), + Network: network, } p.headerConfigInto(&in) return in @@ -218,35 +129,15 @@ func (p *prog) headerConfigInto(in *logHeaderInput) { logHeaderFromConfig(p.cfg, in) } -// sendLogHeader renders the header that leads one upload. +// sendLogHeader renders the header that leads one upload. The stored state can +// be older than the request, and support reads this line to learn the network +// of the moment the user asked. func (p *prog) sendLogHeader() []byte { - in := p.logHeaderInput() - // The stored state can be older than the request, and support reads this - // line to learn the network of the moment the user asked. - in.Network = logHeaderNetworkFresh() + in := p.logHeaderInputWith(buildNetworkSnapshot(p.freshSnapshotInputs())) in.Trigger = logHeaderTriggerSend return renderLogHeader(in) } -// noteNetworkState keeps the newest network that ctrld saw and puts it in the -// header bytes of every open log file. The caller must hold no lock, because -// the render reads the config and the network. -func (p *prog) noteNetworkState(st *netmon.State) { - if st != nil { - p.lastNetworkState.Store(st) - } - p.refreshLogHeader() -} - -// logHeaderNetwork takes the network that the last monitor callback reported. -func (p *prog) logHeaderNetwork() logHeaderNetwork { - state := p.lastNetworkState.Load() - if state == nil { - return logHeaderNetworkFresh() - } - return logHeaderNetworkFromState(state) -} - // logHeaderFiles names the files that this run writes, debug first. func (p *prog) logHeaderFiles() []string { files := make([]string, 0, 3) diff --git a/cmd/cli/log_header_test.go b/cmd/cli/log_header_test.go index 48d49680..867962df 100644 --- a/cmd/cli/log_header_test.go +++ b/cmd/cli/log_header_test.go @@ -40,14 +40,26 @@ type logHeaderTestLine struct { ResolverUID string `json:"resolver_uid"` LogFiles []string `json:"log_files"` Network struct { - DefaultRouteInterface string `json:"default_route_interface"` - HaveV4 bool `json:"have_v4"` - HaveV6 bool `json:"have_v6"` - Interfaces []struct { - Name string `json:"name"` - Up bool `json:"up"` - IPs []string `json:"ips"` + DefaultRouteV4 string `json:"default_route_v4"` + DefaultRouteV6 string `json:"default_route_v6"` + GatewayV4 string `json:"gateway_v4"` + GatewayV6 string `json:"gateway_v6"` + HaveV4 bool `json:"have_v4"` + HaveV6 bool `json:"have_v6"` + Interfaces []struct { + Name string `json:"name"` + Class string `json:"class"` + Up bool `json:"up"` + IPs []string `json:"ips"` + HardwarePort string `json:"hardware_port"` + Service string `json:"service"` } `json:"interfaces"` + Resolvers []string `json:"resolvers"` + SourceIPv4 string `json:"source_ipv4"` + SourceIPv6 string `json:"source_ipv6"` + InterceptTarget string `json:"intercept_target"` + LinkType string `json:"link_type"` + Tethered bool `json:"tethered"` } `json:"network"` Trigger string `json:"trigger"` } @@ -66,12 +78,15 @@ func logHeaderTestInput() logHeaderInput { UpstreamTypes: []string{"doh", "os"}, ResolverUID: redactToken("abcd1234"), LogFiles: []string{"/tmp/ctrld.log", "/tmp/ctrld-journal.log"}, - Network: logHeaderNetwork{ - DefaultRouteInterface: "en0", - HaveV4: true, - Interfaces: []logHeaderInterface{ - {Name: "en0", Up: true, IPs: []string{"192.0.2.10/24"}}, + Network: networkSnapshot{ + DefaultRouteV4: "en0", + GatewayV4: "192.0.2.1", + HaveV4: true, + Interfaces: []snapshotInterface{ + {Name: "en0", Class: "hardware", Up: true, IPs: []string{"192.0.2.10/24"}, HardwarePort: "Wi-Fi", Service: "Wi-Fi"}, }, + Resolvers: []string{"192.0.2.1:53"}, + LinkType: "wifi", }, } } @@ -135,9 +150,14 @@ func Test_renderLogHeaderHoldsEveryField(t *testing.T) { // characters only. {"resolver_uid", parsed.ResolverUID, "abcd***"}, {"log_files", parsed.LogFiles, in.LogFiles}, - {"network.default_route_interface", parsed.Network.DefaultRouteInterface, in.Network.DefaultRouteInterface}, + {"network.default_route_v4", parsed.Network.DefaultRouteV4, in.Network.DefaultRouteV4}, + {"network.default_route_v6", parsed.Network.DefaultRouteV6, in.Network.DefaultRouteV6}, + {"network.gateway_v4", parsed.Network.GatewayV4, in.Network.GatewayV4}, {"network.have_v4", parsed.Network.HaveV4, in.Network.HaveV4}, {"network.have_v6", parsed.Network.HaveV6, in.Network.HaveV6}, + {"network.resolvers", parsed.Network.Resolvers, in.Network.Resolvers}, + {"network.link_type", parsed.Network.LinkType, in.Network.LinkType}, + {"network.tethered", parsed.Network.Tethered, in.Network.Tethered}, } for _, c := range checks { if !reflect.DeepEqual(c.got, c.want) { @@ -149,8 +169,11 @@ func Test_renderLogHeaderHoldsEveryField(t *testing.T) { t.Fatalf("network.interfaces holds %d entries, want 1", len(parsed.Network.Interfaces)) } iface := parsed.Network.Interfaces[0] - if iface.Name != "en0" || !iface.Up || !reflect.DeepEqual(iface.IPs, []string{"192.0.2.10/24"}) { - t.Errorf("network.interfaces[0] = %+v, want en0 up with 192.0.2.10/24", iface) + if iface.Name != "en0" || iface.Class != "hardware" || !iface.Up || iface.HardwarePort != "Wi-Fi" { + t.Errorf("network.interfaces[0] = %+v, want the hardware port en0 up on Wi-Fi", iface) + } + if !reflect.DeepEqual(iface.IPs, []string{"192.0.2.10/24"}) { + t.Errorf("network.interfaces[0].ips = %v, want 192.0.2.10/24", iface.IPs) } if parsed.Time == "" { t.Error("time is empty") @@ -245,8 +268,12 @@ func Test_logHeaderFromConfigReportsOffWhenUnset(t *testing.T) { } } -func Test_logHeaderNetworkFromState(t *testing.T) { +// Test_renderLogHeaderShowsTheSnapshotOfAState renders the header of one +// network state, so a reader of the file sees every interface of the host in +// one order. +func Test_renderLogHeaderShowsTheSnapshotOfAState(t *testing.T) { setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + stubHeaderSnapshotSources(t) state := &netmon.State{ DefaultRouteInterface: "en0", HaveV4: true, @@ -255,52 +282,40 @@ func Test_logHeaderNetworkFromState(t *testing.T) { "awdl0": {Interface: &net.Interface{Name: "awdl0"}}, }, InterfaceIPs: map[string][]netip.Prefix{ - "en0": {netip.MustParsePrefix("192.0.2.10/24"), netip.MustParsePrefix("2001:db8::1/64")}, + "en0": {netip.MustParsePrefix("192.0.2.10/24"), netip.MustParsePrefix("2001:db8::1/64")}, + "awdl0": {netip.MustParsePrefix("fe80::2/64")}, }, } - network := logHeaderNetworkFromState(state) - - want := logHeaderNetwork{ - DefaultRouteInterface: "en0", - HaveV4: true, - Interfaces: []logHeaderInterface{ - {Name: "awdl0", Up: false, IPs: []string{}}, - {Name: "en0", Up: true, IPs: []string{"192.0.2.10/24", "2001:db8::1/64"}}, - }, - } - if !reflect.DeepEqual(network, want) { - t.Fatalf("network = %+v, want %+v", network, want) - } - in := logHeaderTestInput() - in.Network = network + in.Network = buildNetworkSnapshot(snapshotInputs{ + State: state, + RouteV4: defaultRoute{Gateway: "192.0.2.1", Interface: "en0"}, + RouteV6: defaultRoute{Gateway: "fe80::1%en0", Interface: "en0"}, + Meta: interfaceMetaFor, + }) parsed, _ := parseLogHeader(t, renderLogHeader(in)) - if parsed.Network.DefaultRouteInterface != "en0" { - t.Errorf("network.default_route_interface = %q, want %q", parsed.Network.DefaultRouteInterface, "en0") + + if parsed.Network.DefaultRouteV4 != "en0" || parsed.Network.DefaultRouteV6 != "en0" { + t.Errorf("default route = %q, %q, want en0 for both families", parsed.Network.DefaultRouteV4, parsed.Network.DefaultRouteV6) } if len(parsed.Network.Interfaces) != 2 { t.Fatalf("network.interfaces holds %d entries, want 2", len(parsed.Network.Interfaces)) } - if parsed.Network.Interfaces[0].Name != "awdl0" || parsed.Network.Interfaces[0].Up { - t.Errorf("network.interfaces[0] = %+v, want awdl0 down", parsed.Network.Interfaces[0]) + if parsed.Network.Interfaces[0].Name != "awdl0" || parsed.Network.Interfaces[0].Class != "airdrop" || parsed.Network.Interfaces[0].Up { + t.Errorf("network.interfaces[0] = %+v, want awdl0 down in the airdrop class", parsed.Network.Interfaces[0]) } if got := parsed.Network.Interfaces[1].IPs; !reflect.DeepEqual(got, []string{"192.0.2.10/24", "2001:db8::1/64"}) { t.Errorf("network.interfaces[1].ips = %v", got) } } -func Test_logHeaderNetworkFromStateWithoutState(t *testing.T) { - if network := logHeaderNetworkFromState(nil); !reflect.DeepEqual(network, logHeaderNetwork{}) { - t.Errorf("network = %+v, want the zero value", network) - } -} - // setupLogHeaderWiringTest puts ctrld in cd mode with a temporary home // directory, so initInternalLogging writes its files there. Every global that // the logging setup touches goes back when the test ends. func setupLogHeaderWiringTest(t *testing.T) string { t.Helper() + stubHeaderSnapshotSources(t) origSilent, origCdUID, origHomedir, origVerbose := silent, cdUID, homedir, verbose origMainLog, origProxyLogger := mainLog.Load(), ctrld.ProxyLogger.Load() origLevel := zerolog.GlobalLevel() @@ -492,14 +507,22 @@ func Test_logHeaderWiring_restartAppendsHeader(t *testing.T) { func Test_logHeaderWiring_usesLastNetworkState(t *testing.T) { setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + stubHeaderSnapshotSources(t) origRead := readNetworkSourceStateFn t.Cleanup(func() { readNetworkSourceStateFn = origRead }) p := &prog{cfg: &ctrld.Config{}} - p.lastNetworkState.Store(&netmon.State{DefaultRouteInterface: "en7"}) + p.lastNetworkState.Store(&netmon.State{ + DefaultRouteInterface: "en7", + Interface: map[string]netmon.Interface{"en7": {Interface: &net.Interface{Name: "en7", Flags: net.FlagUp}}}, + InterfaceIPs: map[string][]netip.Prefix{"en7": {netip.MustParsePrefix("192.0.2.7/24")}}, + }) - if got := p.logHeaderInput().Network.DefaultRouteInterface; got != "en7" { - t.Errorf("default_route_interface = %q, want %q", got, "en7") + if got := p.logHeaderInput().Network.DefaultRouteV4; got != "en7" { + t.Errorf("default_route_v4 = %q, want %q", got, "en7") + } + if got := p.logHeaderInput().Network.GatewayV4; got != "192.0.2.1" { + t.Errorf("gateway_v4 = %q, want %q", got, "192.0.2.1") } p.lastNetworkState.Store(nil) @@ -522,10 +545,22 @@ func Test_logHeaderWiring_usesLastNetworkState(t *testing.T) { } } +// stubHeaderSnapshotSources keeps the header snapshot away from the host. One +// render reads the hardware ports, the route table, and the NAT64 class, and no +// test may run the commands behind them. +func stubHeaderSnapshotSources(t *testing.T) { + t.Helper() + stubSnapshotDNS64(t) + stubSnapshotGateways(t, "192.0.2.1", "") + stubSnapshotPlatformMeta(t, nil) + stubSnapshotVirtualSet(t) +} + // stubLogHeaderNetworkRead makes the fresh network read return one up // interface, so a header test never depends on the host network. func stubLogHeaderNetworkRead(t *testing.T, name string) { t.Helper() + stubHeaderSnapshotSources(t) origRead := readNetworkSourceStateFn t.Cleanup(func() { readNetworkSourceStateFn = origRead }) readNetworkSourceStateFn = func() (*netmon.State, error) { @@ -538,6 +573,7 @@ func stubLogHeaderNetworkRead(t *testing.T, name string) { func Test_sendLogHeader_usesFreshNetworkAndTrigger(t *testing.T) { setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + stubHeaderSnapshotSources(t) stubLogHeaderNetworkRead(t, "en9") p := &prog{cfg: &ctrld.Config{}} p.lastNetworkState.Store(&netmon.State{ diff --git a/cmd/cli/log_journal.go b/cmd/cli/log_journal.go index 0f851be8..995de485 100644 --- a/cmd/cli/log_journal.go +++ b/cmd/cli/log_journal.go @@ -6,10 +6,12 @@ import ( "regexp" "github.com/rs/zerolog" + "github.com/Control-D-Inc/ctrld" ) -// journalField marks an event for the retained journal stream. -const journalField = "journal" +// journalField marks an event for the retained journal stream. Both packages +// share the name, so one writer keeps the lines of both. +const journalField = ctrld.JournalField // journalMarker is the serialized form of the journal field. The journal // writer keeps a line below warn level only when the line contains it. @@ -30,7 +32,7 @@ func redactRetainedLine(p []byte) []byte { // journal marks an event for the retained journal stream. The event keeps // its own level, so an info event stays info on the console. func journal(e *zerolog.Event) *zerolog.Event { - return e.Bool(journalField, true) + return ctrld.Journal(e) } // journalLevelWriter passes on the lines that the journal keeps and drops the diff --git a/cmd/cli/log_journal_test.go b/cmd/cli/log_journal_test.go index ff6fd499..abd8f7e3 100644 --- a/cmd/cli/log_journal_test.go +++ b/cmd/cli/log_journal_test.go @@ -195,6 +195,7 @@ func startInternalLogging(t *testing.T) (*prog, string) { cdUID = "test-uid" silent = false verbose = 0 + stubHeaderSnapshotSources(t) p := &prog{cfg: &ctrld.Config{}} p.initInternalLogging(nil) diff --git a/cmd/cli/log_writer_silent_test.go b/cmd/cli/log_writer_silent_test.go index 70eddb39..15bca701 100644 --- a/cmd/cli/log_writer_silent_test.go +++ b/cmd/cli/log_writer_silent_test.go @@ -59,6 +59,7 @@ func Test_initInternalLogging_silentCreatesNoFile(t *testing.T) { homedir = dir cdUID = "test-uid" // cd mode, which would otherwise enable internal logging silent = true + stubHeaderSnapshotSources(t) p := &prog{cfg: &ctrld.Config{}} p.initInternalLogging(nil) @@ -94,6 +95,7 @@ func Test_initInternalLogging_createsJournalFile(t *testing.T) { cdUID = "test-uid" silent = false verbose = 0 + stubHeaderSnapshotSources(t) p := &prog{cfg: &ctrld.Config{}} p.initInternalLogging(nil) diff --git a/cmd/cli/net_darwin.go b/cmd/cli/net_darwin.go index 62331610..8d9c4abf 100644 --- a/cmd/cli/net_darwin.go +++ b/cmd/cli/net_darwin.go @@ -3,14 +3,52 @@ package cli import ( "bufio" "bytes" + "context" + "errors" "io" "net" "os/exec" "strings" + "sync" + "time" +) + +// hardwarePortInfo holds the two names that macOS gives a network device: the +// hardware port of the adapter and the network service that uses it. +type hardwarePortInfo struct { + Port string + Service string +} + +// A network event reads the ports of every interface, so the two networksetup +// calls run at most once per window. +const hardwarePortsTTL = time.Minute + +// networksetupTimeout bounds one command. A stuck read must not hold the +// network callback that waits for the port names. +const networksetupTimeout = 5 * time.Second + +// maxNetworksetupOutput bounds one read. A host with many ports stays far +// below it, so more than this is a broken command. +const maxNetworksetupOutput = 1 << 20 + +// maxNetworksetupLine bounds one line, so one token fills no memory. +const maxNetworksetupLine = 64 * 1024 + +var errNetworksetupOutputTooLarge = errors.New("networksetup output above the bound") + +var ( + hardwarePortsMu sync.Mutex + hardwarePortsCache map[string]hardwarePortInfo + hardwarePortsReadAt time.Time + + // Tests replace these seams to avoid a networksetup call. + readHardwarePortsFn = readHardwarePorts + hardwarePortsNowFn = time.Now ) func patchNetIfaceName(iface *net.Interface) (bool, error) { - b, err := exec.Command("networksetup", "-listnetworkserviceorder").Output() + b, err := networksetupOutput("-listnetworkserviceorder") if err != nil { return false, err } @@ -24,7 +62,14 @@ func patchNetIfaceName(iface *net.Interface) (bool, error) { } func networkServiceName(ifaceName string, r io.Reader) string { - scanner := bufio.NewScanner(r) + return serviceNamesByDevice(r)[ifaceName] +} + +// serviceNamesByDevice maps each device of "networksetup +// -listnetworkserviceorder" to the name of the service that owns it. +func serviceNamesByDevice(r io.Reader) map[string]string { + names := make(map[string]string) + scanner := scanNetworksetup(r) prevLine := "" for scanner.Scan() { line := scanner.Text() @@ -32,45 +77,216 @@ func networkServiceName(ifaceName string, r io.Reader) string { // Network services is disabled. continue } - if !strings.Contains(line, "Device: "+ifaceName) { + device := serviceOrderDevice(line) + if device == "" { prevLine = line continue } - parts := strings.SplitN(prevLine, " ", 2) - if len(parts) == 2 { - return strings.TrimSpace(parts[1]) + if _, seen := names[device]; !seen { + names[device] = serviceOrderName(prevLine) } + prevLine = line + } + if scanner.Err() != nil { + // A truncated read names fewer services than the host has. + return nil } - return "" + return names } -// validInterface reports whether the *net.Interface is a valid one. -func validInterface(iface *net.Interface, validIfacesMap map[string]struct{}) bool { - _, ok := validIfacesMap[iface.Name] - return ok +// scanNetworksetup reads the output line by line under a bounded buffer. +func scanNetworksetup(r io.Reader) *bufio.Scanner { + scanner := bufio.NewScanner(r) + scanner.Buffer(nil, maxNetworksetupLine) + return scanner } -// validInterfacesMap returns a set of all valid hardware ports. -func validInterfacesMap() map[string]struct{} { - b, err := exec.Command("networksetup", "-listallhardwareports").Output() - if err != nil { - return nil +// serviceOrderDevice reads the device out of a line like +// "(Hardware Port: Wi-Fi, Device: en0)". +func serviceOrderDevice(line string) string { + _, after, found := strings.Cut(line, "Device:") + if !found { + return "" } - return parseListAllHardwarePorts(bytes.NewReader(b)) + return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(after), ")")) } -// parseListAllHardwarePorts parses output of "networksetup -listallhardwareports" -// and returns map presents all hardware ports. -func parseListAllHardwarePorts(r io.Reader) map[string]struct{} { - m := make(map[string]struct{}) - scanner := bufio.NewScanner(r) +// serviceOrderName reads the service name out of a line like "(1) Wi-Fi". +func serviceOrderName(line string) string { + parts := strings.SplitN(line, " ", 2) + if len(parts) != 2 { + return "" + } + return strings.TrimSpace(parts[1]) +} + +// parseHardwarePorts maps each device of "networksetup -listallhardwareports" +// to its hardware port. Each block names the port before the device. +func parseHardwarePorts(r io.Reader) map[string]hardwarePortInfo { + ports := make(map[string]hardwarePortInfo) + scanner := scanNetworksetup(r) + port := "" for scanner.Scan() { - line := scanner.Text() - after, ok := strings.CutPrefix(line, "Device: ") + line := strings.TrimSpace(scanner.Text()) + if after, ok := strings.CutPrefix(line, "Hardware Port:"); ok { + port = strings.TrimSpace(after) + continue + } + after, ok := strings.CutPrefix(line, "Device:") if !ok { continue } - m[after] = struct{}{} + device := strings.TrimSpace(after) + if device == "" { + continue + } + ports[device] = hardwarePortInfo{Port: port} + port = "" + } + if scanner.Err() != nil { + // A truncated read names fewer ports than the host has. + return nil + } + return ports +} + +// addServiceNames fills the service of every known device. A device that no +// hardware port lists is not a port of this host. +func addServiceNames(m map[string]hardwarePortInfo, r io.Reader) { + for device, service := range serviceNamesByDevice(r) { + info, known := m[device] + if !known { + continue + } + info.Service = service + m[device] = info + } +} + +// hardwarePortsByDevice returns the port and service of every device. The +// commands run outside the mutex, so a slow read blocks no other caller. +func hardwarePortsByDevice() map[string]hardwarePortInfo { + if ports, fresh := cachedHardwarePorts(); fresh { + return ports + } + return storeHardwarePorts(readHardwarePortsFn()) +} + +// cachedHardwarePorts reports whether the read window still holds. +func cachedHardwarePorts() (map[string]hardwarePortInfo, bool) { + hardwarePortsMu.Lock() + defer hardwarePortsMu.Unlock() + if hardwarePortsReadAt.IsZero() { + return nil, false + } + return hardwarePortsCache, hardwarePortsNowFn().Sub(hardwarePortsReadAt) < hardwarePortsTTL +} + +// storeHardwarePorts starts the window again and keeps the last good map. A +// failing command must run once per window, not once per lookup. +func storeHardwarePorts(ports map[string]hardwarePortInfo) map[string]hardwarePortInfo { + hardwarePortsMu.Lock() + defer hardwarePortsMu.Unlock() + hardwarePortsReadAt = hardwarePortsNowFn() + if ports != nil { + hardwarePortsCache = ports + } + return hardwarePortsCache +} + +// readHardwarePorts runs the two networksetup commands. +func readHardwarePorts() map[string]hardwarePortInfo { + b, err := networksetupOutput("-listallhardwareports") + if err != nil { + return nil + } + ports := parseHardwarePorts(bytes.NewReader(b)) + if ports == nil { + // A truncated list must keep the last good map. + return nil + } + b, err = networksetupOutput("-listnetworkserviceorder") + if err != nil { + return ports + } + addServiceNames(ports, bytes.NewReader(b)) + return ports +} + +// networksetupOutput runs one networksetup list under a time bound and a size +// bound. +func networksetupOutput(arg string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), networksetupTimeout) + defer cancel() + cmd := exec.CommandContext(ctx, "networksetup", arg) + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + if err := cmd.Start(); err != nil { + return nil, err + } + out, readErr := readBoundedOutput(stdout) + if readErr != nil { + // A command ends only when something reads the rest of its output. + _, _ = io.Copy(io.Discard, stdout) + } + if err := cmd.Wait(); err != nil { + return nil, err + } + return out, readErr +} + +// readBoundedOutput reads at most maxNetworksetupOutput bytes and reports an +// error above that. +func readBoundedOutput(r io.Reader) ([]byte, error) { + out, err := io.ReadAll(io.LimitReader(r, maxNetworksetupOutput+1)) + if err != nil { + return nil, err + } + if len(out) > maxNetworksetupOutput { + return nil, errNetworksetupOutputTooLarge + } + return out, nil +} + +// platformInterfaceMeta returns the macOS names of an interface. The class +// comes from the interface name, so it stays empty here. +func platformInterfaceMeta(name string) (class, hardwarePort, service string) { + info := hardwarePortsByDevice()[name] + return "", info.Port, info.Service +} + +// platformVirtualInterfaces returns no set. The name rules already class the +// AirDrop and the link-local interfaces of macOS. +func platformVirtualInterfaces() map[string]struct{} { + return nil +} + +// refreshInterfaceMeta drops the read window. The cache has no port name for +// an adapter that appeared, so the next lookup reads the ports again. +func refreshInterfaceMeta() { + hardwarePortsMu.Lock() + defer hardwarePortsMu.Unlock() + hardwarePortsReadAt = time.Time{} +} + +// validInterface reports whether the *net.Interface is a valid one. +func validInterface(iface *net.Interface, validIfacesMap map[string]struct{}) bool { + _, ok := validIfacesMap[iface.Name] + return ok +} + +// validInterfacesMap returns a set of all valid hardware ports. The devices +// come from the port cache, so a network event starts no second command. +func validInterfacesMap() map[string]struct{} { + ports := hardwarePortsByDevice() + if ports == nil { + return nil + } + devices := make(map[string]struct{}, len(ports)) + for device := range ports { + devices[device] = struct{}{} } - return m + return devices } diff --git a/cmd/cli/net_darwin_test.go b/cmd/cli/net_darwin_test.go new file mode 100644 index 00000000..280e4d50 --- /dev/null +++ b/cmd/cli/net_darwin_test.go @@ -0,0 +1,315 @@ +package cli + +import ( + "maps" + "strings" + "testing" + "time" +) + +const listAllHardwarePortsFixture = ` +Hardware Port: Wi-Fi +Device: en0 +Ethernet Address: a0:78:17:68:56:3f + +Hardware Port: iPhone USB +Device: en7 +Ethernet Address: 6e:5a:6b:12:34:56 + +Hardware Port: Thunderbolt Bridge +Device: bridge0 +Ethernet Address: 36:21:bb:3a:7a:40 + +Hardware Port: Ethernet Adapter (en6) +Device: en6 +Ethernet Address: 3a:3e:fc:1e:ab:41 + +Hardware Port: com.wireguard.macos +Device: + +VLAN Configurations +=================== +` + +const listNetworkServiceOrderFixture = `An asterisk (*) denotes that a network service is disabled. +(1) Wi-Fi +(Hardware Port: Wi-Fi, Device: en0) + +(2) iPhone USB +(Hardware Port: iPhone USB, Device: en7) + +(3) Thunderbolt Bridge +(Hardware Port: Thunderbolt Bridge, Device: bridge0) + +(*) Ethernet Adapter (en6) +(Hardware Port: Ethernet Adapter (en6), Device: en6) + +(4) ca-001-stg +(Hardware Port: com.wireguard.macos, Device: ) + +` + +func Test_parseHardwarePorts(t *testing.T) { + ports := parseHardwarePorts(strings.NewReader(listAllHardwarePortsFixture)) + addServiceNames(ports, strings.NewReader(listNetworkServiceOrderFixture)) + + tests := []struct { + device string + wantPort string + wantService string + }{ + {"en0", "Wi-Fi", "Wi-Fi"}, + {"en7", "iPhone USB", "iPhone USB"}, + {"bridge0", "Thunderbolt Bridge", "Thunderbolt Bridge"}, + {"en6", "Ethernet Adapter (en6)", ""}, + } + if len(ports) != len(tests) { + t.Fatalf("parseHardwarePorts returned %d devices, want %d: %v", len(ports), len(tests), ports) + } + for _, tc := range tests { + t.Run(tc.device, func(t *testing.T) { + info := ports[tc.device] + if info.Port != tc.wantPort { + t.Errorf("port of %s = %q, want %q", tc.device, info.Port, tc.wantPort) + } + if info.Service != tc.wantService { + t.Errorf("service of %s = %q, want %q", tc.device, info.Service, tc.wantService) + } + }) + } +} + +func Test_networkServiceName(t *testing.T) { + tests := []struct { + device string + want string + }{ + {"en0", "Wi-Fi"}, + {"en7", "iPhone USB"}, + {"bridge0", "Thunderbolt Bridge"}, + {"en6", ""}, + {"en9", ""}, + } + for _, tc := range tests { + t.Run(tc.device, func(t *testing.T) { + got := networkServiceName(tc.device, strings.NewReader(listNetworkServiceOrderFixture)) + if got != tc.want { + t.Errorf("networkServiceName(%q) = %q, want %q", tc.device, got, tc.want) + } + }) + } +} + +func Test_parseHardwarePorts_dropsATruncatedRead(t *testing.T) { + longLine := strings.Repeat("x", maxNetworksetupLine+1) + + if got := parseHardwarePorts(strings.NewReader(listAllHardwarePortsFixture + longLine)); got != nil { + t.Errorf("parseHardwarePorts of a truncated read = %v, want nil", got) + } + if got := serviceNamesByDevice(strings.NewReader(listNetworkServiceOrderFixture + longLine)); got != nil { + t.Errorf("serviceNamesByDevice of a truncated read = %v, want nil", got) + } +} + +func Test_readBoundedOutput(t *testing.T) { + out, err := readBoundedOutput(strings.NewReader(listAllHardwarePortsFixture)) + if err != nil || string(out) != listAllHardwarePortsFixture { + t.Fatalf("readBoundedOutput of the fixture = %q, %v, want the fixture and no error", out, err) + } + if _, err := readBoundedOutput(strings.NewReader(strings.Repeat("x", maxNetworksetupOutput+1))); err == nil { + t.Fatal("readBoundedOutput of an output above the bound gave no error") + } +} + +// stubHardwarePorts replaces the command and clock seams of the hardware port +// cache, and restores them with the cache itself. +func stubHardwarePorts(t *testing.T, read func() map[string]hardwarePortInfo, now func() time.Time) { + t.Helper() + readBefore, nowBefore := readHardwarePortsFn, hardwarePortsNowFn + hardwarePortsMu.Lock() + cacheBefore, readAtBefore := hardwarePortsCache, hardwarePortsReadAt + hardwarePortsCache, hardwarePortsReadAt = nil, time.Time{} + hardwarePortsMu.Unlock() + t.Cleanup(func() { + readHardwarePortsFn, hardwarePortsNowFn = readBefore, nowBefore + hardwarePortsMu.Lock() + hardwarePortsCache, hardwarePortsReadAt = cacheBefore, readAtBefore + hardwarePortsMu.Unlock() + }) + readHardwarePortsFn, hardwarePortsNowFn = read, now +} + +func Test_hardwarePortsByDevice_caches(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return map[string]hardwarePortInfo{"en0": {Port: "Wi-Fi", Service: "Wi-Fi"}} + }, func() time.Time { return now }) + + if got := hardwarePortsByDevice()["en0"].Port; got != "Wi-Fi" { + t.Fatalf("first read port = %q, want Wi-Fi", got) + } + now = now.Add(59 * time.Second) + hardwarePortsByDevice() + if reads != 1 { + t.Fatalf("reads within the cache window = %d, want 1", reads) + } + now = now.Add(2 * time.Second) + hardwarePortsByDevice() + if reads != 2 { + t.Fatalf("reads after the cache window = %d, want 2", reads) + } +} + +func Test_hardwarePortsByDevice_keepsLastGoodRead(t *testing.T) { + now := time.Unix(1700000000, 0) + fail := false + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + if fail { + return nil + } + return map[string]hardwarePortInfo{"en0": {Port: "Wi-Fi"}} + }, func() time.Time { return now }) + + hardwarePortsByDevice() + fail = true + now = now.Add(2 * time.Minute) + if got := hardwarePortsByDevice()["en0"].Port; got != "Wi-Fi" { + t.Fatalf("port after a failed read = %q, want Wi-Fi", got) + } +} + +func Test_hardwarePortsByDevice_keepsTheWindowAfterAFailedRead(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return nil + }, func() time.Time { return now }) + + hardwarePortsByDevice() + now = now.Add(time.Second) + hardwarePortsByDevice() + if reads != 1 { + t.Fatalf("reads of a failing command within the window = %d, want 1", reads) + } + now = now.Add(2 * time.Minute) + hardwarePortsByDevice() + if reads != 2 { + t.Fatalf("reads after the window = %d, want 2", reads) + } +} + +func Test_refreshInterfaceMeta_dropsTheReadWindow(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return map[string]hardwarePortInfo{"en0": {Port: "Wi-Fi"}} + }, func() time.Time { return now }) + + hardwarePortsByDevice() + hardwarePortsByDevice() + if reads != 1 { + t.Fatalf("reads within the window = %d, want 1", reads) + } + refreshInterfaceMeta() + if got := hardwarePortsByDevice()["en0"].Port; got != "Wi-Fi" { + t.Fatalf("port after a refresh = %q, want Wi-Fi", got) + } + if reads != 2 { + t.Fatalf("reads after a refresh = %d, want 2", reads) + } +} + +func Test_hardwarePortsByDevice_readsOutsideTheLock(t *testing.T) { + now := time.Unix(1700000000, 0) + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + // A command runs outside the mutex, so this read can take it. + if !hardwarePortsMu.TryLock() { + t.Error("the command ran under the hardware ports mutex") + } else { + hardwarePortsMu.Unlock() + } + return map[string]hardwarePortInfo{"en0": {Port: "Wi-Fi"}} + }, func() time.Time { return now }) + + if got := hardwarePortsByDevice()["en0"].Port; got != "Wi-Fi" { + t.Fatalf("port = %q, want Wi-Fi", got) + } +} + +// fixtureHardwarePorts returns the port map of the two command fixtures. +func fixtureHardwarePorts() map[string]hardwarePortInfo { + ports := parseHardwarePorts(strings.NewReader(listAllHardwarePortsFixture)) + addServiceNames(ports, strings.NewReader(listNetworkServiceOrderFixture)) + return ports +} + +func Test_validInterfacesMap_comesFromThePortCache(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return fixtureHardwarePorts() + }, func() time.Time { return now }) + + want := map[string]struct{}{"en0": {}, "en7": {}, "bridge0": {}, "en6": {}} + if got := validInterfacesMap(); !maps.Equal(got, want) { + t.Fatalf("validInterfacesMap = %v, want %v", got, want) + } + validInterfacesMap() + hardwarePortsByDevice() + if reads != 1 { + t.Fatalf("reads for two maps and one lookup = %d, want 1", reads) + } +} + +func Test_validInterfacesMap_namesNoInterfaceWithoutARead(t *testing.T) { + now := time.Unix(1700000000, 0) + stubHardwarePorts(t, func() map[string]hardwarePortInfo { return nil }, func() time.Time { return now }) + + if got := validInterfacesMap(); len(got) != 0 { + t.Fatalf("validInterfacesMap after a failed read = %v, want an empty set", got) + } +} + +// Test_transitionTestGlobalsReadNoPorts counts the port reads of the tests +// that take the transition globals. A test must read the ports of no host. +func Test_transitionTestGlobalsReadNoPorts(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + reads++ + return fixtureHardwarePorts() + }, func() time.Time { return now }) + + TestNetworkChangeDoesNotRestoreDownDefaultRouteSources(t) + TestNetworkChangeSourceMovedToUpInterface(t) + + if reads != 0 { + t.Fatalf("port reads of two transition tests = %d, want 0", reads) + } +} + +func Test_platformInterfaceMeta(t *testing.T) { + now := time.Unix(1700000000, 0) + stubHardwarePorts(t, func() map[string]hardwarePortInfo { + return map[string]hardwarePortInfo{"en7": {Port: "iPhone USB", Service: "iPhone USB"}} + }, func() time.Time { return now }) + + class, hardwarePort, service := platformInterfaceMeta("en7") + if class != "" { + t.Errorf("class = %q, want empty", class) + } + if hardwarePort != "iPhone USB" || service != "iPhone USB" { + t.Errorf("port, service = %q, %q, want iPhone USB twice", hardwarePort, service) + } + if class, hardwarePort, service = platformInterfaceMeta("utun3"); class != "" || hardwarePort != "" || service != "" { + t.Errorf("unknown device gave %q, %q, %q, want empty strings", class, hardwarePort, service) + } + if platformVirtualInterfaces() != nil { + t.Error("platformVirtualInterfaces is not nil on darwin") + } +} diff --git a/cmd/cli/net_linux.go b/cmd/cli/net_linux.go index ea17d3d8..71e76010 100644 --- a/cmd/cli/net_linux.go +++ b/cmd/cli/net_linux.go @@ -4,11 +4,39 @@ import ( "net" "net/netip" "os" + "path/filepath" + "slices" + "strconv" "strings" "tailscale.com/net/netmon" ) +// The kernel gives an Ethernet-like link the type ARPHRD_ETHER. +const arphrdEther = 1 + +// Tests point the readers at a temporary directory. +var ( + sysfsNetRoot = "/sys/class/net" + sysfsVirtualNetRoot = "/sys/devices/virtual/net" +) + +// noiseInterfacePrefixes name the interfaces of a container host. They change +// every few seconds, and no traffic of the host goes through them. +var noiseInterfacePrefixes = []string{"docker", "veth", "virbr", "br-"} + +// usbTetherDrivers name the kernel modules of a phone that shares its data +// connection over the cable. +var usbTetherDrivers = []string{"ipheth", "rndis_host"} + +// sysfsInfo holds what sysfs knows about one interface. +type sysfsInfo struct { + Type int + DevType string + Driver string + Wireless bool +} + func patchNetIfaceName(iface *net.Interface) (bool, error) { return true, nil } // validInterface reports whether the *net.Interface is a valid one. @@ -42,7 +70,7 @@ func validInterfacesMap() map[string]struct{} { // virtualInterfaces returns a map of virtual interfaces on current machine. func virtualInterfaces() map[string]struct{} { s := make(map[string]struct{}) - entries, _ := os.ReadDir("/sys/devices/virtual/net") + entries, _ := os.ReadDir(sysfsVirtualNetRoot) for _, entry := range entries { if entry.IsDir() { s[strings.TrimSpace(entry.Name())] = struct{}{} @@ -50,3 +78,104 @@ func virtualInterfaces() map[string]struct{} { } return s } + +// sysfsInterfaceInfo reads the sysfs entry of an interface. A missing file +// leaves its field empty. +func sysfsInterfaceInfo(name string) sysfsInfo { + dir := filepath.Join(sysfsNetRoot, name) + info := sysfsInfo{DevType: sysfsUeventValue(dir, "DEVTYPE"), Driver: sysfsDriver(dir)} + info.Type, _ = strconv.Atoi(readSysfsAttribute(dir, "type")) + if entry, err := os.Stat(filepath.Join(dir, "wireless")); err == nil && entry.IsDir() { + info.Wireless = true + } + return info +} + +// readSysfsAttribute reads one attribute without its trailing newline. +func readSysfsAttribute(dir, file string) string { + b, err := os.ReadFile(filepath.Join(dir, file)) + if err != nil { + return "" + } + return strings.TrimSpace(string(b)) +} + +// sysfsUeventValue reads one entry of the uevent file. +func sysfsUeventValue(dir, key string) string { + for _, line := range strings.Split(readSysfsAttribute(dir, "uevent"), "\n") { + after, ok := strings.CutPrefix(strings.TrimSpace(line), key+"=") + if !ok { + continue + } + return strings.TrimSpace(after) + } + return "" +} + +// sysfsDriver names the kernel module behind an interface. The driver link of +// the device points at the module, and the uevent of the device names it too. +func sysfsDriver(dir string) string { + device := filepath.Join(dir, "device") + if target, err := os.Readlink(filepath.Join(device, "driver")); err == nil { + return filepath.Base(target) + } + if driver := sysfsUeventValue(device, "DRIVER"); driver != "" { + return driver + } + return sysfsUeventValue(dir, "DRIVER") +} + +// sysfsHardwarePort names the link in the words that the link type mapping +// reads. The port names of macOS use the same words. +func sysfsHardwarePort(info sysfsInfo) string { + switch { + case info.Wireless: + return "Wi-Fi" + case slices.Contains(usbTetherDrivers, info.Driver): + return hardwarePortIPhoneUSB + case info.DevType != "": + return info.DevType + case info.Type == arphrdEther: + return "Ethernet" + } + return "" +} + +// sysfsClass names the kind of an interface. The name rule comes before the +// device type, so an interface that went away keeps its class. A bridge, a +// bond, and a VLAN carry the traffic of the host, so they stay hardware. +func sysfsClass(name string, info sysfsInfo) string { + switch { + case info.DevType == "tun" || info.DevType == "tap": + return "tunnel" + case hasInterfacePrefix(name, noiseInterfacePrefixes...): + return "virtual" + case info.Wireless || info.Type == arphrdEther: + return "hardware" + } + return "" +} + +// platformInterfaceMeta returns the Linux class and link name of an interface. +// Linux has no service name. +func platformInterfaceMeta(name string) (class, hardwarePort, service string) { + info := sysfsInterfaceInfo(name) + return sysfsClass(name, info), sysfsHardwarePort(info), "" +} + +// platformVirtualInterfaces returns the container interfaces of the host. The +// kernel lists the uplink bridges and the bonds beside them, and those carry +// the traffic of the host. +func platformVirtualInterfaces() map[string]struct{} { + noise := make(map[string]struct{}) + for name := range virtualInterfaces() { + if hasInterfacePrefix(name, noiseInterfacePrefixes...) { + noise[name] = struct{}{} + } + } + return noise +} + +// refreshInterfaceMeta does no work. Each sysfs read answers from the kernel +// of this moment, so there is no cache to refresh. +func refreshInterfaceMeta() {} diff --git a/cmd/cli/net_linux_test.go b/cmd/cli/net_linux_test.go new file mode 100644 index 00000000..db2ba0f3 --- /dev/null +++ b/cmd/cli/net_linux_test.go @@ -0,0 +1,201 @@ +package cli + +import ( + "maps" + "os" + "path/filepath" + "testing" +) + +// writeSysfsInterface builds one interface entry under a fake sysfs root. +func writeSysfsInterface(t *testing.T, root, name string, files map[string]string, wireless bool) { + t.Helper() + dir := filepath.Join(root, name) + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatalf("create %s: %v", dir, err) + } + for file, content := range files { + if err := os.WriteFile(filepath.Join(dir, file), []byte(content), 0o644); err != nil { + t.Fatalf("write %s: %v", file, err) + } + } + if !wireless { + return + } + if err := os.MkdirAll(filepath.Join(dir, "wireless"), 0o755); err != nil { + t.Fatalf("create the wireless directory: %v", err) + } +} + +// writeSysfsDriverLink points the device entry of an interface at a driver. +func writeSysfsDriverLink(t *testing.T, root, name, driver string) { + t.Helper() + device := filepath.Join(root, name, "device") + if err := os.MkdirAll(device, 0o755); err != nil { + t.Fatalf("create %s: %v", device, err) + } + if err := os.Symlink(filepath.Join(root, "drivers", driver), filepath.Join(device, "driver")); err != nil { + t.Fatalf("link the driver of %s: %v", name, err) + } +} + +// writeSysfsDriverUevent names the driver in the uevent of the device. +func writeSysfsDriverUevent(t *testing.T, root, name, driver string) { + t.Helper() + device := filepath.Join(root, name, "device") + if err := os.MkdirAll(device, 0o755); err != nil { + t.Fatalf("create %s: %v", device, err) + } + if err := os.WriteFile(filepath.Join(device, "uevent"), []byte("DRIVER="+driver+"\n"), 0o644); err != nil { + t.Fatalf("write the uevent of %s: %v", name, err) + } +} + +// stubSysfsNetRoot points the sysfs reader at a temporary directory. +func stubSysfsNetRoot(t *testing.T) string { + t.Helper() + before := sysfsNetRoot + t.Cleanup(func() { sysfsNetRoot = before }) + sysfsNetRoot = t.TempDir() + return sysfsNetRoot +} + +func Test_sysfsInterfaceInfo(t *testing.T) { + root := stubSysfsNetRoot(t) + writeSysfsInterface(t, root, "wlan0", map[string]string{ + "type": "1\n", + "uevent": "INTERFACE=wlan0\nDEVTYPE=wlan\nIFINDEX=3\n", + }, true) + writeSysfsInterface(t, root, "eth0", map[string]string{ + "type": "1\n", + "uevent": "INTERFACE=eth0\nIFINDEX=2\n", + }, false) + writeSysfsInterface(t, root, "tun0", map[string]string{ + "type": "65534\n", + "uevent": "DEVTYPE=tun\n", + }, false) + + tests := []struct { + name string + want sysfsInfo + }{ + {"wlan0", sysfsInfo{Type: 1, DevType: "wlan", Wireless: true}}, + {"eth0", sysfsInfo{Type: 1}}, + {"tun0", sysfsInfo{Type: 65534, DevType: "tun"}}, + {"absent0", sysfsInfo{}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := sysfsInterfaceInfo(tc.name); got != tc.want { + t.Errorf("sysfsInterfaceInfo(%q) = %+v, want %+v", tc.name, got, tc.want) + } + }) + } +} + +func Test_sysfsClass(t *testing.T) { + tests := []struct { + name string + iface string + info sysfsInfo + want string + }{ + {"wireless", "wlan0", sysfsInfo{Wireless: true}, "hardware"}, + {"ethernet", "eth0", sysfsInfo{Type: 1}, "hardware"}, + {"tun devtype", "tun0", sysfsInfo{Type: 65534, DevType: "tun"}, "tunnel"}, + {"tap devtype", "tap0", sysfsInfo{Type: 1, DevType: "tap"}, "tunnel"}, + {"unknown", "sit0", sysfsInfo{Type: 65534}, ""}, + {"docker bridge", "docker0", sysfsInfo{Type: 1, DevType: "bridge"}, "virtual"}, + {"container link", "veth1a2b3c", sysfsInfo{Type: 1}, "virtual"}, + {"libvirt bridge", "virbr0", sysfsInfo{Type: 1, DevType: "bridge"}, "virtual"}, + {"compose bridge", "br-1a2b3c", sysfsInfo{Type: 1, DevType: "bridge"}, "virtual"}, + {"uplink bridge", "br0", sysfsInfo{Type: 1, DevType: "bridge"}, "hardware"}, + {"bond", "bond0", sysfsInfo{Type: 1, DevType: "bond"}, "hardware"}, + {"vlan uplink", "eth0.100", sysfsInfo{Type: 1, DevType: "vlan"}, "hardware"}, + {"removed container link", "veth9f8e7d", sysfsInfo{}, "virtual"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := sysfsClass(tc.iface, tc.info); got != tc.want { + t.Errorf("sysfsClass(%q, %+v) = %q, want %q", tc.iface, tc.info, got, tc.want) + } + }) + } +} + +func Test_platformVirtualInterfaces_namesTheContainerInterfaces(t *testing.T) { + root := t.TempDir() + before := sysfsVirtualNetRoot + t.Cleanup(func() { sysfsVirtualNetRoot = before }) + sysfsVirtualNetRoot = root + for _, name := range []string{"lo", "docker0", "veth1a2b3c", "virbr0", "br-1a2b3c", "br0", "bond0"} { + if err := os.MkdirAll(filepath.Join(root, name), 0o755); err != nil { + t.Fatalf("create %s: %v", name, err) + } + } + + want := map[string]struct{}{"docker0": {}, "veth1a2b3c": {}, "virbr0": {}, "br-1a2b3c": {}} + if got := platformVirtualInterfaces(); !maps.Equal(got, want) { + t.Errorf("platformVirtualInterfaces = %v, want %v", got, want) + } + if got := virtualInterfaces(); len(got) != 7 { + t.Errorf("virtualInterfaces = %v, want every entry of the kernel directory", got) + } +} + +func Test_platformInterfaceMeta(t *testing.T) { + root := stubSysfsNetRoot(t) + writeSysfsInterface(t, root, "ctrld-test-eth0", map[string]string{ + "type": "1\n", + "uevent": "DEVTYPE=bridge\n", + }, false) + + class, hardwarePort, service := platformInterfaceMeta("ctrld-test-eth0") + if class != "hardware" { + t.Errorf("class = %q, want hardware", class) + } + if hardwarePort != "bridge" { + t.Errorf("hardware port = %q, want bridge", hardwarePort) + } + if service != "" { + t.Errorf("service = %q, want empty", service) + } +} + +func Test_platformInterfaceMeta_namesTheLink(t *testing.T) { + root := stubSysfsNetRoot(t) + writeSysfsInterface(t, root, "wlan0", map[string]string{"type": "1\n", "uevent": "DEVTYPE=wlan\n"}, true) + writeSysfsInterface(t, root, "eth0", map[string]string{"type": "1\n", "uevent": "INTERFACE=eth0\n"}, false) + writeSysfsInterface(t, root, "br0", map[string]string{"type": "1\n", "uevent": "DEVTYPE=bridge\n"}, false) + writeSysfsInterface(t, root, "tun0", map[string]string{"type": "65534\n", "uevent": "DEVTYPE=tun\n"}, false) + writeSysfsInterface(t, root, "usb0", map[string]string{"type": "1\n"}, false) + writeSysfsDriverLink(t, root, "usb0", "ipheth") + writeSysfsInterface(t, root, "usb1", map[string]string{"type": "1\n"}, false) + writeSysfsDriverUevent(t, root, "usb1", "rndis_host") + writeSysfsInterface(t, root, "eth1", map[string]string{"type": "1\n"}, false) + writeSysfsDriverUevent(t, root, "eth1", "e1000e") + + tests := []struct { + iface, class, hardwarePort, linkType string + }{ + {"wlan0", "hardware", "Wi-Fi", "wifi"}, + {"eth0", "hardware", "Ethernet", "ethernet"}, + {"eth1", "hardware", "Ethernet", "ethernet"}, + {"br0", "hardware", "bridge", "unknown"}, + {"tun0", "tunnel", "tun", "tunnel"}, + {"usb0", "hardware", hardwarePortIPhoneUSB, "usb_tether"}, + {"usb1", "hardware", hardwarePortIPhoneUSB, "usb_tether"}, + } + for _, tc := range tests { + t.Run(tc.iface, func(t *testing.T) { + class, hardwarePort, service := platformInterfaceMeta(tc.iface) + if class != tc.class || hardwarePort != tc.hardwarePort || service != "" { + t.Errorf("platformInterfaceMeta(%q) = %q, %q, %q, want %q, %q, empty", + tc.iface, class, hardwarePort, service, tc.class, tc.hardwarePort) + } + if got := linkTypeFor(hardwarePort, class); got != tc.linkType { + t.Errorf("linkTypeFor(%q, %q) = %q, want %q", hardwarePort, class, got, tc.linkType) + } + }) + } +} diff --git a/cmd/cli/net_others.go b/cmd/cli/net_others.go index f3472781..1e95b935 100644 --- a/cmd/cli/net_others.go +++ b/cmd/cli/net_others.go @@ -20,3 +20,18 @@ func validInterfacesMap() map[string]struct{} { } return map[string]struct{}{defaultRoute.InterfaceName: {}} } + +// platformInterfaceMeta returns no names. These platforms have no source for +// them, so the interface name decides the class. +func platformInterfaceMeta(name string) (class, hardwarePort, service string) { + return "", "", "" +} + +// platformVirtualInterfaces returns no set. +func platformVirtualInterfaces() map[string]struct{} { + return nil +} + +// refreshInterfaceMeta does no work. These platforms keep no interface names, +// so an adapter that appears leaves nothing stale. +func refreshInterfaceMeta() {} diff --git a/cmd/cli/net_windows.go b/cmd/cli/net_windows.go index bed06b57..226e327f 100644 --- a/cmd/cli/net_windows.go +++ b/cmd/cli/net_windows.go @@ -5,6 +5,9 @@ import ( "log" "net" "os" + "strings" + "sync" + "time" "github.com/microsoft/wmi/pkg/base/host" "github.com/microsoft/wmi/pkg/base/instance" @@ -13,6 +16,31 @@ import ( "github.com/microsoft/wmi/pkg/hardware/network/netadapter" ) +// adapterInfo holds what one WMI query knows about a network adapter. +type adapterInfo struct { + Name string + Description string + Virtual bool + ConnectorPresent bool + Hardware bool +} + +// Every network change reads the adapter set, so one query serves a window. +const adapterCacheTTL = time.Minute + +var ( + adapterCacheMu sync.Mutex + adapterCache []adapterInfo + // The set before the last read still names the adapter that went away, + // so a removed adapter keeps its class in the journal. + adapterCachePrevious []adapterInfo + adapterCacheReadAt time.Time + + // Tests replace these seams to avoid a WMI query. + readAdaptersFn = readAdapters + adapterCacheNowFn = time.Now +) + func patchNetIfaceName(iface *net.Interface) (bool, error) { return true, nil } @@ -35,6 +63,123 @@ func validInterfacesMap() map[string]struct{} { // validInterfaces returns a list of all physical interfaces. func validInterfaces() []string { + var names []string + for _, adapter := range adapters() { + if !adapter.ConnectorPresent { + mainLog.Load().Debug().Str("method", "validInterfaces").Str("interface", adapter.Name).Msg("skipping non-physical adapter") + continue + } + if !adapter.Hardware { + mainLog.Load().Debug().Str("method", "validInterfaces").Str("interface", adapter.Name).Msg("skipping non-hardware interface") + continue + } + names = append(names, adapter.Name) + } + return names +} + +// adapters returns the adapter set of the host, from the cache while it is +// younger than the window. The query runs under the mutex, so two callers +// start one query. +func adapters() []adapterInfo { + adapterCacheMu.Lock() + defer adapterCacheMu.Unlock() + now := adapterCacheNowFn() + if !adapterCacheReadAt.IsZero() && now.Sub(adapterCacheReadAt) < adapterCacheTTL { + return adapterCache + } + read := readAdaptersFn() + // A failing query waits for the window too, so it starts no query storm. + adapterCacheReadAt = now + if read == nil { + // Keep the last good read, so a failed query loses no adapter. + return adapterCache + } + adapterCachePrevious, adapterCache = adapterCache, read + return read +} + +// refreshInterfaceMeta drops the read window. The cache still describes the +// adapter that appeared or went away, and the next lookup reads the set. +func refreshInterfaceMeta() { + adapterCacheMu.Lock() + defer adapterCacheMu.Unlock() + adapterCacheReadAt = time.Time{} +} + +// tunnelAdapterWords name the VPN drivers of Windows. The description is the +// only signal, because WMI describes a tunnel adapter as an Ethernet adapter. +var tunnelAdapterWords = []string{"tap", "wintun", "wireguard", "vpn"} + +// adapterClass names the kind of an adapter. WMI marks a software adapter +// virtual and gives it no connector. +func adapterClass(adapter adapterInfo) string { + switch { + case tunnelAdapter(adapter.Description): + return "tunnel" + case adapter.Virtual && !adapter.ConnectorPresent: + return "virtual" + } + return "hardware" +} + +// tunnelAdapter reports a description that names a VPN driver. +func tunnelAdapter(description string) bool { + lower := strings.ToLower(description) + for _, word := range tunnelAdapterWords { + if strings.Contains(lower, word) { + return true + } + } + return false +} + +// platformInterfaceMeta returns the Windows class and adapter description of +// an interface. Windows has no service name. +func platformInterfaceMeta(name string) (class, hardwarePort, service string) { + adapter, known := adapterByName(name) + if !known { + return "", "", "" + } + return adapterClass(adapter), adapter.Description, "" +} + +// adapterByName finds an adapter in the current set, or in the set before the +// last read. An adapter that went away keeps its names that way. +func adapterByName(name string) (adapterInfo, bool) { + if adapter, known := findAdapter(adapters(), name); known { + return adapter, true + } + adapterCacheMu.Lock() + defer adapterCacheMu.Unlock() + return findAdapter(adapterCachePrevious, name) +} + +func findAdapter(set []adapterInfo, name string) (adapterInfo, bool) { + for _, adapter := range set { + if adapter.Name == name { + return adapter, true + } + } + return adapterInfo{}, false +} + +// platformVirtualInterfaces returns the adapters that carry no traffic of the +// host network. +func platformVirtualInterfaces() map[string]struct{} { + virtual := make(map[string]struct{}) + for _, adapter := range adapters() { + if adapterClass(adapter) != "virtual" { + continue + } + virtual[adapter.Name] = struct{}{} + } + return virtual +} + +// readAdapters runs one WMI query for every network adapter. It returns nil +// when the query fails. +func readAdapters() []adapterInfo { log.SetOutput(io.Discard) defer log.SetOutput(os.Stderr) whost := host.NewWmiLocalHost() @@ -47,47 +192,53 @@ func validInterfaces() []string { mainLog.Load().Warn().Err(err).Msg("failed to get wmi network adapter") return nil } - var adapters []string + var read []adapterInfo for _, i := range instances { adapter, err := netadapter.NewNetworkAdapter(i) if err != nil { mainLog.Load().Warn().Err(err).Msg("failed to get network adapter") continue } - - name, err := adapter.GetPropertyName() - if err != nil { - mainLog.Load().Warn().Err(err).Msg("failed to get interface name") + info, ok := readAdapterInfo(adapter) + if !ok { continue } + read = append(read, info) + } + return read +} - // From: https://learn.microsoft.com/en-us/previous-versions/windows/desktop/legacy/hh968170(v=vs.85) - // - // "Indicates if a connector is present on the network adapter. This value is set to TRUE - // if this is a physical adapter or FALSE if this is not a physical adapter." - physical, err := adapter.GetPropertyConnectorPresent() - if err != nil { - mainLog.Load().Debug().Str("method", "validInterfaces").Str("interface", name).Msg("failed to get network adapter connector present property") - continue - } - if !physical { - mainLog.Load().Debug().Str("method", "validInterfaces").Str("interface", name).Msg("skipping non-physical adapter") - continue - } +// readAdapterInfo reads the properties of one adapter. It reports false when a +// property that decides validity is unreadable. +func readAdapterInfo(adapter *netadapter.NetworkAdapter) (adapterInfo, bool) { + name, err := adapter.GetPropertyName() + if err != nil { + mainLog.Load().Warn().Err(err).Msg("failed to get interface name") + return adapterInfo{}, false + } - // Check if it's a hardware interface. Checking only for connector present is not enough - // because some interfaces are not physical but have a connector. - hardware, err := adapter.GetPropertyHardwareInterface() - if err != nil { - mainLog.Load().Debug().Str("method", "validInterfaces").Str("interface", name).Msg("failed to get network adapter hardware interface property") - continue - } - if !hardware { - mainLog.Load().Debug().Str("method", "validInterfaces").Str("interface", name).Msg("skipping non-hardware interface") - continue - } + // From: https://learn.microsoft.com/en-us/previous-versions/windows/desktop/legacy/hh968170(v=vs.85) + // + // "Indicates if a connector is present on the network adapter. This value is set to TRUE + // if this is a physical adapter or FALSE if this is not a physical adapter." + connectorPresent, err := adapter.GetPropertyConnectorPresent() + if err != nil { + mainLog.Load().Debug().Str("method", "readAdapterInfo").Str("interface", name).Msg("failed to get network adapter connector present property") + return adapterInfo{}, false + } - adapters = append(adapters, name) + // Check if it's a hardware interface. Checking only for connector present is not enough + // because some interfaces are not physical but have a connector. + hardware, err := adapter.GetPropertyHardwareInterface() + if err != nil { + mainLog.Load().Debug().Str("method", "readAdapterInfo").Str("interface", name).Msg("failed to get network adapter hardware interface property") + return adapterInfo{}, false } - return adapters + + info := adapterInfo{Name: name, ConnectorPresent: connectorPresent, Hardware: hardware} + // The class and the description describe the adapter only, so an + // unreadable one costs no validity. + info.Description, _ = adapter.GetPropertyInterfaceDescription() + info.Virtual, _ = adapter.GetPropertyVirtual() + return info, true } diff --git a/cmd/cli/net_windows_test.go b/cmd/cli/net_windows_test.go index a15f119b..2f7395be 100644 --- a/cmd/cli/net_windows_test.go +++ b/cmd/cli/net_windows_test.go @@ -5,6 +5,7 @@ import ( "bytes" "slices" "strings" + "sync" "testing" "time" ) @@ -40,3 +41,143 @@ func validInterfacesPowershell() []string { } return res } + +func Test_adapterClass(t *testing.T) { + tests := []struct { + name string + adapter adapterInfo + want string + }{ + {"virtual without a connector", adapterInfo{Virtual: true}, "virtual"}, + {"virtual with a connector", adapterInfo{Virtual: true, ConnectorPresent: true}, "hardware"}, + {"physical adapter", adapterInfo{ConnectorPresent: true, Hardware: true}, "hardware"}, + {"unknown adapter", adapterInfo{}, "hardware"}, + {"tap driver", adapterInfo{Description: "TAP-Windows Adapter V9", Virtual: true}, "tunnel"}, + {"wintun driver", adapterInfo{Description: "Wintun Userspace Tunnel", Virtual: true}, "tunnel"}, + {"wireguard driver", adapterInfo{Description: "WireGuard Tunnel #1", Virtual: true}, "tunnel"}, + {"vpn description", adapterInfo{Description: "Cisco AnyConnect VPN Adapter", ConnectorPresent: true}, "tunnel"}, + {"lower case description", adapterInfo{Description: "openvpn tap-windows6"}, "tunnel"}, + {"ethernet adapter", adapterInfo{Description: "Intel(R) Ethernet Connection I219-V", ConnectorPresent: true, Hardware: true}, "hardware"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := adapterClass(tc.adapter); got != tc.want { + t.Errorf("adapterClass(%+v) = %q, want %q", tc.adapter, got, tc.want) + } + }) + } +} + +// stubAdapters replaces the WMI and clock seams of the adapter cache, and +// restores them with the cache itself. +func stubAdapters(t *testing.T, read func() []adapterInfo, now func() time.Time) { + t.Helper() + readBefore, nowBefore := readAdaptersFn, adapterCacheNowFn + adapterCacheMu.Lock() + cacheBefore, readAtBefore := adapterCache, adapterCacheReadAt + adapterCache, adapterCacheReadAt = nil, time.Time{} + adapterCacheMu.Unlock() + t.Cleanup(func() { + readAdaptersFn, adapterCacheNowFn = readBefore, nowBefore + adapterCacheMu.Lock() + adapterCache, adapterCacheReadAt = cacheBefore, readAtBefore + adapterCacheMu.Unlock() + }) + readAdaptersFn, adapterCacheNowFn = read, now +} + +func Test_adapterCache(t *testing.T) { + now := time.Unix(1700000000, 0) + reads := 0 + stubAdapters(t, func() []adapterInfo { + reads++ + return []adapterInfo{ + {Name: "Ethernet", Description: "Intel I219-V", ConnectorPresent: true, Hardware: true}, + {Name: "Hyper-V Switch", Description: "Hyper-V Virtual Switch", Virtual: true}, + } + }, func() time.Time { return now }) + + if got := validInterfaces(); !slices.Equal(got, []string{"Ethernet"}) { + t.Fatalf("validInterfaces = %v, want [Ethernet]", got) + } + now = now.Add(59 * time.Second) + validInterfacesMap() + if reads != 1 { + t.Fatalf("reads within the cache window = %d, want 1", reads) + } + now = now.Add(2 * time.Second) + adapters() + if reads != 2 { + t.Fatalf("reads after the cache window = %d, want 2", reads) + } + refreshInterfaceMeta() + if reads != 2 { + t.Fatalf("reads for a refresh alone = %d, want 2", reads) + } + adapters() + if reads != 3 { + t.Fatalf("reads after a refresh = %d, want 3", reads) + } + + virtual := platformVirtualInterfaces() + if _, ok := virtual["Hyper-V Switch"]; !ok || len(virtual) != 1 { + t.Fatalf("platformVirtualInterfaces = %v, want the virtual switch only", virtual) + } + class, hardwarePort, service := platformInterfaceMeta("Ethernet") + if class != "hardware" || hardwarePort != "Intel I219-V" || service != "" { + t.Fatalf("platformInterfaceMeta = %q, %q, %q, want hardware, Intel I219-V, empty", class, hardwarePort, service) + } +} + +func Test_adapterCacheReadsOnceForTwoCallers(t *testing.T) { + now := time.Unix(1700000000, 0) + var mu sync.Mutex + reads := 0 + stubAdapters(t, func() []adapterInfo { + mu.Lock() + reads++ + mu.Unlock() + time.Sleep(10 * time.Millisecond) + return []adapterInfo{{Name: "Ethernet", Description: "Intel I219-V", ConnectorPresent: true, Hardware: true}} + }, func() time.Time { return now }) + + var wg sync.WaitGroup + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + adapters() + }() + } + wg.Wait() + + mu.Lock() + defer mu.Unlock() + if reads != 1 { + t.Fatalf("queries for four callers = %d, want 1", reads) + } +} + +func Test_adapterCacheKeepsTheRemovedAdapter(t *testing.T) { + now := time.Unix(1700000000, 0) + removed := false + stubAdapters(t, func() []adapterInfo { + read := []adapterInfo{{Name: "Ethernet", Description: "Intel I219-V", ConnectorPresent: true, Hardware: true}} + if removed { + return read + } + return append(read, adapterInfo{Name: "WireGuard", Description: "WireGuard Tunnel #1", Virtual: true}) + }, func() time.Time { return now }) + + adapters() + removed = true + refreshInterfaceMeta() + + class, hardwarePort, _ := platformInterfaceMeta("WireGuard") + if class != "tunnel" || hardwarePort != "WireGuard Tunnel #1" { + t.Fatalf("meta of the removed adapter = %q, %q, want tunnel, WireGuard Tunnel #1", class, hardwarePort) + } + if class, _, _ := platformInterfaceMeta("Absent"); class != "" { + t.Fatalf("meta of an adapter that no read named = %q, want empty", class) + } +} diff --git a/cmd/cli/network_delta.go b/cmd/cli/network_delta.go new file mode 100644 index 00000000..18ea6bd9 --- /dev/null +++ b/cmd/cli/network_delta.go @@ -0,0 +1,301 @@ +package cli + +import ( + "net" + "net/netip" + "slices" + "strings" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// interfaceChange describes one interface that a network delta changed. +type interfaceChange struct { + Name string + Action string + Class string + HardwarePort string + Service string + IPsBefore []string + IPsAfter []string + Flags string + MTU int + IsDefaultRoute bool +} + +// interfaceMeta holds the platform description of one interface. An empty +// Class asks the caller to read the class from the interface name. +type interfaceMeta struct { + Class string + HardwarePort string + Service string + LinkType string +} + +// interfaceMetaFunc reads the platform description of one interface. +type interfaceMetaFunc func(name string) interfaceMeta + +// virtualInterfaceSetFn names the virtual adapters of the host. Each platform +// file fills it, because a name alone does not tell a container bridge from a +// physical port. +var virtualInterfaceSetFn = func() map[string]struct{} { return nil } + +// interfaceSnapshot holds the part of one interface state that the diff +// compares. +type interfaceSnapshot struct { + present bool + up bool + mac string + mtu int + flags string + ips []string +} + +// interfaceClass groups an interface by the traffic it carries. The class +// decides which changes are noise. +func interfaceClass(name string, virtual bool) string { + switch { + case strings.HasPrefix(name, "lo"): + return "loopback" + case hasInterfacePrefix(name, "awdl", "llw"): + return "airdrop" + case hasInterfacePrefix(name, "utun", "tun", "tap", "wg", "ipsec", "ppp"): + return "tunnel" + case virtual: + return "virtual" + default: + return "hardware" + } +} + +// deltaBeforeState picks the state that the diff of one callback compares +// against, and makes a current callback the baseline of the next one. netmon +// caches major snapshots only, so every minor callback of an epoch carries the +// same Old, and a difference that lasts would look new in each of them. A +// callback that a newer snapshot replaced reads the baseline and leaves it. +// The caller holds networkSourceMu. +// +// netmon runs its callbacks in their own goroutines, so a minor callback of a +// new epoch can run before the major callback that opened the epoch. That +// major then reports its own changes against its Old and leaves the newer +// baseline in place. A major whose Old is the cache of the epoch before +// continues from the last reported state of that epoch, so a change that a +// minor callback reported does not appear again. +func (p *prog) deltaBeforeState(delta *netmon.ChangeDelta, epoch *netmon.State, current, major bool) *netmon.State { + before := p.networkDeltaState + switch { + case before == nil: + before = delta.Old + case current && major && p.networkDeltaEpoch == epoch: + return delta.Old + case p.networkDeltaEpoch != epoch && delta.Old != p.networkDeltaEpoch: + before = delta.Old + } + if current { + p.networkDeltaState, p.networkDeltaEpoch = delta.New, epoch + } + return before +} + +// diffNetworkDelta describes every interface that changed between two states. +// It walks every interface, not only the valid hardware ports, because the +// journal must show the port that went away too. +func diffNetworkDelta(beforeState, afterState *netmon.State) []interfaceChange { + before, after := networkStateOrEmpty(beforeState), networkStateOrEmpty(afterState) + virtual := virtualInterfaceSetFn() + var changes []interfaceChange + for _, name := range interfaceNameUnion(before, after) { + snapBefore, snapAfter := readInterfaceSnapshot(before, name), readInterfaceSnapshot(after, name) + action := interfaceAction(snapBefore, snapAfter) + if action == "" { + continue + } + known := snapAfter + if !snapAfter.present { + known = snapBefore + } + _, isVirtual := virtual[name] + changes = append(changes, interfaceChange{ + Name: name, + Action: action, + Class: interfaceClass(name, isVirtual), + IPsBefore: snapBefore.ips, + IPsAfter: snapAfter.ips, + Flags: known.flags, + MTU: known.mtu, + IsDefaultRoute: name == after.DefaultRouteInterface, + }) + } + return changes +} + +// noiseDelta reports a delta that no part of the daemon must act on. AirDrop +// and the virtual adapters of a container host change every few seconds, and +// each change would otherwise start the whole reconcile chain. A time jump +// means the host woke, so the daemon acts on the delta whatever it holds. +func noiseDelta(beforeState, afterState *netmon.State, timeJumped bool, changes []interfaceChange) bool { + if len(changes) == 0 || timeJumped { + return false + } + before, after := networkStateOrEmpty(beforeState), networkStateOrEmpty(afterState) + if before.DefaultRouteInterface != after.DefaultRouteInterface { + return false + } + if before.HaveV4 != after.HaveV4 || before.HaveV6 != after.HaveV6 { + return false + } + source4, source6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + for _, change := range changes { + if !noiseClass(change.Class) || change.IsDefaultRoute { + return false + } + if carriesHostTraffic(change.IPsBefore, source4, source6) { + return false + } + if carriesHostTraffic(change.IPsAfter, source4, source6) { + return false + } + } + return true +} + +// noiseClass names the classes that change on their own. The kernel moves the +// link-local address of an AirDrop or a container adapter every few seconds. +func noiseClass(class string) bool { + return class == "airdrop" || class == "virtual" +} + +// carriesHostTraffic reports an address list that reaches another host. A +// bridge, a bond, or a tunnel adapter can own the traffic of the host, so a +// routable address or the resolver source address makes the interface real +// whatever its name says. +func carriesHostTraffic(ips []string, source4, source6 net.IP) bool { + for _, ip := range ips { + prefix, err := netip.ParsePrefix(ip) + if err != nil { + continue + } + addr := prefix.Addr() + if addr.IsGlobalUnicast() || sameAddress(addr, source4) || sameAddress(addr, source6) { + return true + } + } + return false +} + +// sameAddress compares one interface address with a resolver source address. +func sameAddress(addr netip.Addr, ip net.IP) bool { + source, ok := netip.AddrFromSlice(ip) + return ok && source.Unmap() == addr +} + +// describeInterfaceChanges fills the platform fields of the changes that reach +// the journal. macOS reads them with two subprocesses, so only a delta that +// the daemon acts on pays for them. The platform class wins where it knows +// one, because a name rule cannot see a virtual adapter on Windows. +func describeInterfaceChanges(changes []interfaceChange, meta interfaceMetaFunc) { + if meta == nil { + return + } + for i := range changes { + info := meta(changes[i].Name) + changes[i].HardwarePort, changes[i].Service = info.HardwarePort, info.Service + if info.Class != "" { + changes[i].Class = info.Class + } + } +} + +// interfaceAction names the first difference that the journal reports. The +// order runs from the coarse change to the fine one, so one interface gives +// one action. +func interfaceAction(before, after interfaceSnapshot) string { + switch { + case before.present && !after.present: + return "removed" + case !before.present && after.present: + return "added" + case before.up && !after.up: + return "down" + case !before.up && after.up: + return "up" + case hasExtraEntry(before.ips, after.ips): + return "ip_removed" + case hasExtraEntry(after.ips, before.ips): + return "ip_added" + case before.mac != after.mac: + return "mac_changed" + case before.mtu != after.mtu: + return "mtu_changed" + default: + return "" + } +} + +func readInterfaceSnapshot(state *netmon.State, name string) interfaceSnapshot { + snapshot := interfaceSnapshot{ips: sortedPrefixStrings(state.InterfaceIPs[name])} + iface, ok := state.Interface[name] + if !ok { + return snapshot + } + snapshot.present = true + if iface.Interface == nil { + return snapshot + } + snapshot.up = iface.IsUp() + snapshot.mtu = iface.MTU + snapshot.flags = iface.Flags.String() + if len(iface.HardwareAddr) > 0 { + snapshot.mac = iface.HardwareAddr.String() + } + return snapshot +} + +func interfaceNameUnion(before, after *netmon.State) []string { + names := make([]string, 0, len(before.Interface)+len(after.Interface)) + for name := range before.Interface { + names = append(names, name) + } + for name := range after.Interface { + if _, both := before.Interface[name]; !both { + names = append(names, name) + } + } + slices.Sort(names) + return names +} + +func sortedPrefixStrings(prefixes []netip.Prefix) []string { + ips := prefixStrings(prefixes) + slices.Sort(ips) + return ips +} + +// networkStateOrEmpty keeps the diff free of nil checks. The first delta after +// start carries no old state. +func networkStateOrEmpty(state *netmon.State) *netmon.State { + if state != nil { + return state + } + return &netmon.State{} +} + +func hasInterfacePrefix(name string, prefixes ...string) bool { + for _, prefix := range prefixes { + if strings.HasPrefix(name, prefix) { + return true + } + } + return false +} + +// hasExtraEntry reports an entry of list that other does not hold. +func hasExtraEntry(list, other []string) bool { + for _, entry := range list { + if !slices.Contains(other, entry) { + return true + } + } + return false +} diff --git a/cmd/cli/network_delta_test.go b/cmd/cli/network_delta_test.go new file mode 100644 index 00000000..1c06ff54 --- /dev/null +++ b/cmd/cli/network_delta_test.go @@ -0,0 +1,488 @@ +package cli + +import ( + "net" + "net/netip" + "slices" + "testing" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// deltaTestInterface describes one interface of a test network state. +type deltaTestInterface struct { + name string + up bool + mac string + mtu int + ips []string +} + +var ( + deltaTestUpFlags = net.FlagUp | net.FlagBroadcast | net.FlagMulticast + deltaTestDownFlags = net.Flags(0) +) + +func deltaTestState(t *testing.T, route string, ifaces ...deltaTestInterface) *netmon.State { + t.Helper() + state := &netmon.State{ + DefaultRouteInterface: route, + Interface: map[string]netmon.Interface{}, + InterfaceIPs: map[string][]netip.Prefix{}, + } + for _, iface := range ifaces { + flags := deltaTestDownFlags + if iface.up { + flags = deltaTestUpFlags + } + var mac net.HardwareAddr + if iface.mac != "" { + parsed, err := net.ParseMAC(iface.mac) + if err != nil { + t.Fatalf("ParseMAC(%q) error: %v", iface.mac, err) + } + mac = parsed + } + state.Interface[iface.name] = netmon.Interface{Interface: &net.Interface{ + Name: iface.name, + Flags: flags, + MTU: iface.mtu, + HardwareAddr: mac, + }} + for _, ip := range iface.ips { + state.InterfaceIPs[iface.name] = append(state.InterfaceIPs[iface.name], netip.MustParsePrefix(ip)) + } + } + return state +} + +func sameInterfaceChange(got, want interfaceChange) bool { + return got.Name == want.Name && + got.Action == want.Action && + got.Class == want.Class && + got.HardwarePort == want.HardwarePort && + got.Service == want.Service && + slices.Equal(got.IPsBefore, want.IPsBefore) && + slices.Equal(got.IPsAfter, want.IPsAfter) && + got.Flags == want.Flags && + got.MTU == want.MTU && + got.IsDefaultRoute == want.IsDefaultRoute +} + +func assertInterfaceChanges(t *testing.T, got, want []interfaceChange) { + t.Helper() + if len(got) != len(want) { + t.Fatalf("changes = %+v, want %+v", got, want) + } + for i := range want { + if !sameInterfaceChange(got[i], want[i]) { + t.Errorf("change %d = %+v, want %+v", i, got[i], want[i]) + } + } +} + +func Test_diffNetworkDelta(t *testing.T) { + en0 := deltaTestInterface{name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}} + awdl0 := deltaTestInterface{name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:02", mtu: 1484, ips: []string{"fe80::1/64"}} + awdl0NewMAC := deltaTestInterface{name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:03", mtu: 1484, ips: []string{"fe80::1/64"}} + en5 := deltaTestInterface{name: "en5", up: true, mac: "aa:bb:cc:dd:ee:05", mtu: 1500, ips: []string{"192.0.2.50/24"}} + + for _, tt := range []struct { + name string + routeOld string + routeNew string + nilBefore bool + before []deltaTestInterface + after []deltaTestInterface + meta interfaceMetaFunc + want []interfaceChange + }{ + { + name: "airdrop_mac_change_with_link_local_only", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + want: []interfaceChange{{ + Name: "awdl0", Action: "mac_changed", Class: "airdrop", + IPsBefore: []string{"fe80::1/64"}, IPsAfter: []string{"fe80::1/64"}, + Flags: deltaTestUpFlags.String(), MTU: 1484, + }}, + }, + { + name: "airdrop_mac_and_link_local_change", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, { + name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:03", mtu: 1484, ips: []string{"fe80::2/64"}, + }}, + want: []interfaceChange{{ + Name: "awdl0", Action: "ip_removed", Class: "airdrop", + IPsBefore: []string{"fe80::1/64"}, IPsAfter: []string{"fe80::2/64"}, + Flags: deltaTestUpFlags.String(), MTU: 1484, + }}, + }, + { + name: "hardware_address_added_with_airdrop_change", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, + ips: []string{"192.0.2.10/24", "192.0.2.11/24"}, + }, awdl0NewMAC}, + want: []interfaceChange{ + { + Name: "awdl0", Action: "mac_changed", Class: "airdrop", + IPsBefore: []string{"fe80::1/64"}, IPsAfter: []string{"fe80::1/64"}, + Flags: deltaTestUpFlags.String(), MTU: 1484, + }, + { + Name: "en0", Action: "ip_added", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24", "192.0.2.11/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }, + }, + }, + { + name: "interface_removed", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, en5}, + after: []deltaTestInterface{en0}, + want: []interfaceChange{{ + Name: "en5", Action: "removed", Class: "hardware", + IPsBefore: []string{"192.0.2.50/24"}, IPsAfter: nil, + Flags: deltaTestUpFlags.String(), MTU: 1500, + }}, + }, + { + name: "interface_added", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0}, + after: []deltaTestInterface{en0, en5}, + want: []interfaceChange{{ + Name: "en5", Action: "added", Class: "hardware", + IPsBefore: nil, IPsAfter: []string{"192.0.2.50/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, + }}, + }, + { + name: "address_removed", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, + ips: []string{"192.0.2.10/24", "192.0.2.11/24"}, + }}, + after: []deltaTestInterface{en0}, + want: []interfaceChange{{ + Name: "en0", Action: "ip_removed", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24", "192.0.2.11/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }}, + }, + { + name: "interface_down", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0}, + after: []deltaTestInterface{{ + name: "en0", up: false, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}, + }}, + want: []interfaceChange{{ + Name: "en0", Action: "down", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestDownFlags.String(), MTU: 1500, IsDefaultRoute: true, + }}, + }, + { + name: "interface_up", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{{ + name: "en0", up: false, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}, + }}, + after: []deltaTestInterface{en0}, + want: []interfaceChange{{ + Name: "en0", Action: "up", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }}, + }, + { + name: "mtu_changed", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0}, + after: []deltaTestInterface{{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1400, ips: []string{"192.0.2.10/24"}, + }}, + want: []interfaceChange{{ + Name: "en0", Action: "mtu_changed", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1400, IsDefaultRoute: true, + }}, + }, + { + name: "default_route_moves_to_the_new_interface", + routeOld: "en0", routeNew: "en5", + before: []deltaTestInterface{en0}, + after: []deltaTestInterface{{ + name: "en0", up: false, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}, + }, en5}, + want: []interfaceChange{ + { + Name: "en0", Action: "down", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestDownFlags.String(), MTU: 1500, + }, + { + Name: "en5", Action: "added", Class: "hardware", + IPsBefore: nil, IPsAfter: []string{"192.0.2.50/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }, + }, + }, + { + name: "nil_old_state_adds_every_interface", + routeNew: "en0", + nilBefore: true, + after: []deltaTestInterface{en0}, + want: []interfaceChange{{ + Name: "en0", Action: "added", Class: "hardware", + IPsBefore: nil, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1500, IsDefaultRoute: true, + }}, + }, + { + name: "no_change", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0}, + want: nil, + }, + { + name: "meta_fills_the_platform_fields", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, {name: "utun3", up: true, mtu: 1400}}, + after: []deltaTestInterface{{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1400, ips: []string{"192.0.2.10/24"}, + }, {name: "utun3", up: true, mtu: 1380}}, + meta: func(name string) interfaceMeta { + if name == "utun3" { + return interfaceMeta{Class: "virtual"} + } + return interfaceMeta{HardwarePort: "Wi-Fi", Service: "Wi-Fi"} + }, + want: []interfaceChange{ + { + Name: "en0", Action: "mtu_changed", Class: "hardware", + HardwarePort: "Wi-Fi", Service: "Wi-Fi", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1400, IsDefaultRoute: true, + }, + { + Name: "utun3", Action: "mtu_changed", Class: "virtual", + Flags: deltaTestUpFlags.String(), MTU: 1380, + }, + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + var before *netmon.State + if !tt.nilBefore { + before = deltaTestState(t, tt.routeOld, tt.before...) + } + changes := diffNetworkDelta(before, deltaTestState(t, tt.routeNew, tt.after...)) + describeInterfaceChanges(changes, tt.meta) + assertInterfaceChanges(t, changes, tt.want) + }) + } +} + +func Test_diffNetworkDeltaVirtualSet(t *testing.T) { + virtual := virtualInterfaceSetFn + t.Cleanup(func() { virtualInterfaceSetFn = virtual }) + virtualInterfaceSetFn = func() map[string]struct{} { return map[string]struct{}{"docker0": {}} } + + before := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mtu: 1500, ips: []string{"192.0.2.10/24"}}, + deltaTestInterface{name: "docker0", up: true, mtu: 1500}, + ) + after := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mtu: 1400, ips: []string{"192.0.2.10/24"}}, + deltaTestInterface{name: "docker0", up: true, mtu: 1400}, + ) + assertInterfaceChanges(t, diffNetworkDelta(before, after), []interfaceChange{ + { + Name: "docker0", Action: "mtu_changed", Class: "virtual", + Flags: deltaTestUpFlags.String(), MTU: 1400, + }, + { + Name: "en0", Action: "mtu_changed", Class: "hardware", + IPsBefore: []string{"192.0.2.10/24"}, IPsAfter: []string{"192.0.2.10/24"}, + Flags: deltaTestUpFlags.String(), MTU: 1400, IsDefaultRoute: true, + }, + }) +} + +func Test_interfaceClass(t *testing.T) { + for _, tt := range []struct { + name string + virtual bool + want string + }{ + {name: "lo0", want: "loopback"}, + {name: "awdl0", want: "airdrop"}, + {name: "llw0", want: "airdrop"}, + {name: "utun3", want: "tunnel"}, + {name: "tun0", want: "tunnel"}, + {name: "tap0", want: "tunnel"}, + {name: "wg0", want: "tunnel"}, + {name: "ipsec0", want: "tunnel"}, + {name: "ppp0", want: "tunnel"}, + {name: "docker0", virtual: true, want: "virtual"}, + {name: "docker0", want: "hardware"}, + {name: "en0", want: "hardware"}, + } { + t.Run(tt.name+"_"+tt.want, func(t *testing.T) { + if got := interfaceClass(tt.name, tt.virtual); got != tt.want { + t.Errorf("interfaceClass(%q, %v) = %q, want %q", tt.name, tt.virtual, got, tt.want) + } + }) + } +} + +func Test_noiseDelta(t *testing.T) { + en0 := deltaTestInterface{name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}} + en5 := deltaTestInterface{name: "en5", up: true, mac: "aa:bb:cc:dd:ee:05", mtu: 1500, ips: []string{"192.0.2.50/24"}} + awdl0 := deltaTestInterface{name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:02", mtu: 1484, ips: []string{"fe80::1/64"}} + awdl0NewMAC := deltaTestInterface{name: "awdl0", up: true, mac: "aa:bb:cc:dd:ee:03", mtu: 1484, ips: []string{"fe80::1/64"}} + en0Extra := deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, + ips: []string{"192.0.2.10/24", "192.0.2.11/24"}, + } + + docker0 := deltaTestInterface{name: "docker0", up: true, mtu: 1500} + docker0NewMTU := deltaTestInterface{name: "docker0", up: true, mtu: 1400} + + for _, tt := range []struct { + name string + routeOld string + routeNew string + before []deltaTestInterface + after []deltaTestInterface + virtual []string + timeJumped bool + beforeV4, beforeV6 bool + afterV4, afterV6 bool + source4 string + want bool + }{ + { + name: "airdrop_only", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + want: true, + }, + { + name: "hardware_and_airdrop", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0Extra, awdl0NewMAC}, + want: false, + }, + { + name: "no_change", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0}, + want: false, + }, + { + name: "default_route_moved", + routeOld: "en0", routeNew: "en5", + before: []deltaTestInterface{en0, en5, awdl0}, + after: []deltaTestInterface{en0, en5, awdl0NewMAC}, + want: false, + }, + { + name: "virtual_only", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, docker0}, + after: []deltaTestInterface{en0, docker0NewMTU}, + virtual: []string{"docker0"}, + want: true, + }, + { + name: "a_time_jump_is_never_noise", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + timeJumped: true, + want: false, + }, + { + name: "the_default_route_runs_over_a_virtual_interface", + routeOld: "docker0", routeNew: "docker0", + before: []deltaTestInterface{en0, docker0}, + after: []deltaTestInterface{en0, docker0NewMTU}, + virtual: []string{"docker0"}, + want: false, + }, + { + name: "a_virtual_interface_holds_a_routable_address", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, {name: "docker0", up: true, mtu: 1500, ips: []string{"172.17.0.1/16"}}}, + after: []deltaTestInterface{en0, {name: "docker0", up: true, mtu: 1400, ips: []string{"172.17.0.1/16"}}}, + virtual: []string{"docker0"}, + want: false, + }, + { + name: "an_airdrop_interface_holds_the_resolver_source", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, {name: "awdl0", up: true, mtu: 1484, ips: []string{"169.254.10.5/16"}}}, + after: []deltaTestInterface{en0, {name: "awdl0", up: true, mtu: 1400, ips: []string{"169.254.10.5/16"}}}, + source4: "169.254.10.5", + want: false, + }, + { + name: "have_v4_flips", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + beforeV4: true, + want: false, + }, + { + name: "have_v6_flips", + routeOld: "en0", routeNew: "en0", + before: []deltaTestInterface{en0, awdl0}, + after: []deltaTestInterface{en0, awdl0NewMAC}, + afterV6: true, + want: false, + }, + } { + t.Run(tt.name, func(t *testing.T) { + stubSnapshotVirtualSet(t, tt.virtual...) + stubResolverSource(t, tt.source4) + before := deltaTestState(t, tt.routeOld, tt.before...) + after := deltaTestState(t, tt.routeNew, tt.after...) + before.HaveV4, before.HaveV6 = tt.beforeV4, tt.beforeV6 + after.HaveV4, after.HaveV6 = tt.afterV4, tt.afterV6 + changes := diffNetworkDelta(before, after) + if got := noiseDelta(before, after, tt.timeJumped, changes); got != tt.want { + t.Errorf("noiseDelta = %v, want %v, changes %+v", got, tt.want, changes) + } + }) + } +} + +// stubResolverSource sets the address that the resolver sends its queries +// from. The tests own that global, so each case starts from a known source. +func stubResolverSource(t *testing.T, source4 string) { + t.Helper() + original4, original6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + t.Cleanup(func() { + ctrld.SetDefaultLocalIPv4(original4) + ctrld.SetDefaultLocalIPv6(original6) + }) + ctrld.SetDefaultLocalIPv4(net.ParseIP(source4)) + ctrld.SetDefaultLocalIPv6(nil) +} diff --git a/cmd/cli/network_events.go b/cmd/cli/network_events.go new file mode 100644 index 00000000..959fb296 --- /dev/null +++ b/cmd/cli/network_events.go @@ -0,0 +1,167 @@ +package cli + +import ( + "time" + + "github.com/rs/zerolog" + "tailscale.com/net/netmon" +) + +// networkEventsNowFn is the clock of the network events. Tests replace it. +var networkEventsNowFn = time.Now + +const ( + hostWokeMessage = "Host woke" + + // noiseDeltaMessage names the deltas that no part of the daemon acts on. + noiseDeltaMessage = "Network delta noise" + + // interfaceChangedMessage names one interface that a delta changed. + interfaceChangedMessage = "Network interface changed" + + // networkTransitionMessage names the outcome of one delta. + networkTransitionMessage = "Network transition" + + // transitionOutcomeIgnored names a delta that left every usable interface + // as it was. + transitionOutcomeIgnored = "ignored" + + // transitionOutcomeSnapshotSuperseded names a delta whose snapshot netmon + // already replaced. A newer callback reports the same interfaces. + transitionOutcomeSnapshotSuperseded = "snapshot_superseded" +) + +// noteHostWoke reports one wake to the wake reporter. The caller passes the +// network of the wake, because the state that ctrld holds still describes the +// network before the sleep. netmon reports a time jump and measures no gap, so +// its report waits for the detector, and the journal gets one event per wake. +func (p *prog) noteHostWoke(source string, gap time.Duration, state *netmon.State) { + now := networkEventsNowFn() + // A resume moves the resolvers of the host, so every wake source starts the + // fast poll. + if p.dnsConfig != nil { + p.dnsConfig.noteActivity(now) + } + p.wake.note(now, wakeReport{source: source, gap: gap, state: state}, logHostWoke) +} + +// logHostWoke writes the Host woke journal event of one report. +func logHostWoke(report wakeReport) { + gapKnown := report.gap > 0 + event := journal(mainLog.Load().Info()).Str("source", report.source).Bool("gap_known", gapKnown) + if gapKnown { + event.Int64("gap_ms", report.gap.Milliseconds()) + } + event.Str("default_route", defaultRouteOf(report.state)). + Int("interfaces_up", upInterfaceCount(report.state)). + Msg(hostWokeMessage) +} + +// noteNoiseDelta records one delta of a noise storm. AirDrop and the virtual +// adapters of a container host change every few seconds, so the journal keeps +// one line per window while the debug stream keeps them all. +func (p *prog) noteNoiseDelta(changes []interfaceChange) { + names := changedInterfaceNames(changes) + mainLog.Load().Debug().Strs("interfaces", names).Msg(noiseDeltaMessage) + if summary, emit := p.networkNoise.add(names, networkEventsNowFn()); emit { + logNoiseSummary(summary) + } +} + +// flushNoiseSummary closes the open noise window, because the delta that the +// daemon acts on ends the storm. +func (p *prog) flushNoiseSummary() { + if summary, emit := p.networkNoise.flush(networkEventsNowFn()); emit { + logNoiseSummary(summary) + } +} + +// logNoiseSummary writes one journal line for a window of noise deltas. +func logNoiseSummary(summary noiseSummary) { + journal(mainLog.Load().Info()). + Int("count", summary.Count). + Strs("interfaces", summary.Interfaces). + Str("first_at", summary.First.Format(time.RFC3339)). + Str("last_at", summary.Last.Format(time.RFC3339)). + Msg(noiseDeltaMessage) +} + +// logInterfaceChanges puts the interfaces of one transition in the journal. +// Support reads back the port that lost its address at the time of an outage. +// An AirDrop or a container adapter changes every few seconds, so the +// transition names it and the journal keeps the interfaces that carry the +// traffic of the host. +func logInterfaceChanges(transitionID uint64, changes []interfaceChange) { + for _, change := range changes { + if noiseClass(change.Class) { + continue + } + journal(mainLog.Load().Info()). + Uint64("transition_id", transitionID). + Str("interface", change.Name). + Str("action", change.Action). + Str("class", change.Class). + Str("hardware_port", change.HardwarePort). + Str("service", change.Service). + Strs("ips_before", change.IPsBefore). + Strs("ips_after", change.IPsAfter). + Str("flags", change.Flags). + Int("mtu", change.MTU). + Bool("is_default_route", change.IsDefaultRoute). + Msg(interfaceChangedMessage) + } +} + +// networkTransitionEvent picks the level of one transition line. An ignored +// delta and a superseded snapshot arrive many times per minute, so the journal +// keeps the outcomes that can change the resolver state only. +func networkTransitionEvent(outcome string) *zerolog.Event { + if outcome == transitionOutcomeIgnored || outcome == transitionOutcomeSnapshotSuperseded { + return mainLog.Load().Debug() + } + return journal(mainLog.Load().Info()) +} + +// changedInterfaceNames names the interfaces of one delta, in the order of the +// diff. +func changedInterfaceNames(changes []interfaceChange) []string { + names := make([]string, 0, len(changes)) + for _, change := range changes { + names = append(names, change.Name) + } + return names +} + +// hasAddOrRemove reports a delta that changed the adapter set of the host. The +// platform names of an adapter come from a cache, and that cache holds nothing +// for an adapter that just appeared. +func hasAddOrRemove(changes []interfaceChange) bool { + for _, change := range changes { + if change.Action == "added" || change.Action == "removed" { + return true + } + } + return false +} + +func defaultRouteOf(state *netmon.State) string { + if state == nil { + return "" + } + return state.DefaultRouteInterface +} + +// upInterfaceCount counts the interfaces with the up flag. A netmon entry +// without an embedded interface counts as down. +func upInterfaceCount(state *netmon.State) int { + if state == nil { + return 0 + } + up := 0 + for _, iface := range state.Interface { + if iface.Interface != nil && iface.IsUp() { + up++ + } + } + return up +} diff --git a/cmd/cli/network_events_test.go b/cmd/cli/network_events_test.go new file mode 100644 index 00000000..c8f47c1c --- /dev/null +++ b/cmd/cli/network_events_test.go @@ -0,0 +1,804 @@ +package cli + +import ( + "fmt" + "net" + "net/netip" + "slices" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// hostWokeTestState names the network that a wake event reports. +func hostWokeTestState() *netmon.State { + return &netmon.State{ + DefaultRouteInterface: "en0", + Interface: map[string]netmon.Interface{ + "en0": {Interface: &net.Interface{Name: "en0", Flags: net.FlagUp}}, + "en5": {Interface: &net.Interface{Name: "en5"}}, + "utun": {Interface: &net.Interface{Name: "utun", Flags: net.FlagUp}}, + }, + InterfaceIPs: map[string][]netip.Prefix{}, + } +} + +// hostWokeTestClock hands the wake events a clock that the test moves. +func hostWokeTestClock(t *testing.T, clock *time.Time) { + t.Helper() + original := networkEventsNowFn + t.Cleanup(func() { networkEventsNowFn = original }) + networkEventsNowFn = func() time.Time { return *clock } +} + +// hostWokeTestHold captures the hold timer of the wake reporter, so a test +// decides when the hold ends. +func hostWokeTestHold(p *prog) *[]func() { + var armed []func() + p.wake.after = func(_ time.Duration, fn func()) { armed = append(armed, fn) } + return &armed +} + +// Test_noteHostWokeNetmonFirstWaitsForTheDetectorGap covers the order of the +// 2026-09-14 capture. netmon sees the time jump first and measures no gap, so +// its report waits for the detector, and the journal gets one event with the +// gap. +func Test_noteHostWokeNetmonFirstWaitsForTheDetectorGap(t *testing.T) { + logs := captureDebugMainLog(t) + clock := time.Date(2026, 9, 17, 7, 14, 0, 0, time.UTC) + hostWokeTestClock(t, &clock) + p := &prog{dnsConfig: newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return nil, nil })} + armed := hostWokeTestHold(p) + + p.noteHostWoke("netmon", 0, hostWokeTestState()) + if events := jsonLogEvents(t, logs, hostWokeMessage); len(events) != 0 { + t.Fatalf("got %d Host woke events before the hold ended, want 0", len(events)) + } + if got := p.dnsConfig.nextDelay(clock); got != dnsConfigFastInterval { + t.Fatalf("DNS configuration poll delay after a wake = %s, want %s", got, dnsConfigFastInterval) + } + clock = clock.Add(2 * time.Second) + p.noteHostWoke("detector", 90*time.Second, hostWokeTestState()) + for _, fire := range *armed { + fire() + } + + events := jsonLogEvents(t, logs, hostWokeMessage) + if len(events) != 1 { + t.Fatalf("got %d Host woke events, want one report with the gap of the detector", len(events)) + } + wantField(t, events[0], "source", "detector") + wantField(t, events[0], "gap_known", true) + wantField(t, events[0], "gap_ms", float64(90000)) + wantField(t, events[0], "default_route", "en0") + wantField(t, events[0], "interfaces_up", float64(2)) + wantField(t, events[0], "journal", true) + + clock = clock.Add(5 * time.Second) + p.noteHostWoke("netmon", 0, hostWokeTestState()) + if events = jsonLogEvents(t, logs, hostWokeMessage); len(events) != 1 { + t.Fatalf("got %d Host woke events, want no second report inside the window", len(events)) + } +} + +// Test_noteHostWokeNetmonAloneReportsAfterTheHold covers a host without the +// detector. The netmon report goes out without a gap when the hold ends. +func Test_noteHostWokeNetmonAloneReportsAfterTheHold(t *testing.T) { + logs := captureDebugMainLog(t) + clock := time.Date(2026, 9, 17, 7, 14, 0, 0, time.UTC) + hostWokeTestClock(t, &clock) + p := &prog{} + armed := hostWokeTestHold(p) + + p.noteHostWoke("netmon", 0, hostWokeTestState()) + if len(*armed) != 1 { + t.Fatalf("holds armed = %d, want 1", len(*armed)) + } + (*armed)[0]() + + events := jsonLogEvents(t, logs, hostWokeMessage) + if len(events) != 1 { + t.Fatalf("got %d Host woke events after the hold, want 1", len(events)) + } + wantField(t, events[0], "source", "netmon") + wantField(t, events[0], "gap_known", false) + if _, measured := events[0]["gap_ms"]; measured { + t.Fatalf("the netmon report carries a gap that it cannot measure: %v", events[0]) + } +} + +// Test_noteHostWokeDetectorFirstReportsOneEvent covers the other order. The +// gap is known with the first report, so netmon adds nothing. +func Test_noteHostWokeDetectorFirstReportsOneEvent(t *testing.T) { + logs := captureDebugMainLog(t) + resume := time.Date(2026, 9, 17, 7, 14, 0, 0, time.UTC) + clock := resume + hostWokeTestClock(t, &clock) + p := &prog{} + armed := hostWokeTestHold(p) + + p.noteHostWoke("detector", 90*time.Second, hostWokeTestState()) + clock = resume.Add(2 * time.Second) + p.noteHostWoke("netmon", 0, hostWokeTestState()) + + events := jsonLogEvents(t, logs, hostWokeMessage) + if len(events) != 1 || len(*armed) != 0 { + t.Fatalf("got %d Host woke events and %d holds inside the window, want 1 and 0", len(events), len(*armed)) + } + wantField(t, events[0], "source", "detector") + wantField(t, events[0], "gap_ms", float64(90000)) + + // netmon polls every 15 s, so the window has to outlast one poll. + clock = resume.Add(29 * time.Second) + p.noteHostWoke("netmon", 0, hostWokeTestState()) + if events = jsonLogEvents(t, logs, hostWokeMessage); len(events) != 1 || len(*armed) != 0 { + t.Fatalf("got %d Host woke events and %d holds, want one report for a wake that netmon reports late", len(events), len(*armed)) + } + + clock = resume.Add(31 * time.Second) + p.noteHostWoke("netmon", 0, hostWokeTestState()) + for _, fire := range *armed { + fire() + } + if events = jsonLogEvents(t, logs, hostWokeMessage); len(events) != 2 { + t.Fatalf("got %d Host woke events after the window, want 2", len(events)) + } +} + +// networkEventsHarness drives the network callback on a clock that the test +// owns. Every host read stays behind a stub, so no test starts networksetup or +// reads the route table of the machine. +type networkEventsHarness struct { + prog *prog + logs *syncBuffer + now time.Time + reconciled int + ignored int +} + +func newNetworkEventsHarness(t *testing.T, valid ...string) *networkEventsHarness { + t.Helper() + sourceTestGlobals(t) + stubHeaderSnapshotSources(t) + stubSnapshotVirtualSet(t) + h := &networkEventsHarness{ + prog: &prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}}, + logs: captureDebugMainLog(t), + now: time.Date(2026, 9, 17, 7, 14, 0, 0, time.UTC), + } + oldNow := networkEventsNowFn + networkEventsNowFn = func() time.Time { return h.now } + t.Cleanup(func() { networkEventsNowFn = oldNow }) + validSet := make(map[string]struct{}, len(valid)) + for _, name := range valid { + validSet[name] = struct{}{} + } + networkChangeValidInterfacesFn = func() map[string]struct{} { return validSet } + networkChangeReconcileFn = func(_ *prog, _ uint64) { h.reconciled++ } + networkChangeIgnoredInterceptFn = func(_ *prog, _ *netmon.ChangeDelta, _ time.Time) { h.ignored++ } + return h +} + +func (h *networkEventsHarness) handle(before, after *netmon.State, timeJumped bool) { + h.prog.handleNetworkChange(&netmon.ChangeDelta{Old: before, New: after, TimeJumped: timeJumped}, true) +} + +// stubNetmonCache gives every callback of the harness one cache snapshot. +// netmon caches major snapshots only, so the epoch of a minor callback stays +// at that snapshot. +func (h *networkEventsHarness) stubNetmonCache(t *testing.T, cache *netmon.State) { + t.Helper() + oldCurrent, oldRead := networkChangeCurrentStateFn, readNetworkSourceStateFn + t.Cleanup(func() { networkChangeCurrentStateFn, readNetworkSourceStateFn = oldCurrent, oldRead }) + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return cache } + readNetworkSourceStateFn = func() (*netmon.State, error) { return cache, nil } +} + +// handleMinor drives one minor callback. netmon puts the cached major snapshot +// in Old, whatever the callback before it reported. +func (h *networkEventsHarness) handleMinor(cache, after *netmon.State) { + h.prog.handleNetworkChange(&netmon.ChangeDelta{Monitor: netmon.NewStatic(), Old: cache, New: after}, false) +} + +// interfaceEventsFor counts the journal events of one interface. +func (h *networkEventsHarness) interfaceEventsFor(t *testing.T, name string) int { + t.Helper() + count := 0 + for _, event := range jsonLogEvents(t, h.logs, interfaceChangedMessage) { + if event["interface"] == name { + count++ + } + } + return count +} + +// eventsAtLevel selects the captured events of one message and one level. +func (h *networkEventsHarness) eventsAtLevel(t *testing.T, message, level string) []map[string]any { + t.Helper() + var selected []map[string]any + for _, event := range jsonLogEvents(t, h.logs, message) { + if event["level"] == level { + selected = append(selected, event) + } + } + return selected +} + +var networkEventsHardwareIface = deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.10/24"}, +} + +// networkEventsAddressChange returns a delta that moves the address of en0. +func networkEventsAddressChange(t *testing.T) (before, after *netmon.State) { + t.Helper() + return deltaTestState(t, "en0", networkEventsHardwareIface), + deltaTestState(t, "en0", deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.20/24"}, + }) +} + +// transitionEvent returns the one transition event of a callback. +func (h *networkEventsHarness) transitionEvent(t *testing.T) map[string]any { + t.Helper() + events := jsonLogEvents(t, h.logs, networkTransitionMessage) + if len(events) != 1 { + t.Fatalf("got %d %q events, want 1", len(events), networkTransitionMessage) + } + return events[0] +} + +// wantNoField fails when an event carries a field that it must not. +func wantNoField(t *testing.T, event map[string]any, field string) { + t.Helper() + if value, carried := event[field]; carried { + t.Fatalf("field %q: got %v, want no field", field, value) + } +} + +// noiseDeltaStates returns a delta that moves the AirDrop link-local address. +func noiseDeltaStates(t *testing.T, step int) (before, after *netmon.State) { + t.Helper() + airdrop := func(address int) deltaTestInterface { + return deltaTestInterface{name: "awdl0", up: true, mtu: 1484, ips: []string{fmt.Sprintf("fe80::%d/64", address)}} + } + return deltaTestState(t, "en0", networkEventsHardwareIface, airdrop(step)), + deltaTestState(t, "en0", networkEventsHardwareIface, airdrop(step+1)) +} + +func wantStringsField(t *testing.T, event map[string]any, field string, want ...string) { + t.Helper() + values, ok := event[field].([]any) + if !ok { + t.Fatalf("field %q: got %v, want a list", field, event[field]) + } + got := make([]string, 0, len(values)) + for _, value := range values { + text, isText := value.(string) + if !isText { + t.Fatalf("field %q: got %v, want strings", field, event[field]) + } + got = append(got, text) + } + if !slices.Equal(got, want) { + t.Fatalf("field %q: got %v, want %v", field, got, want) + } +} + +func Test_networkEventsNoiseDeltaSkipsTheHandler(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + + before, after := noiseDeltaStates(t, 1) + h.handle(before, after, false) + + if h.reconciled != 0 || h.ignored != 0 { + t.Fatalf("noise delta ran the handler: reconciled=%d ignored=%d, want 0/0", h.reconciled, h.ignored) + } + debugLines := h.eventsAtLevel(t, noiseDeltaMessage, "debug") + if len(debugLines) != 1 { + t.Fatalf("got %d debug %q lines, want 1", len(debugLines), noiseDeltaMessage) + } + wantStringsField(t, debugLines[0], "interfaces", "awdl0") + if events := jsonLogEvents(t, h.logs, networkTransitionMessage); len(events) != 0 { + t.Fatalf("got %d %q events for a noise delta, want 0", len(events), networkTransitionMessage) + } +} + +func Test_networkEventsNoiseStormEmitsOneSummaryPerWindow(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + start := h.now + + for step := 0; step <= 10; step++ { + h.now = start.Add(time.Duration(step) * time.Minute) + before, after := noiseDeltaStates(t, step) + h.handle(before, after, false) + } + + summaries := h.eventsAtLevel(t, noiseDeltaMessage, "info") + if len(summaries) != 2 { + t.Fatalf("got %d journal %q events, want 2", len(summaries), noiseDeltaMessage) + } + wantField(t, summaries[0], "journal", true) + wantField(t, summaries[0], "count", float64(1)) + wantField(t, summaries[1], "count", float64(10)) + wantField(t, summaries[1], "first_at", start.Add(time.Minute).Format(time.RFC3339)) + wantField(t, summaries[1], "last_at", start.Add(10*time.Minute).Format(time.RFC3339)) + wantStringsField(t, summaries[1], "interfaces", "awdl0") +} + +func Test_networkEventsLogsEveryChangedInterface(t *testing.T) { + h := newNetworkEventsHarness(t, "en0", "en5") + before := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mtu: 1500, ips: []string{"192.0.2.10/24", "2001:db8::10/64"}}, + deltaTestInterface{name: "en5", up: true, mtu: 1500, ips: []string{"198.51.100.10/24"}}, + ) + before.HaveV4, before.HaveV6 = true, true + after := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mtu: 1500, ips: []string{"192.0.2.20/24"}}, + deltaTestInterface{name: "en5", up: true, mtu: 1500, ips: []string{"198.51.100.20/24"}}, + ) + after.HaveV4, after.HaveV6 = true, false + + h.handle(before, after, false) + + changes := jsonLogEvents(t, h.logs, interfaceChangedMessage) + if len(changes) != 2 { + t.Fatalf("got %d %q events, want 2", len(changes), interfaceChangedMessage) + } + for index, name := range []string{"en0", "en5"} { + wantField(t, changes[index], "journal", true) + wantField(t, changes[index], "level", "info") + wantField(t, changes[index], "interface", name) + wantField(t, changes[index], "action", "ip_removed") + wantField(t, changes[index], "class", "hardware") + wantField(t, changes[index], "transition_id", float64(1)) + wantField(t, changes[index], "mtu", float64(1500)) + } + wantField(t, changes[0], "is_default_route", true) + wantField(t, changes[1], "is_default_route", false) + wantStringsField(t, changes[0], "ips_before", "192.0.2.10/24", "2001:db8::10/64") + wantStringsField(t, changes[0], "ips_after", "192.0.2.20/24") + + transitions := jsonLogEvents(t, h.logs, networkTransitionMessage) + if len(transitions) != 1 { + t.Fatalf("got %d %q events, want 1", len(transitions), networkTransitionMessage) + } + wantField(t, transitions[0], "journal", true) + wantField(t, transitions[0], "level", "info") + wantField(t, transitions[0], "outcome", "accepted") + wantField(t, transitions[0], "time_jumped", false) + wantField(t, transitions[0], "have_v4_before", true) + wantField(t, transitions[0], "have_v4_after", true) + wantField(t, transitions[0], "have_v6_before", true) + wantField(t, transitions[0], "have_v6_after", false) + wantStringsField(t, transitions[0], "changed_interfaces", "en0", "en5") + if h.reconciled != 1 { + t.Fatalf("reconciled = %d, want 1", h.reconciled) + } + if snapshots := jsonLogEvents(t, h.logs, networkSnapshotMessage); len(snapshots) != 1 { + t.Fatalf("got %d %q events, want 1", len(snapshots), networkSnapshotMessage) + } +} + +func Test_networkEventsTimeJumpReportsTheWake(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + armed := hostWokeTestHold(h.prog) + before, after := noiseDeltaStates(t, 1) + + h.handle(before, after, true) + for _, fire := range *armed { + fire() + } + + wakes := jsonLogEvents(t, h.logs, hostWokeMessage) + if len(wakes) != 1 { + t.Fatalf("got %d %q events, want 1", len(wakes), hostWokeMessage) + } + wantField(t, wakes[0], "journal", true) + wantField(t, wakes[0], "source", "netmon") + wantField(t, wakes[0], "gap_known", false) + wantField(t, wakes[0], "default_route", "en0") + if _, measured := wakes[0]["gap_ms"]; measured { + t.Fatalf("the netmon report carries a gap that it cannot measure: %v", wakes[0]) + } +} + +func Test_networkEventsStoreTheIPv6Flag(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + // The one-time probe wins over the first store, so consume it first. + available := ctrld.HasIPv6() + t.Cleanup(func() { ctrld.SetIPv6Available(available) }) + ctrld.SetIPv6Available(true) + + before := deltaTestState(t, "en0", networkEventsHardwareIface) + before.HaveV6 = true + after := deltaTestState(t, "en0", deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.20/24"}, + }) + after.HaveV6 = false + + h.handle(before, after, false) + + if ctrld.HasIPv6() { + t.Fatal("HasIPv6 = true after a callback with have_v6 false, want false") + } +} + +// Test_networkEventsMinorDeltaDiffsAgainstTheStateBeforeIt covers a difference +// that lasts. The cached major snapshot holds no link-local address of en0, so +// a diff against the cache reports the same change in every minor callback. +func Test_networkEventsMinorDeltaDiffsAgainstTheStateBeforeIt(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + en0WithLinkLocal := deltaTestInterface{ + name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, + ips: []string{"192.0.2.10/24", "fe80::a/64"}, + } + airdrop := func(address int) deltaTestInterface { + return deltaTestInterface{name: "awdl0", up: true, mtu: 1484, ips: []string{fmt.Sprintf("fe80::%d/64", address)}} + } + cache := deltaTestState(t, "en0", networkEventsHardwareIface, airdrop(1)) + h.stubNetmonCache(t, cache) + + h.handleMinor(cache, deltaTestState(t, "en0", en0WithLinkLocal, airdrop(2))) + h.handleMinor(cache, deltaTestState(t, "en0", en0WithLinkLocal, airdrop(3))) + + if got := h.interfaceEventsFor(t, "en0"); got != 1 { + t.Fatalf("got %d %q events for en0, want 1", got, interfaceChangedMessage) + } + if noise := h.eventsAtLevel(t, noiseDeltaMessage, "debug"); len(noise) != 1 { + t.Fatalf("got %d debug %q lines, want 1 for the second delta", len(noise), noiseDeltaMessage) + } +} + +// Test_networkEventsNoiseDeltaReadsNoPlatformMeta pins the class-first order. +// The macOS meta read starts two subprocesses per call, so a storm of noise +// deltas must not reach it. +func Test_networkEventsNoiseDeltaReadsNoPlatformMeta(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + reads := countPlatformMetaReads(t) + + before, after := noiseDeltaStates(t, 1) + h.handle(before, after, false) + + if *reads != 0 { + t.Fatalf("a noise delta read the platform meta %d times, want 0", *reads) + } +} + +// countPlatformMetaReads counts the platform meta reads of a test. One read +// starts two subprocesses on macOS, so the count is the cost of a delta. +func countPlatformMetaReads(t *testing.T) *int { + t.Helper() + reads := 0 + original := platformInterfaceMetaFn + t.Cleanup(func() { platformInterfaceMetaFn = original }) + platformInterfaceMetaFn = func(string) (class, hardwarePort, service string) { + reads++ + return "", "", "" + } + return &reads +} + +// Test_networkEventsNoiseStormEndsWithTheNextTransition covers the last window +// of a storm. The count of the deltas that a real change follows must not wait +// for the next storm. +func Test_networkEventsNoiseStormEndsWithTheNextTransition(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + start := h.now + + for step := 0; step < 4; step++ { + h.now = start.Add(time.Duration(step) * time.Minute) + before, after := noiseDeltaStates(t, step) + h.handle(before, after, false) + } + h.now = start.Add(5 * time.Minute) + before, after := networkEventsAddressChange(t) + h.handle(before, after, false) + + summaries := h.eventsAtLevel(t, noiseDeltaMessage, "info") + if len(summaries) != 2 { + t.Fatalf("got %d journal %q events, want the first delta and the end of the storm", len(summaries), noiseDeltaMessage) + } + wantField(t, summaries[1], "count", float64(3)) + wantField(t, summaries[1], "first_at", start.Add(time.Minute).Format(time.RFC3339)) + wantField(t, summaries[1], "last_at", start.Add(3*time.Minute).Format(time.RFC3339)) + wantStringsField(t, summaries[1], "interfaces", "awdl0") +} + +// Test_networkEventsStoreTheNetworkBeforeTheHeaderRender proves that the +// callback stores its network under the source lock and renders the header +// after the unlock, so the header read of the hardware ports starts no +// process under the lock. +func Test_networkEventsStoreTheNetworkBeforeTheHeaderRender(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + lockHeld, renders := false, 0 + original := defaultRoutesFn + t.Cleanup(func() { defaultRoutesFn = original }) + // The header render reads the routes, so this seam runs where the + // callback renders the header. + defaultRoutesFn = func() (defaultRoute, defaultRoute) { + renders++ + if h.prog.networkSourceMu.TryLock() { + h.prog.networkSourceMu.Unlock() + } else { + lockHeld = true + } + return defaultRoute{Gateway: "192.0.2.1"}, defaultRoute{} + } + + before, after := networkEventsAddressChange(t) + h.handle(before, after, false) + + if renders == 0 { + t.Fatal("the callback rendered no header") + } + if lockHeld { + t.Fatal("the header render ran under the source lock") + } + if h.prog.lastNetworkState.Load() != after { + t.Fatal("the callback did not store its network") + } +} + +// Test_networkEventsIgnoredTransitionStaysOutOfTheJournal pins the level of an +// ignored delta. Such a delta arrives many times per minute, so it belongs in +// the debug stream and not in the journal. +func Test_networkEventsIgnoredTransitionStaysOutOfTheJournal(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + before := deltaTestState(t, "en0", networkEventsHardwareIface, + deltaTestInterface{name: "en5", up: true, mtu: 1500, ips: []string{"198.51.100.10/24"}}) + after := deltaTestState(t, "en0", networkEventsHardwareIface, + deltaTestInterface{name: "en5", up: true, mtu: 1500, ips: []string{"198.51.100.20/24"}}) + + h.handle(before, after, false) + + transition := h.transitionEvent(t) + wantField(t, transition, "outcome", transitionOutcomeIgnored) + wantField(t, transition, "level", "debug") + wantNoField(t, transition, "journal") + if snapshots := jsonLogEvents(t, h.logs, networkSnapshotMessage); len(snapshots) != 0 { + t.Fatalf("got %d %q events for an ignored delta, want 0", len(snapshots), networkSnapshotMessage) + } + if h.ignored != 1 || h.reconciled != 0 { + t.Fatalf("ignored delta: ignored=%d reconciled=%d, want 1/0", h.ignored, h.reconciled) + } +} + +// Test_networkEventsSupersededTransitionEntersTheJournal pins the level of a +// delta that a newer accepted delta replaced. Support reads it to tell a lost +// reconcile from one that never started. +func Test_networkEventsSupersededTransitionEntersTheJournal(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + h.prog.networkAcceptedGen.Store(5) + + before, after := networkEventsAddressChange(t) + h.handle(before, after, false) + + transition := h.transitionEvent(t) + wantField(t, transition, "outcome", "superseded") + wantField(t, transition, "level", "info") + wantField(t, transition, "journal", true) + if h.reconciled != 0 { + t.Fatalf("a superseded delta reconciled %d times, want 0", h.reconciled) + } +} + +// Test_networkEventsSupersededSnapshotReportsTheOutcome covers the delta whose +// snapshot netmon already replaced. The callback logged nothing at all, so the +// journal showed a silent gap between two deltas. +func Test_networkEventsSupersededSnapshotReportsTheOutcome(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + cache := deltaTestState(t, "en0", networkEventsHardwareIface) + h.stubNetmonCache(t, cache) + reads := countPlatformMetaReads(t) + stale, after := networkEventsAddressChange(t) + + // netmon holds a newer snapshot than the one that this callback carries. + h.prog.handleNetworkChange(&netmon.ChangeDelta{Monitor: netmon.NewStatic(), Old: stale, New: after}, false) + + transition := h.transitionEvent(t) + wantField(t, transition, "outcome", "snapshot_superseded") + wantField(t, transition, "level", "debug") + wantNoField(t, transition, "journal") + wantStringsField(t, transition, "changed_interfaces", "en0") + if events := jsonLogEvents(t, h.logs, interfaceChangedMessage); len(events) != 0 { + t.Fatalf("got %d %q events for a superseded snapshot, want 0", len(events), interfaceChangedMessage) + } + if h.reconciled != 0 || h.ignored != 0 { + t.Fatalf("superseded snapshot ran the handler: reconciled=%d ignored=%d, want 0/0", h.reconciled, h.ignored) + } + if *reads != 0 { + t.Fatalf("a superseded snapshot read the platform meta %d times, want 0", *reads) + } +} + +// Test_networkEventsLogsTheInterfacesOutsideTheNoiseClass covers a mixed +// delta. An AirDrop change beside a real one must not fill the journal, but +// the transition still names every interface that changed. +func Test_networkEventsLogsTheInterfacesOutsideTheNoiseClass(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + before := deltaTestState(t, "en0", networkEventsHardwareIface, + deltaTestInterface{name: "awdl0", up: true, mtu: 1484, ips: []string{"fe80::1/64"}}) + after := deltaTestState(t, "en0", + deltaTestInterface{name: "en0", up: true, mac: "aa:bb:cc:dd:ee:01", mtu: 1500, ips: []string{"192.0.2.20/24"}}, + deltaTestInterface{name: "awdl0", up: true, mtu: 1484, ips: []string{"fe80::2/64"}}) + + h.handle(before, after, false) + + events := jsonLogEvents(t, h.logs, interfaceChangedMessage) + if len(events) != 1 { + t.Fatalf("got %d %q events, want one for the hardware port", len(events), interfaceChangedMessage) + } + wantField(t, events[0], "interface", "en0") + wantField(t, events[0], "class", "hardware") + wantStringsField(t, h.transitionEvent(t), "changed_interfaces", "awdl0", "en0") +} + +// reorderTestState builds a state with en0 on one address and awdl0 on one +// link-local address, so a test can replay callbacks out of order. +func reorderTestState(en0 string, awdl string) *netmon.State { + return &netmon.State{ + Interface: map[string]netmon.Interface{ + "en0": {Interface: &net.Interface{Name: "en0", Flags: net.FlagUp}}, + "awdl0": {Interface: &net.Interface{Name: "awdl0", Flags: net.FlagUp}}, + }, + InterfaceIPs: map[string][]netip.Prefix{ + "en0": {netip.MustParsePrefix(en0)}, + "awdl0": {netip.MustParsePrefix(awdl)}, + }, + DefaultRouteInterface: "en0", + HaveV4: true, + } +} + +// firstV4Of returns the first IPv4 address of one interface of a state. +func firstV4Of(state *netmon.State, name string) net.IP { + for _, prefix := range state.InterfaceIPs[name] { + if prefix.Addr().Is4() { + return net.IP(prefix.Addr().AsSlice()) + } + } + return nil +} + +// Test_networkEventsSupersededCallbackKeepsTheBaseline replays the callbacks +// out of order: B runs, a late A arrives while the monitor already caches C, +// then C runs. The late A must not become the baseline of C, or the address +// change from B to C looks like AirDrop noise and the stale source stays. +func Test_networkEventsSupersededCallbackKeepsTheBaseline(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + stateA := reorderTestState("192.0.2.20/24", "fe80::1/64") + stateB := reorderTestState("192.0.2.10/24", "fe80::1/64") + stateC := reorderTestState("192.0.2.20/24", "fe80::2/64") + cache := stateB + oldCurrent := networkChangeCurrentStateFn + t.Cleanup(func() { networkChangeCurrentStateFn = oldCurrent }) + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return cache } + networkChangeReconcileFn = func(p *prog, _ uint64) { + h.reconciled++ + ctrld.SetDefaultLocalIPv4(firstV4Of(p.networkSourceState, "en0")) + } + monitor := netmon.NewStatic() + + h.prog.handleNetworkChange(&netmon.ChangeDelta{Monitor: monitor, Old: stateA, New: stateB}, true) + cache = stateC + h.prog.handleNetworkChange(&netmon.ChangeDelta{Monitor: monitor, Old: &netmon.State{}, New: stateA}, true) + h.prog.handleNetworkChange(&netmon.ChangeDelta{Monitor: monitor, Old: stateB, New: stateC}, true) + + if h.reconciled != 2 { + t.Fatalf("reconciles = %d, want 2: the late callback must not hide the change from B to C", h.reconciled) + } + if got := ctrld.GetDefaultLocalIPv4().String(); got != "192.0.2.20" { + t.Fatalf("resolver source = %s, want 192.0.2.20", got) + } +} + +// orderTestState builds a state for the callback order tests: en0 on one +// address, plus extra interfaces with their addresses and their up flag. +func orderTestState(route, en0 string, extra map[string][]string, down map[string]bool) *netmon.State { + state := &netmon.State{ + Interface: map[string]netmon.Interface{}, + InterfaceIPs: map[string][]netip.Prefix{}, + DefaultRouteInterface: route, + HaveV4: true, + } + add := func(name string, up bool, ips []string) { + flags := net.Flags(0) + if up { + flags = net.FlagUp + } + state.Interface[name] = netmon.Interface{Interface: &net.Interface{Name: name, Flags: flags}} + for _, ip := range ips { + state.InterfaceIPs[name] = append(state.InterfaceIPs[name], netip.MustParsePrefix(ip)) + } + } + if en0 != "" { + add("en0", true, []string{en0}) + } + for name, ips := range extra { + add(name, !down[name], ips) + } + return state +} + +// Test_networkEventsMajorAfterItsMinorKeepsItsChanges runs a minor callback +// of the new epoch before the major callback that opened the epoch. netmon +// starts one goroutine per callback, so this order happens. The major must +// still report the address change and reconcile. +func Test_networkEventsMajorAfterItsMinorKeepsItsChanges(t *testing.T) { + for _, order := range []string{"in_order", "minor_first", "minor_first_time_jumped"} { + t.Run(order, func(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + hostWokeTestHold(h.prog) + s0 := orderTestState("en0", "192.0.2.10/24", map[string][]string{"awdl0": {"fe80::1/64"}}, nil) + s2 := orderTestState("en0", "192.0.2.20/24", map[string][]string{"awdl0": {"fe80::1/64"}}, nil) + s3 := orderTestState("en0", "192.0.2.20/24", map[string][]string{"awdl0": {"fe80::2/64"}}, nil) + h.stubNetmonCache(t, s2) + monitor := netmon.NewStatic() + major := &netmon.ChangeDelta{Monitor: monitor, Old: s0, New: s2, TimeJumped: order == "minor_first_time_jumped"} + minor := &netmon.ChangeDelta{Monitor: monitor, Old: s2, New: s3} + if order == "in_order" { + h.prog.handleNetworkChange(major, true) + h.prog.handleNetworkChange(minor, false) + } else { + h.prog.handleNetworkChange(minor, false) + h.prog.handleNetworkChange(major, true) + } + + if h.reconciled != 1 { + t.Fatalf("reconciled = %d, want 1 for the move of en0 from .10 to .20", h.reconciled) + } + if got := h.interfaceEventsFor(t, "en0"); got != 1 { + t.Fatalf("en0 interface events = %d, want 1", got) + } + if h.prog.networkDeltaState != s3 { + t.Fatal("the baseline must stay at the newest state of the epoch") + } + }) + } +} + +// Test_networkEventsEpochChangeReportsAMinorChangeOnce closes an epoch with a +// major callback whose Old is the cache of the epoch. The interface that a +// minor callback of that epoch reported must not appear again. +func Test_networkEventsEpochChangeReportsAMinorChangeOnce(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") + s0 := orderTestState("en0", "192.0.2.10/24", nil, nil) + s1 := orderTestState("en0", "192.0.2.10/24", map[string][]string{"ipsec0": {"10.0.0.2/32"}}, nil) + s2 := orderTestState("en0", "192.0.2.20/24", map[string][]string{"ipsec0": {"10.0.0.2/32"}}, nil) + cache := s0 + oldCurrent, oldRead := networkChangeCurrentStateFn, readNetworkSourceStateFn + t.Cleanup(func() { networkChangeCurrentStateFn, readNetworkSourceStateFn = oldCurrent, oldRead }) + networkChangeCurrentStateFn = func(*netmon.ChangeDelta) *netmon.State { return cache } + readNetworkSourceStateFn = func() (*netmon.State, error) { return cache, nil } + monitor := netmon.NewStatic() + + h.prog.handleNetworkChange(&netmon.ChangeDelta{Monitor: monitor, Old: s0, New: s1}, false) + cache = s2 + h.prog.handleNetworkChange(&netmon.ChangeDelta{Monitor: monitor, Old: s0, New: s2}, true) + + if got := h.interfaceEventsFor(t, "ipsec0"); got != 1 { + t.Fatalf("ipsec0 interface events = %d, want 1", got) + } + if got := h.interfaceEventsFor(t, "en0"); got != 1 { + t.Fatalf("en0 interface events = %d, want 1", got) + } +} + +// Test_networkEventsNewValidInterfaceReconciles plugs a valid interface in +// with its address in one delta while the other valid interface is down. The +// new interface is active, so the change must reconcile. +func Test_networkEventsNewValidInterfaceReconciles(t *testing.T) { + h := newNetworkEventsHarness(t, "en0", "en7") + before := orderTestState("", "", map[string][]string{"en0": nil}, map[string]bool{"en0": true}) + before.HaveV4 = false + after := orderTestState("en7", "", map[string][]string{"en0": nil, "en7": {"192.0.2.30/24"}}, map[string]bool{"en0": true}) + + h.handle(before, after, false) + + if h.reconciled != 1 { + event := h.transitionEvent(t) + t.Fatalf("reconciled = %d, want 1; outcome %v", h.reconciled, event["outcome"]) + } +} diff --git a/cmd/cli/network_snapshot.go b/cmd/cli/network_snapshot.go new file mode 100644 index 00000000..9af1c022 --- /dev/null +++ b/cmd/cli/network_snapshot.go @@ -0,0 +1,301 @@ +package cli + +import ( + "cmp" + "net/netip" + "slices" + "strings" + + "github.com/rs/zerolog" + "tailscale.com/net/netmon" + "tailscale.com/net/tsaddr" +) + +// hardwarePortIPhoneUSB is the macOS port name of a phone that shares its +// connection over the cable. +const hardwarePortIPhoneUSB = "iPhone USB" + +// snapshotInterfaceLimit bounds the interface list of one snapshot. A +// container host holds hundreds of interfaces, and each entry costs bytes in +// every journal line and in every log file header. +const snapshotInterfaceLimit = 32 + +// v6GlobalPrefix holds the v6 addresses that reach the Internet. +var v6GlobalPrefix = netip.MustParsePrefix("2000::/3") + +// hotspotGatewayPrefix is the range that an iPhone hotspot hands out. A +// gateway inside it names a tethered host on every platform. +var hotspotGatewayPrefix = netip.MustParsePrefix("172.20.10.0/28") + +// ethernetPortWords name a wired port. macOS spells the same link in several +// ways, one per adapter family. +var ethernetPortWords = []string{"Ethernet", "LAN", "Thunderbolt"} + +// defaultRoute names the next hop and the interface of one default route. Both +// values come from the same entry of the route table, so a reader can trust +// the pair. +type defaultRoute struct { + Gateway string + Interface string +} + +// snapshotInterface is one interface of a network snapshot. +type snapshotInterface struct { + Name string + Class string + Up bool + IPs []string + HardwarePort string + Service string +} + +// networkSnapshot describes the host network at one moment. The journal event +// and the header of a log file render the same value. +type networkSnapshot struct { + DefaultRouteV4 string + DefaultRouteV6 string + GatewayV4 string + GatewayV6 string + HaveV4 bool + HaveV6 bool + Interfaces []snapshotInterface + InterfacesOmitted int + Resolvers []string + SourceIPv4 string + SourceIPv6 string + InterceptTarget string + BypassActive bool + RecoveryRunning bool + PFStabilizing bool + LinkType string + Tethered bool + CLATPresent bool + NAT64Prefix string + DNSLessTarget bool +} + +// snapshotInputs holds every value a snapshot needs. The caller reads them, so +// the builder stays free of locks and of system calls. +type snapshotInputs struct { + State *netmon.State + RouteV4 defaultRoute + RouteV6 defaultRoute + Resolvers []string + SourceIPv4 string + SourceIPv6 string + InterceptTarget string + BypassActive bool + RecoveryRunning bool + PFStabilizing bool + NAT64Prefix string + CLATPresent bool + Meta interfaceMetaFunc +} + +// buildNetworkSnapshot makes the snapshot of one moment. It is pure, so a +// caller can take the inputs under a lock and build outside it. +func buildNetworkSnapshot(in snapshotInputs) networkSnapshot { + snapshot := networkSnapshot{ + GatewayV4: in.RouteV4.Gateway, + GatewayV6: in.RouteV6.Gateway, + DefaultRouteV4: in.RouteV4.Interface, + DefaultRouteV6: in.RouteV6.Interface, + Resolvers: in.Resolvers, + SourceIPv4: in.SourceIPv4, + SourceIPv6: in.SourceIPv6, + InterceptTarget: in.InterceptTarget, + BypassActive: in.BypassActive, + RecoveryRunning: in.RecoveryRunning, + PFStabilizing: in.PFStabilizing, + NAT64Prefix: in.NAT64Prefix, + CLATPresent: in.CLATPresent, + DNSLessTarget: in.InterceptTarget != "", + } + + routeMeta := interfaceMeta{} + var routeAddresses []netip.Prefix + if in.State != nil { + meta := interfaceMetaWithClass(in.Meta) + routeMeta = meta(in.State.DefaultRouteInterface) + routeAddresses = in.State.InterfaceIPs[in.State.DefaultRouteInterface] + snapshot.HaveV4, snapshot.HaveV6 = in.State.HaveV4, in.State.HaveV6 + snapshot.Interfaces, snapshot.InterfacesOmitted = snapshotInterfaces(in.State, meta) + if snapshot.DefaultRouteV4 == "" { + snapshot.DefaultRouteV4 = defaultRouteV4Interface(in.State) + } + } + snapshot.LinkType = linkTypeFor(routeMeta.HardwarePort, routeMeta.Class) + snapshot.Tethered = tetheredNetwork(in.RouteV4.Gateway, routeMeta.HardwarePort, routeAddresses) + return snapshot +} + +// interfaceMetaWithClass fills the class that the platform metadata leaves +// empty. A host that names no port still tells a tunnel from a hardware port +// by the interface name. +func interfaceMetaWithClass(meta interfaceMetaFunc) interfaceMetaFunc { + return func(name string) interfaceMeta { + info := interfaceMeta{} + if meta != nil { + info = meta(name) + } + if info.Class == "" { + info.Class = interfaceClass(name, false) + } + return info + } +} + +// snapshotInterfaces lists the interfaces of the state, sorted by name, and +// counts the ones it leaves out. A reader compares two snapshots line by line, +// so the order must not move. An interface that is down and holds no address +// carries no traffic, and a container host holds hundreds of them. +func snapshotInterfaces(state *netmon.State, meta interfaceMetaFunc) (list []snapshotInterface, omitted int) { + list = make([]snapshotInterface, 0, len(state.Interface)) + for name, iface := range state.Interface { + up := iface.Interface != nil && iface.IsUp() + ips := prefixStrings(state.InterfaceIPs[name]) + if !up && len(ips) == 0 { + omitted++ + continue + } + info := meta(name) + list = append(list, snapshotInterface{ + Name: name, + Class: info.Class, + Up: up, + IPs: ips, + HardwarePort: info.HardwarePort, + Service: info.Service, + }) + } + slices.SortFunc(list, func(a, b snapshotInterface) int { + return cmp.Compare(a.Name, b.Name) + }) + if len(list) > snapshotInterfaceLimit { + omitted += len(list) - snapshotInterfaceLimit + list = list[:snapshotInterfaceLimit] + } + return list, omitted +} + +// defaultRouteV4Interface names the interface that the network monitor holds +// for the default route. It fills the v4 route of a platform that reads no +// route table. An interface without a usable v4 address carries no v4 traffic, +// whatever the monitor says. +func defaultRouteV4Interface(state *netmon.State) string { + route := state.DefaultRouteInterface + if route == "" { + return "" + } + for _, prefix := range state.InterfaceIPs[route] { + if usableV4(prefix.Addr().Unmap()) { + return route + } + } + return "" +} + +// globalAddress reports whether an address reaches another host. +func globalAddress(address netip.Addr) bool { + return address.IsGlobalUnicast() && !address.IsLoopback() && !address.IsLinkLocalUnicast() +} + +// usableV4 and usableV6 grade one address the way the network monitor grades +// it. A fresh read of the interfaces must report the same families as a +// monitor callback, or two snapshots of one moment disagree. The monitor +// counts a unique local address of the v6 family, because some networks reach +// the Internet from it. The range of the Tailscale mesh is the exception, +// because that mesh carries no Internet traffic. +func usableV4(address netip.Addr) bool { + return address.Is4() && globalAddress(address) +} + +func usableV6(address netip.Addr) bool { + if !address.Is6() { + return false + } + if v6GlobalPrefix.Contains(address) { + return true + } + return address.IsPrivate() && !tsaddr.TailscaleULARange().Contains(address) +} + +// tetheredNetwork reports whether the host runs on a phone connection. A +// tethered host pays for its traffic and loses the link on every phone move. +// A platform that reads no route table gives no gateway, so the address that +// the phone handed to the host tells the same story. +func tetheredNetwork(gatewayV4, hardwarePort string, routeAddresses []netip.Prefix) bool { + if hardwarePort == hardwarePortIPhoneUSB { + return true + } + if gateway, err := netip.ParseAddr(gatewayV4); err == nil && hotspotGatewayPrefix.Contains(gateway.Unmap()) { + return true + } + for _, prefix := range routeAddresses { + if hotspotGatewayPrefix.Contains(prefix.Addr().Unmap()) { + return true + } + } + return false +} + +// linkTypeFor names the link of the default route. The port name tells more +// than the interface name, which is only a number on macOS. +func linkTypeFor(hardwarePort, class string) string { + switch { + case hardwarePort == "Wi-Fi": + return "wifi" + case hardwarePort == hardwarePortIPhoneUSB: + return "usb_tether" + case ethernetPort(hardwarePort): + return "ethernet" + case class == "tunnel": + return "tunnel" + } + return "unknown" +} + +func ethernetPort(hardwarePort string) bool { + for _, word := range ethernetPortWords { + if strings.Contains(hardwarePort, word) { + return true + } + } + return false +} + +// snapshotDict renders the snapshot with the field names of the spec. The +// journal event and the header line share it, so both stay in one shape. +func snapshotDict(snapshot networkSnapshot) *zerolog.Event { + interfaces := zerolog.Arr() + for _, iface := range snapshot.Interfaces { + interfaces = interfaces.Dict(zerolog.Dict(). + Str("name", iface.Name). + Str("class", iface.Class). + Bool("up", iface.Up). + Strs("ips", iface.IPs). + Str("hardware_port", iface.HardwarePort). + Str("service", iface.Service)) + } + return zerolog.Dict(). + Str("default_route_v4", snapshot.DefaultRouteV4). + Str("default_route_v6", snapshot.DefaultRouteV6). + Str("gateway_v4", snapshot.GatewayV4). + Str("gateway_v6", snapshot.GatewayV6). + Bool("have_v4", snapshot.HaveV4). + Bool("have_v6", snapshot.HaveV6). + Array("interfaces", interfaces). + Int("interfaces_omitted", snapshot.InterfacesOmitted). + Strs("resolvers", snapshot.Resolvers). + Str("source_ipv4", snapshot.SourceIPv4). + Str("source_ipv6", snapshot.SourceIPv6). + Str("intercept_target", snapshot.InterceptTarget). + Bool("bypass_active", snapshot.BypassActive). + Bool("recovery_running", snapshot.RecoveryRunning). + Bool("pf_stabilizing", snapshot.PFStabilizing). + Str("link_type", snapshot.LinkType). + Bool("tethered", snapshot.Tethered). + Bool("clat_present", snapshot.CLATPresent). + Str("nat64_prefix", snapshot.NAT64Prefix). + Bool("dns_less_target", snapshot.DNSLessTarget) +} diff --git a/cmd/cli/network_snapshot_prog.go b/cmd/cli/network_snapshot_prog.go new file mode 100644 index 00000000..22b1f435 --- /dev/null +++ b/cmd/cli/network_snapshot_prog.go @@ -0,0 +1,226 @@ +package cli + +import ( + "net" + "reflect" + "slices" + "sync" + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// networkSnapshotMessage is the message of the snapshot event. A log tool +// selects every snapshot of a run by this message. +const networkSnapshotMessage = "Network snapshot" + +// snapshotTriggerStart names the snapshot of the start of a run. +const snapshotTriggerStart = "start" + +// snapshotMinInterval bounds the snapshots of a flapping network. +const snapshotMinInterval = 60 * time.Second + +// snapshotNowFn is the clock of the snapshot limit. A test replaces it. +var snapshotNowFn = time.Now + +// defaultRoutesFn reads the default route of each family. A test replaces it, +// because the route table of the host is not a fixture. +var defaultRoutesFn = defaultRoutes + +// platformInterfaceMetaFn reads the platform names of one interface. A test +// replaces it, because the ports of the host are not a fixture. +var platformInterfaceMetaFn = platformInterfaceMeta + +// The delta diff has to know the virtual adapters of the host, and only the +// platform file reads them. +func init() { + virtualInterfaceSetFn = platformVirtualInterfaces +} + +// interfaceMetaFor describes one interface for the snapshot and for the delta +// diff. +func interfaceMetaFor(name string) interfaceMeta { + platformClass, hardwarePort, service := platformInterfaceMetaFn(name) + meta := interfaceMeta{ + Class: interfaceClassFor(name, platformClass), + HardwarePort: hardwarePort, + Service: service, + } + meta.LinkType = linkTypeFor(hardwarePort, meta.Class) + return meta +} + +// interfaceClassFor picks the class of one interface. A name rule wins for the +// loopback, the AirDrop, and the tunnel interfaces, because Windows describes +// a tunnel adapter as an Ethernet adapter. +func interfaceClassFor(name, platformClass string) string { + if class := interfaceClass(name, false); class != "hardware" { + return class + } + if platformClass != "" { + return platformClass + } + if _, virtual := virtualInterfaceSetFn()[name]; virtual { + return "virtual" + } + return "hardware" +} + +// snapshotInputs reads the sources of one snapshot from the network that the +// last monitor callback reported. No callback ran before the first header of a +// run, so that header reads the network itself. +func (p *prog) snapshotInputs() snapshotInputs { + state := p.lastNetworkState.Load() + if state == nil { + state = freshNetworkState() + } + return p.snapshotInputsFor(state) +} + +// freshSnapshotInputs reads the sources of one snapshot from the network of +// this moment. Support reads the header of an upload to learn the network of +// the moment the user asked, which can differ from the last callback. +func (p *prog) freshSnapshotInputs() snapshotInputs { + return p.snapshotInputsFor(freshNetworkState()) +} + +// networkSnapshot describes the host network of this moment. +func (p *prog) networkSnapshot() networkSnapshot { + return buildNetworkSnapshot(p.snapshotInputs()) +} + +// logNetworkSnapshot puts the host network in the journal. Support reads the +// snapshot of each trigger to learn the network at the time of an outage. +func (p *prog) logNetworkSnapshot(trigger string) { + snapshot := p.networkSnapshot() + if !p.snapshotWrites.allow(trigger, snapshot, snapshotNowFn()) { + return + } + journal(mainLog.Load().Info()). + Str("trigger", trigger). + Dict("network", snapshotDict(snapshot)). + Msg(networkSnapshotMessage) +} + +// snapshotLimiter holds the snapshot that reached the journal last, so a burst +// of triggers writes one line. +type snapshotLimiter struct { + mu sync.Mutex + last networkSnapshot + written time.Time + seen bool +} + +// allow reports whether one snapshot reaches the journal, and records the +// snapshots that do. +func (s *snapshotLimiter) allow(trigger string, snapshot networkSnapshot, now time.Time) bool { + s.mu.Lock() + defer s.mu.Unlock() + if !s.permits(trigger, snapshot, now) { + return false + } + s.last, s.written, s.seen = snapshot, now, true + return true +} + +// permits decides one snapshot. The first snapshot of a run tells a reader +// where the run started. A snapshot equal to the one before it carries +// nothing. A new default route and a new resolver set explain an outage, so +// they pass at once. Every other snapshot waits out the interval, because a +// flapping interface writes some kilobytes per flap. +func (s *snapshotLimiter) permits(trigger string, snapshot networkSnapshot, now time.Time) bool { + if trigger == snapshotTriggerStart || !s.seen { + return true + } + if sameNetworkSnapshot(s.last, snapshot) { + return false + } + if routeOrResolversChanged(s.last, snapshot) { + return true + } + return now.Sub(s.written) >= snapshotMinInterval +} + +// sameNetworkSnapshot compares two snapshots field by field. The trigger is no +// part of a snapshot, so two triggers of one network give one line. +func sameNetworkSnapshot(before, after networkSnapshot) bool { + return reflect.DeepEqual(before, after) +} + +// routeOrResolversChanged names the changes that a reader of an outage needs +// at once. +func routeOrResolversChanged(before, after networkSnapshot) bool { + return before.DefaultRouteV4 != after.DefaultRouteV4 || + before.DefaultRouteV6 != after.DefaultRouteV6 || + !slices.Equal(before.Resolvers, after.Resolvers) +} + +// snapshotInputsFor reads every source of one snapshot. It takes no writer +// lock, so any path of the daemon can log a snapshot. +func (p *prog) snapshotInputsFor(state *netmon.State) snapshotInputs { + routeV4, routeV6 := defaultRoutesFn() + inputs := snapshotInputs{ + State: state, + RouteV4: routeV4, + RouteV6: routeV6, + Resolvers: ctrld.OsResolverNameservers(), + SourceIPv4: addressString(ctrld.GetDefaultLocalIPv4()), + SourceIPv6: addressString(ctrld.GetDefaultLocalIPv6()), + InterceptTarget: p.interceptTargetSnapshot(), + BypassActive: p.recoveryBypass.Load(), + RecoveryRunning: p.recoveryRunning.Load(), + PFStabilizing: p.pfStabilizing.Load(), + Meta: interfaceMetaFor, + } + if prefix, ok := p.dns64PrefixSnapshot(); ok { + inputs.NAT64Prefix = prefix.String() + } + if state != nil { + inputs.CLATPresent = dns64StateHasCLAT(state) + } + return inputs +} + +// freshNetworkState reads the interfaces of the host now. netmon fills the +// route and the families of a callback state, so a read of this package has to +// fill them itself. +func freshNetworkState() *netmon.State { + state, err := readNetworkSourceStateFn() + if err != nil || state == nil { + return nil + } + fresh := *state + if route, err := netmon.DefaultRouteInterface(); err == nil { + fresh.DefaultRouteInterface = route + } + fresh.HaveV4, fresh.HaveV6 = reachableAddressFamilies(&fresh) + return &fresh +} + +// reachableAddressFamilies reports which families the host can reach. It +// grades each address with the rule of the network monitor, so the first +// header of a run reports the same families as the first callback. +func reachableAddressFamilies(state *netmon.State) (haveV4, haveV6 bool) { + for name, iface := range state.Interface { + if iface.Interface == nil || !iface.IsUp() { + continue + } + for _, prefix := range state.InterfaceIPs[name] { + address := prefix.Addr().Unmap() + haveV4 = haveV4 || usableV4(address) + haveV6 = haveV6 || usableV6(address) + } + } + return haveV4, haveV6 +} + +// addressString names one source address. An address that ctrld does not hold +// leaves an empty field, because net.IP prints an empty value as the word for +// a nil address. +func addressString(ip net.IP) string { + if len(ip) == 0 { + return "" + } + return ip.String() +} diff --git a/cmd/cli/network_snapshot_prog_test.go b/cmd/cli/network_snapshot_prog_test.go new file mode 100644 index 00000000..7d3ab15f --- /dev/null +++ b/cmd/cli/network_snapshot_prog_test.go @@ -0,0 +1,348 @@ +package cli + +import ( + "net" + "net/netip" + "reflect" + "slices" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +// stubSnapshotPlatformMeta makes the platform names of each interface a +// fixture, so no test depends on the ports of the host. +func stubSnapshotPlatformMeta(t *testing.T, meta map[string]interfaceMeta) { + t.Helper() + orig := platformInterfaceMetaFn + t.Cleanup(func() { platformInterfaceMetaFn = orig }) + platformInterfaceMetaFn = func(name string) (class, hardwarePort, service string) { + info := meta[name] + return info.Class, info.HardwarePort, info.Service + } +} + +// stubSnapshotVirtualSet names the virtual adapters of the fixture host. +func stubSnapshotVirtualSet(t *testing.T, names ...string) { + t.Helper() + orig := virtualInterfaceSetFn + t.Cleanup(func() { virtualInterfaceSetFn = orig }) + virtual := make(map[string]struct{}, len(names)) + for _, name := range names { + virtual[name] = struct{}{} + } + virtualInterfaceSetFn = func() map[string]struct{} { return virtual } +} + +// stubSnapshotGateways replaces the route table read, because the gateways of +// the host are not a fixture. +func stubSnapshotGateways(t *testing.T, v4, v6 string) { + t.Helper() + stubSnapshotRoutes(t, defaultRoute{Gateway: v4}, defaultRoute{Gateway: v6}) +} + +// stubSnapshotRoutes replaces the route table read with one default route per +// family. +func stubSnapshotRoutes(t *testing.T, v4, v6 defaultRoute) { + t.Helper() + orig := defaultRoutesFn + t.Cleanup(func() { defaultRoutesFn = orig }) + defaultRoutesFn = func() (defaultRoute, defaultRoute) { return v4, v6 } +} + +// stubSnapshotDNS64 reports a dual-stack host, so the NAT64 check starts no +// discovery and touches no name server. +func stubSnapshotDNS64(t *testing.T) { + t.Helper() + orig := dns64NetworkClassFn + t.Cleanup(func() { dns64NetworkClassFn = orig }) + dns64NetworkClassFn = func() (hasUsableIPv4, hasCLAT bool, err error) { return true, false, nil } +} + +// stubSnapshotSources sets the local source addresses and puts the values of +// the host back when the test ends. +func stubSnapshotSources(t *testing.T, v4, v6 string) { + t.Helper() + origV4, origV6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + t.Cleanup(func() { + ctrld.SetDefaultLocalIPv4(origV4) + ctrld.SetDefaultLocalIPv6(origV6) + }) + ctrld.SetDefaultLocalIPv4(net.ParseIP(v4)) + ctrld.SetDefaultLocalIPv6(net.ParseIP(v6)) +} + +// tetheredSnapshotProg returns a prog on a phone hotspot over the cable. +func tetheredSnapshotProg(t *testing.T) *prog { + t.Helper() + stubSnapshotDNS64(t) + stubSnapshotGateways(t, "172.20.10.1", "") + stubSnapshotPlatformMeta(t, map[string]interfaceMeta{ + "en7": {HardwarePort: hardwarePortIPhoneUSB, Service: "iPhone USB"}, + }) + stubSnapshotVirtualSet(t) + stubSnapshotSources(t, "172.20.10.5", "") + + p := &prog{} + p.lastNetworkState.Store(&netmon.State{ + DefaultRouteInterface: "en7", + HaveV4: true, + Interface: map[string]netmon.Interface{ + "en7": {Interface: &net.Interface{Name: "en7", Flags: net.FlagUp}}, + "lo0": {Interface: &net.Interface{Name: "lo0", Flags: net.FlagUp}}, + }, + InterfaceIPs: map[string][]netip.Prefix{ + "en7": {netip.MustParsePrefix("172.20.10.5/28")}, + "lo0": {netip.MustParsePrefix("127.0.0.1/8")}, + }, + }) + p.recoveryBypass.Store(true) + p.pfStabilizing.Store(true) + p.interceptDNSTargetSetValue = "127.0.0.1" + p.publishInterceptTarget("127.0.0.1") + return p +} + +func Test_networkSnapshotFromTheStoredState(t *testing.T) { + p := tetheredSnapshotProg(t) + + snapshot := p.networkSnapshot() + + checks := []struct { + field string + got, want any + }{ + {"default_route_v4", snapshot.DefaultRouteV4, "en7"}, + {"default_route_v6", snapshot.DefaultRouteV6, ""}, + {"gateway_v4", snapshot.GatewayV4, "172.20.10.1"}, + {"have_v4", snapshot.HaveV4, true}, + {"source_ipv4", snapshot.SourceIPv4, "172.20.10.5"}, + {"source_ipv6", snapshot.SourceIPv6, ""}, + {"intercept_target", snapshot.InterceptTarget, "127.0.0.1"}, + {"dns_less_target", snapshot.DNSLessTarget, true}, + {"bypass_active", snapshot.BypassActive, true}, + {"recovery_running", snapshot.RecoveryRunning, false}, + {"pf_stabilizing", snapshot.PFStabilizing, true}, + {"link_type", snapshot.LinkType, "usb_tether"}, + {"tethered", snapshot.Tethered, true}, + {"nat64_prefix", snapshot.NAT64Prefix, ""}, + {"clat_present", snapshot.CLATPresent, false}, + } + for _, c := range checks { + if c.got != c.want { + t.Errorf("%s = %v, want %v", c.field, c.got, c.want) + } + } + + want := []snapshotInterface{ + {Name: "en7", Class: "hardware", Up: true, IPs: []string{"172.20.10.5/28"}, HardwarePort: hardwarePortIPhoneUSB, Service: "iPhone USB"}, + {Name: "lo0", Class: "loopback", Up: true, IPs: []string{"127.0.0.1/8"}}, + } + if !reflect.DeepEqual(snapshot.Interfaces, want) { + t.Errorf("interfaces = %+v, want %+v", snapshot.Interfaces, want) + } +} + +// Test_networkSnapshotReadsTheNetworkWithoutAStoredState covers the first +// header of a run. No network callback reached ctrld yet, so the snapshot must +// read the interfaces itself and name the families it finds. +func Test_networkSnapshotReadsTheNetworkWithoutAStoredState(t *testing.T) { + stubSnapshotDNS64(t) + stubSnapshotGateways(t, "192.0.2.1", "") + stubSnapshotPlatformMeta(t, nil) + stubSnapshotSources(t, "", "") + stubLogHeaderNetworkRead(t, "en9") + p := &prog{} + + snapshot := p.networkSnapshot() + + if len(snapshot.Interfaces) != 1 || snapshot.Interfaces[0].Name != "en9" || !snapshot.Interfaces[0].Up { + t.Fatalf("interfaces = %+v, want one up en9", snapshot.Interfaces) + } + if !snapshot.HaveV4 { + t.Error("have_v4 = false, want true for a global unicast v4 address") + } + if snapshot.HaveV6 { + t.Error("have_v6 = true, want false without a v6 address") + } +} + +// Test_reachableAddressFamiliesGradesTheV6AddressLikeNetmon covers a host on a +// unique local address. The network monitor counts such an address as usable, +// so a fresh read must count it too. +func Test_reachableAddressFamiliesGradesTheV6AddressLikeNetmon(t *testing.T) { + for _, tt := range []struct { + name string + ips []string + wantV6 bool + }{ + {name: "unique_local", ips: []string{"fd00::2/64"}, wantV6: true}, + {name: "tailscale_range", ips: []string{"fd7a:115c:a1e0::2/128"}}, + {name: "global", ips: []string{"2001:db8::2/64"}, wantV6: true}, + {name: "link_local", ips: []string{"fe80::2/64"}}, + } { + t.Run(tt.name, func(t *testing.T) { + state := snapshotTestState("en0", snapshotTestInterface{"en0", true, tt.ips}) + + _, haveV6 := reachableAddressFamilies(state) + + if haveV6 != tt.wantV6 { + t.Errorf("have_v6 = %v, want %v for %v", haveV6, tt.wantV6, tt.ips) + } + }) + } +} + +func Test_networkSnapshotLogsOneJournalLine(t *testing.T) { + logs := captureDebugMainLog(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("start") + + events := jsonLogEvents(t, logs, networkSnapshotMessage) + if len(events) != 1 { + t.Fatalf("%q lines = %d, want 1", networkSnapshotMessage, len(events)) + } + wantField(t, events[0], "trigger", "start") + wantField(t, events[0], journalField, true) + wantField(t, events[0], "level", "info") + network, ok := events[0]["network"].(map[string]any) + if !ok { + t.Fatalf("network = %v, want an object", events[0]["network"]) + } + wantField(t, network, "default_route_v4", "en7") + wantField(t, network, "tethered", true) + wantField(t, network, "link_type", "usb_tether") +} + +// snapshotTestClock hands out the time that a test sets, because the snapshot +// limit counts seconds between two lines. +type snapshotTestClock struct { + now time.Time +} + +// stubSnapshotClock puts the clock of the snapshot limit under the control of +// one test. +func stubSnapshotClock(t *testing.T) *snapshotTestClock { + t.Helper() + clock := &snapshotTestClock{now: time.Date(2026, 9, 17, 10, 0, 0, 0, time.UTC)} + orig := snapshotNowFn + t.Cleanup(func() { snapshotNowFn = orig }) + snapshotNowFn = func() time.Time { return clock.now } + return clock +} + +func (c *snapshotTestClock) advance(d time.Duration) { c.now = c.now.Add(d) } + +// snapshotTriggerLines names the trigger of every snapshot line, in order. +func snapshotTriggerLines(t *testing.T, logs *syncBuffer) []string { + t.Helper() + var triggers []string + for _, event := range jsonLogEvents(t, logs, networkSnapshotMessage) { + trigger, _ := event["trigger"].(string) + triggers = append(triggers, trigger) + } + return triggers +} + +// Test_logNetworkSnapshotSkipsAnEqualSnapshot covers a recovery on a network +// that did not change. The begin and the end of that recovery render the same +// object, so the second line carries nothing. +func Test_logNetworkSnapshotSkipsAnEqualSnapshot(t *testing.T) { + logs := captureDebugMainLog(t) + clock := stubSnapshotClock(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("recovery_begin") + clock.advance(5 * time.Minute) + p.logNetworkSnapshot("recovery_end") + + if got := snapshotTriggerLines(t, logs); !slices.Equal(got, []string{"recovery_begin"}) { + t.Errorf("snapshot triggers = %v, want the first line alone", got) + } +} + +// Test_logNetworkSnapshotCapsTheRate covers a flapping interface. Two +// snapshots that differ in no route and in no resolver give one line per +// minute. +func Test_logNetworkSnapshotCapsTheRate(t *testing.T) { + logs := captureDebugMainLog(t) + clock := stubSnapshotClock(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("transition") + clock.advance(30 * time.Second) + p.recoveryRunning.Store(true) + p.logNetworkSnapshot("transition") + clock.advance(30 * time.Second) + p.logNetworkSnapshot("transition") + + if got := snapshotTriggerLines(t, logs); len(got) != 2 { + t.Errorf("snapshot lines = %d, want 2: %v", len(got), got) + } +} + +// Test_logNetworkSnapshotWritesARouteChangeAtOnce covers the change that +// explains an outage. The cap must not hold it back. +func Test_logNetworkSnapshotWritesARouteChangeAtOnce(t *testing.T) { + logs := captureDebugMainLog(t) + clock := stubSnapshotClock(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("transition") + clock.advance(time.Second) + stubSnapshotRoutes(t, defaultRoute{Gateway: "192.168.1.1", Interface: "en0"}, defaultRoute{}) + p.logNetworkSnapshot("transition") + clock.advance(time.Second) + p.recoveryRunning.Store(true) + p.logNetworkSnapshot("recovery_begin") + + if got := snapshotTriggerLines(t, logs); !slices.Equal(got, []string{"transition", "transition"}) { + t.Errorf("snapshot triggers = %v, want the two transitions", got) + } +} + +// Test_logNetworkSnapshotAlwaysWritesTheStartLine covers the first line of a +// run. A reader needs the network of the start, whatever came before it. +func Test_logNetworkSnapshotAlwaysWritesTheStartLine(t *testing.T) { + logs := captureDebugMainLog(t) + clock := stubSnapshotClock(t) + p := tetheredSnapshotProg(t) + + p.logNetworkSnapshot("transition") + clock.advance(time.Second) + p.logNetworkSnapshot(snapshotTriggerStart) + + if got := snapshotTriggerLines(t, logs); !slices.Equal(got, []string{"transition", "start"}) { + t.Errorf("snapshot triggers = %v, want the transition and the start", got) + } +} + +func Test_interfaceMetaFor(t *testing.T) { + stubSnapshotVirtualSet(t, "bridge100") + stubSnapshotPlatformMeta(t, map[string]interfaceMeta{ + "lo0": {Class: "hardware"}, + "en0": {HardwarePort: "Wi-Fi", Service: "Wi-Fi"}, + "eth0": {Class: "hardware", HardwarePort: "Intel I219-V"}, + }) + + for _, tt := range []struct { + name string + want interfaceMeta + }{ + {"lo0", interfaceMeta{Class: "loopback", LinkType: "unknown"}}, + {"awdl0", interfaceMeta{Class: "airdrop", LinkType: "unknown"}}, + {"utun3", interfaceMeta{Class: "tunnel", LinkType: "tunnel"}}, + {"en0", interfaceMeta{Class: "hardware", HardwarePort: "Wi-Fi", Service: "Wi-Fi", LinkType: "wifi"}}, + {"bridge100", interfaceMeta{Class: "virtual", LinkType: "unknown"}}, + {"eth0", interfaceMeta{Class: "hardware", HardwarePort: "Intel I219-V", LinkType: "unknown"}}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := interfaceMetaFor(tt.name); got != tt.want { + t.Errorf("interfaceMetaFor(%q) = %+v, want %+v", tt.name, got, tt.want) + } + }) + } +} diff --git a/cmd/cli/network_snapshot_test.go b/cmd/cli/network_snapshot_test.go new file mode 100644 index 00000000..e582e913 --- /dev/null +++ b/cmd/cli/network_snapshot_test.go @@ -0,0 +1,557 @@ +package cli + +import ( + "encoding/json" + "fmt" + "net" + "net/netip" + "slices" + "strings" + "testing" + + "github.com/rs/zerolog" + "tailscale.com/net/netmon" +) + +// snapshotTestInterface names one interface of a test network state. +type snapshotTestInterface struct { + name string + up bool + ips []string +} + +// snapshotTestState builds a netmon state with several interfaces, because the +// shared helper of the transition tests holds one interface only. +func snapshotTestState(route string, interfaces ...snapshotTestInterface) *netmon.State { + state := &netmon.State{ + DefaultRouteInterface: route, + Interface: map[string]netmon.Interface{}, + InterfaceIPs: map[string][]netip.Prefix{}, + } + for _, iface := range interfaces { + flags := net.Flags(0) + if iface.up { + flags = net.FlagUp + } + state.Interface[iface.name] = netmon.Interface{Interface: &net.Interface{Name: iface.name, Flags: flags}} + for _, ip := range iface.ips { + state.InterfaceIPs[iface.name] = append(state.InterfaceIPs[iface.name], netip.MustParsePrefix(ip)) + } + } + return state +} + +func snapshotTestMeta(meta map[string]interfaceMeta) interfaceMetaFunc { + return func(name string) interfaceMeta { return meta[name] } +} + +func Test_tetheredNetwork(t *testing.T) { + for _, tt := range []struct { + name string + gatewayV4 string + hardwarePort string + hostIPs []string + want bool + }{ + {name: "hotspot_gateway", gatewayV4: "172.20.10.1", hardwarePort: "Wi-Fi", want: true}, + {name: "hotspot_range_end", gatewayV4: "172.20.10.15", hardwarePort: "Wi-Fi", want: true}, + {name: "above_hotspot_range", gatewayV4: "172.20.10.16", hardwarePort: "Wi-Fi"}, + {name: "below_hotspot_range", gatewayV4: "172.20.9.255", hardwarePort: "Wi-Fi"}, + {name: "home_gateway", gatewayV4: "192.168.1.1", hardwarePort: "Wi-Fi"}, + {name: "iphone_usb_port", gatewayV4: "192.168.1.1", hardwarePort: "iPhone USB", want: true}, + {name: "no_gateway_no_port"}, + {name: "unparsable_gateway", gatewayV4: "not-an-address"}, + {name: "v6_gateway", gatewayV4: "fe80::1", hardwarePort: "Ethernet"}, + {name: "hotspot_host_address_without_a_gateway", hostIPs: []string{"172.20.10.4/28"}, want: true}, + {name: "hotspot_host_address_of_a_second_address", hostIPs: []string{"fe80::1/64", "172.20.10.4/28"}, want: true}, + {name: "home_host_address_without_a_gateway", hostIPs: []string{"192.168.1.20/24"}}, + } { + t.Run(tt.name, func(t *testing.T) { + hostIPs := make([]netip.Prefix, 0, len(tt.hostIPs)) + for _, ip := range tt.hostIPs { + hostIPs = append(hostIPs, netip.MustParsePrefix(ip)) + } + if got := tetheredNetwork(tt.gatewayV4, tt.hardwarePort, hostIPs); got != tt.want { + t.Fatalf("tetheredNetwork(%q, %q, %v) = %v, want %v", tt.gatewayV4, tt.hardwarePort, tt.hostIPs, got, tt.want) + } + }) + } +} + +// Test_buildNetworkSnapshot_tetheredWithoutAGateway covers Linux and Windows. +// Both read no route table in this release, so the address of the host is the +// only sign of a phone hotspot. +func Test_buildNetworkSnapshot_tetheredWithoutAGateway(t *testing.T) { + state := snapshotTestState("wlan0", snapshotTestInterface{"wlan0", true, []string{"172.20.10.4/28"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + if !snapshot.Tethered { + t.Error("tethered = false, want true for a host address of the hotspot range") + } +} + +// Test_usableV6 pins the grade of a v6 address to the rule of the network +// monitor. A fresh read and a monitor callback must report the same have_v6. +func Test_usableV6(t *testing.T) { + for _, tt := range []struct { + address string + want bool + }{ + {"2001:db8::1", true}, + {"2606:4700:4700::1111", true}, + {"fd00::1", true}, + {"fd7a:115c:a1e0::1", false}, + {"fe80::1", false}, + {"::1", false}, + {"192.0.2.10", false}, + } { + t.Run(tt.address, func(t *testing.T) { + if got := usableV6(netip.MustParseAddr(tt.address).Unmap()); got != tt.want { + t.Errorf("usableV6(%q) = %v, want %v", tt.address, got, tt.want) + } + }) + } +} + +func Test_linkTypeFor(t *testing.T) { + for _, tt := range []struct { + name string + hardwarePort string + class string + want string + }{ + {"wifi_port", "Wi-Fi", "hardware", "wifi"}, + {"ethernet_port", "Ethernet", "hardware", "ethernet"}, + {"thunderbolt_port", "Thunderbolt Ethernet Slot 1", "hardware", "ethernet"}, + {"usb_lan_port", "USB 10/100/1000 LAN", "hardware", "ethernet"}, + {"thunderbolt_bridge", "Thunderbolt Bridge", "hardware", "ethernet"}, + {"iphone_usb_port", "iPhone USB", "hardware", "usb_tether"}, + {"tunnel_class", "", "tunnel", "tunnel"}, + {"no_port_no_class", "", "", "unknown"}, + {"unknown_port", "Bluetooth PAN", "hardware", "unknown"}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := linkTypeFor(tt.hardwarePort, tt.class); got != tt.want { + t.Fatalf("linkTypeFor(%q, %q) = %q, want %q", tt.hardwarePort, tt.class, got, tt.want) + } + }) + } +} + +func Test_buildNetworkSnapshot_tetheredWiFiHotspot(t *testing.T) { + state := snapshotTestState("en0", snapshotTestInterface{"en0", true, []string{"172.20.10.2/28"}}) + state.HaveV4 = true + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + RouteV4: defaultRoute{Gateway: "172.20.10.1", Interface: "en0"}, + Meta: snapshotTestMeta(map[string]interfaceMeta{"en0": {Class: "hardware", HardwarePort: "Wi-Fi", Service: "Wi-Fi"}}), + }) + + if !snapshot.Tethered { + t.Error("tethered = false, want true for the hotspot gateway") + } + if snapshot.LinkType != "wifi" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "wifi") + } + if snapshot.DefaultRouteV4 != "en0" || snapshot.DefaultRouteV6 != "" { + t.Errorf("default routes = %q/%q, want en0 and empty", snapshot.DefaultRouteV4, snapshot.DefaultRouteV6) + } + if !snapshot.HaveV4 || snapshot.HaveV6 { + t.Errorf("have_v4/have_v6 = %v/%v, want true/false", snapshot.HaveV4, snapshot.HaveV6) + } + want := []snapshotInterface{{Name: "en0", Class: "hardware", Up: true, IPs: []string{"172.20.10.2/28"}, HardwarePort: "Wi-Fi", Service: "Wi-Fi"}} + if !slices.EqualFunc(snapshot.Interfaces, want, sameSnapshotInterface) { + t.Errorf("interfaces = %+v, want %+v", snapshot.Interfaces, want) + } +} + +func Test_buildNetworkSnapshot_usbTether(t *testing.T) { + state := snapshotTestState("en7", snapshotTestInterface{"en7", true, []string{"172.20.10.3/28"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + Meta: snapshotTestMeta(map[string]interfaceMeta{"en7": {Class: "hardware", HardwarePort: "iPhone USB", Service: "iPhone USB"}}), + }) + + if snapshot.LinkType != "usb_tether" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "usb_tether") + } + if !snapshot.Tethered { + t.Error("tethered = false, want true for the iPhone USB port") + } +} + +func Test_buildNetworkSnapshot_tunnelDefaultRoute(t *testing.T) { + state := snapshotTestState("utun3", snapshotTestInterface{"utun3", true, []string{"100.64.0.2/32"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + Meta: snapshotTestMeta(map[string]interfaceMeta{"utun3": {Class: "tunnel"}}), + }) + + if snapshot.LinkType != "tunnel" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "tunnel") + } + if snapshot.Tethered { + t.Error("tethered = true, want false without a hotspot gateway") + } +} + +// Test_buildNetworkSnapshot_defaultRouteV4WithoutARouteTable covers the +// platforms that read no route table. The v4 route falls back to the interface +// of the network monitor, and that interface must hold a usable v4 address. +func Test_buildNetworkSnapshot_defaultRouteV4WithoutARouteTable(t *testing.T) { + for _, tt := range []struct { + name string + ips []string + wantV4 string + otherIPs []string + }{ + {name: "dual_stack", ips: []string{"192.0.2.10/24", "2001:db8::10/64"}, wantV4: "en0"}, + {name: "v4_only", ips: []string{"192.0.2.10/24", "fe80::1/64"}, wantV4: "en0"}, + {name: "v6_only", ips: []string{"2001:db8::10/64"}}, + {name: "link_local_only", ips: []string{"169.254.1.2/16", "fe80::1/64"}}, + {name: "loopback_only", ips: []string{"127.0.0.1/8"}}, + {name: "address_on_another_interface", ips: nil, otherIPs: []string{"192.0.2.20/24", "2001:db8::20/64"}}, + } { + t.Run(tt.name, func(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en0", true, tt.ips}, + snapshotTestInterface{"en1", true, tt.otherIPs}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + if snapshot.DefaultRouteV4 != tt.wantV4 { + t.Errorf("default_route_v4 = %q, want %q", snapshot.DefaultRouteV4, tt.wantV4) + } + if snapshot.DefaultRouteV6 != "" { + t.Errorf("default_route_v6 = %q, want an empty value without a route table", snapshot.DefaultRouteV6) + } + }) + } +} + +// Test_buildNetworkSnapshot_defaultRoutesComeFromTheRouteTable covers a host +// whose v6 default route leaves over a tunnel. The v6 route must name that +// tunnel, not the interface of the v4 route. +func Test_buildNetworkSnapshot_defaultRoutesComeFromTheRouteTable(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24", "2001:db8::10/64"}}, + snapshotTestInterface{"utun4", true, []string{"2001:db8:1::2/64"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + RouteV4: defaultRoute{Gateway: "192.0.2.1", Interface: "en0"}, + RouteV6: defaultRoute{Gateway: "fe80::1%utun4", Interface: "utun4"}, + }) + + if snapshot.DefaultRouteV4 != "en0" || snapshot.DefaultRouteV6 != "utun4" { + t.Errorf("default routes = %q/%q, want en0 and utun4", snapshot.DefaultRouteV4, snapshot.DefaultRouteV6) + } + if snapshot.GatewayV4 != "192.0.2.1" || snapshot.GatewayV6 != "fe80::1%utun4" { + t.Errorf("gateways = %q/%q, want the gateways of the route table", snapshot.GatewayV4, snapshot.GatewayV6) + } +} + +func Test_buildNetworkSnapshot_sortsInterfacesAndTakesMeta(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en1", false, []string{"192.0.2.11/24"}}, + snapshotTestInterface{"awdl0", true, []string{"fe80::2/64"}}, + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + Meta: snapshotTestMeta(map[string]interfaceMeta{ + "en0": {Class: "hardware", HardwarePort: "Wi-Fi", Service: "Wi-Fi"}, + "awdl0": {Class: "airdrop"}, + "en1": {Class: "hardware", HardwarePort: "Ethernet", Service: "Ethernet"}, + }), + }) + + want := []snapshotInterface{ + {Name: "awdl0", Class: "airdrop", Up: true, IPs: []string{"fe80::2/64"}}, + {Name: "en0", Class: "hardware", Up: true, IPs: []string{"192.0.2.10/24"}, HardwarePort: "Wi-Fi", Service: "Wi-Fi"}, + {Name: "en1", Class: "hardware", Up: false, IPs: []string{"192.0.2.11/24"}, HardwarePort: "Ethernet", Service: "Ethernet"}, + } + if !slices.EqualFunc(snapshot.Interfaces, want, sameSnapshotInterface) { + t.Fatalf("interfaces = %+v, want %+v", snapshot.Interfaces, want) + } +} + +// Test_buildNetworkSnapshot_capsTheInterfaceList covers a container host. Such +// a host holds hundreds of interfaces, and a snapshot of all of them fills the +// journal. +func Test_buildNetworkSnapshot_capsTheInterfaceList(t *testing.T) { + interfaces := make([]snapshotTestInterface, 0, 40) + for i := range 36 { + interfaces = append(interfaces, snapshotTestInterface{fmt.Sprintf("veth%02d", i), true, nil}) + } + for i := range 4 { + interfaces = append(interfaces, snapshotTestInterface{fmt.Sprintf("down%d", i), false, nil}) + } + state := snapshotTestState("veth00", interfaces...) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + if len(snapshot.Interfaces) != snapshotInterfaceLimit { + t.Fatalf("interfaces = %d entries, want %d", len(snapshot.Interfaces), snapshotInterfaceLimit) + } + if snapshot.InterfacesOmitted != 8 { + t.Errorf("interfaces_omitted = %d, want 8", snapshot.InterfacesOmitted) + } + if first, last := snapshot.Interfaces[0].Name, snapshot.Interfaces[snapshotInterfaceLimit-1].Name; first != "veth00" || last != "veth31" { + t.Errorf("interfaces run from %q to %q, want veth00 to veth31", first, last) + } +} + +// Test_buildNetworkSnapshot_keepsUpOrAddressedInterfaces covers the rule that +// selects the interfaces of the list. +func Test_buildNetworkSnapshot_keepsUpOrAddressedInterfaces(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}, + snapshotTestInterface{"en1", true, nil}, + snapshotTestInterface{"en2", false, []string{"192.0.2.20/24"}}, + snapshotTestInterface{"en3", false, nil}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + var names []string + for _, iface := range snapshot.Interfaces { + names = append(names, iface.Name) + } + if !slices.Equal(names, []string{"en0", "en1", "en2"}) { + t.Errorf("interfaces = %v, want en0, en1, and en2", names) + } + if snapshot.InterfacesOmitted != 1 { + t.Errorf("interfaces_omitted = %d, want 1", snapshot.InterfacesOmitted) + } +} + +func Test_buildNetworkSnapshot_classesByNameWithoutMeta(t *testing.T) { + state := snapshotTestState("utun3", + snapshotTestInterface{"utun3", true, []string{"100.64.0.2/32"}}, + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state}) + + classes := map[string]string{} + for _, iface := range snapshot.Interfaces { + classes[iface.Name] = iface.Class + } + if classes["utun3"] != "tunnel" || classes["en0"] != "hardware" { + t.Fatalf("classes = %v, want utun3 tunnel and en0 hardware", classes) + } + if snapshot.LinkType != "tunnel" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "tunnel") + } +} + +func Test_buildNetworkSnapshot_fillsTheEmptyClassOfTheMeta(t *testing.T) { + state := snapshotTestState("utun3", + snapshotTestInterface{"utun3", true, []string{"100.64.0.2/32"}}, + snapshotTestInterface{"awdl0", true, []string{"fe80::2/64"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + Meta: snapshotTestMeta(map[string]interfaceMeta{"utun3": {HardwarePort: ""}}), + }) + + classes := map[string]string{} + for _, iface := range snapshot.Interfaces { + classes[iface.Name] = iface.Class + } + if classes["utun3"] != "tunnel" || classes["awdl0"] != "airdrop" { + t.Fatalf("classes = %v, want utun3 tunnel and awdl0 airdrop", classes) + } + if snapshot.LinkType != "tunnel" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "tunnel") + } +} + +func Test_buildNetworkSnapshot_nilStateKeepsFlags(t *testing.T) { + snapshot := buildNetworkSnapshot(snapshotInputs{ + RouteV4: defaultRoute{Gateway: "192.168.1.1"}, + RouteV6: defaultRoute{Gateway: "fe80::1"}, + Resolvers: []string{"192.168.1.1:53"}, + SourceIPv4: "192.168.1.20", + SourceIPv6: "2001:db8::20", + InterceptTarget: "127.0.0.1", + BypassActive: true, + RecoveryRunning: true, + PFStabilizing: true, + NAT64Prefix: "64:ff9b::/96", + CLATPresent: true, + }) + + if len(snapshot.Interfaces) != 0 { + t.Errorf("interfaces = %+v, want none", snapshot.Interfaces) + } + if snapshot.DefaultRouteV4 != "" || snapshot.DefaultRouteV6 != "" || snapshot.HaveV4 || snapshot.HaveV6 { + t.Errorf("state fields are set without a state: %+v", snapshot) + } + if snapshot.LinkType != "unknown" { + t.Errorf("link_type = %q, want %q", snapshot.LinkType, "unknown") + } + if !snapshot.BypassActive || !snapshot.RecoveryRunning || !snapshot.PFStabilizing || !snapshot.CLATPresent { + t.Errorf("flags dropped: %+v", snapshot) + } + if !snapshot.DNSLessTarget { + t.Error("dns_less_target = false, want true with an intercept target") + } + if snapshot.GatewayV4 != "192.168.1.1" || snapshot.GatewayV6 != "fe80::1" { + t.Errorf("gateways = %q/%q, want the input values", snapshot.GatewayV4, snapshot.GatewayV6) + } + if snapshot.SourceIPv4 != "192.168.1.20" || snapshot.SourceIPv6 != "2001:db8::20" { + t.Errorf("sources = %q/%q, want the input values", snapshot.SourceIPv4, snapshot.SourceIPv6) + } + if snapshot.NAT64Prefix != "64:ff9b::/96" || !slices.Equal(snapshot.Resolvers, []string{"192.168.1.1:53"}) { + t.Errorf("prefix and resolvers dropped: %+v", snapshot) + } +} + +func Test_buildNetworkSnapshot_noInterceptTargetIsNotDNSLess(t *testing.T) { + snapshot := buildNetworkSnapshot(snapshotInputs{State: snapshotTestState("en0", snapshotTestInterface{"en0", true, nil})}) + if snapshot.DNSLessTarget { + t.Error("dns_less_target = true, want false without an intercept target") + } +} + +// snapshotDictKeys are the field names that spec R2.2 names for the snapshot, +// plus the count of the interfaces that the list leaves out. +var snapshotDictKeys = []string{ + "default_route_v4", "default_route_v6", "gateway_v4", "gateway_v6", + "have_v4", "have_v6", "interfaces", "interfaces_omitted", "resolvers", + "source_ipv4", "source_ipv6", "intercept_target", "bypass_active", + "recovery_running", "pf_stabilizing", "link_type", "tethered", + "clat_present", "nat64_prefix", "dns_less_target", +} + +var snapshotInterfaceDictKeys = []string{"class", "hardware_port", "ips", "name", "service", "up"} + +// renderSnapshotDict writes the dict under one field and parses the line, so +// the test reads the keys that a reader of the journal sees. +func renderSnapshotDict(t *testing.T, snapshot networkSnapshot) (string, map[string]any) { + t.Helper() + buf := &syncBuffer{} + logger := zerolog.New(buf) + logger.Log().Dict("network", snapshotDict(snapshot)).Msg("Network snapshot") + + line := strings.TrimSpace(buf.String()) + event := map[string]any{} + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatalf("log line is not JSON: %q: %v", line, err) + } + network, ok := event["network"].(map[string]any) + if !ok { + t.Fatalf("line has no network object: %q", line) + } + return line, network +} + +func Test_snapshotDict_rendersEveryKeyOnce(t *testing.T) { + line, network := renderSnapshotDict(t, buildNetworkSnapshot(snapshotInputs{})) + + if got := sortedKeys(network); !slices.Equal(got, slices.Sorted(slices.Values(snapshotDictKeys))) { + t.Fatalf("keys = %v, want %v", got, snapshotDictKeys) + } + for _, key := range snapshotDictKeys { + if count := strings.Count(line, `"`+key+`":`); count != 1 { + t.Errorf("key %q appears %d times, want 1: %s", key, count, line) + } + } + if network["interfaces"] == nil || len(network["interfaces"].([]any)) != 0 { + t.Errorf("interfaces = %v, want an empty array", network["interfaces"]) + } + if network["resolvers"] == nil || len(network["resolvers"].([]any)) != 0 { + t.Errorf("resolvers = %v, want an empty array", network["resolvers"]) + } +} + +func Test_snapshotDict_rendersSortedInterfaces(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en1", true, nil}, + snapshotTestInterface{"awdl0", true, []string{"fe80::2/64"}}, + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{ + State: state, + RouteV4: defaultRoute{Gateway: "192.0.2.1", Interface: "en0"}, + Resolvers: []string{"192.0.2.1:53"}, + Meta: snapshotTestMeta(map[string]interfaceMeta{"en0": {Class: "hardware", HardwarePort: "Wi-Fi", Service: "Wi-Fi"}}), + }) + _, network := renderSnapshotDict(t, snapshot) + + interfaces, ok := network["interfaces"].([]any) + if !ok || len(interfaces) != 3 { + t.Fatalf("interfaces = %v, want three entries", network["interfaces"]) + } + var names []string + for _, entry := range interfaces { + object, ok := entry.(map[string]any) + if !ok { + t.Fatalf("interface entry is not an object: %v", entry) + } + if got := sortedKeys(object); !slices.Equal(got, snapshotInterfaceDictKeys) { + t.Fatalf("interface keys = %v, want %v", got, snapshotInterfaceDictKeys) + } + names = append(names, object["name"].(string)) + } + if !slices.Equal(names, []string{"awdl0", "en0", "en1"}) { + t.Errorf("interface order = %v, want awdl0, en0, en1", names) + } + if ips, ok := interfaces[2].(map[string]any)["ips"].([]any); !ok || len(ips) != 0 { + t.Errorf("ips of en1 = %v, want an empty array", interfaces[2].(map[string]any)["ips"]) + } + wantValues := map[string]any{"gateway_v4": "192.0.2.1", "link_type": "wifi", "default_route_v4": "en0", "tethered": false} + for key, want := range wantValues { + wantField(t, network, key, want) + } + if resolvers, ok := network["resolvers"].([]any); !ok || len(resolvers) != 1 || resolvers[0] != "192.0.2.1:53" { + t.Errorf("resolvers = %v, want one entry", network["resolvers"]) + } +} + +func Test_defaultRoutes_returnAddressesOrNothing(t *testing.T) { + v4, v6 := defaultRoutes() + // A host without a route of a family gives an empty value, so the test + // logs the pair and holds every value it gets to the format. + t.Logf("default routes: v4 %+v, v6 %+v", v4, v6) + + if v4.Gateway != "" { + address, err := netip.ParseAddr(v4.Gateway) + if err != nil || !address.Is4() { + t.Errorf("gateway_v4 = %q, want an IPv4 address or nothing: %v", v4.Gateway, err) + } + } + if v6.Gateway != "" { + address, err := netip.ParseAddr(v6.Gateway) + if err != nil || !address.Is6() { + t.Errorf("gateway_v6 = %q, want an IPv6 address or nothing: %v", v6.Gateway, err) + } + } + // A tunnel default has no next hop, so an interface without a gateway is a + // valid pair. A gateway without an interface is not. + if v4.Gateway != "" && v4.Interface == "" { + t.Errorf("gateway_v4 = %q without an interface, want the interface of the route", v4.Gateway) + } + if v6.Gateway != "" && v6.Interface == "" { + t.Errorf("gateway_v6 = %q without an interface, want the interface of the route", v6.Gateway) + } +} + +func sameSnapshotInterface(a, b snapshotInterface) bool { + return a.Name == b.Name && a.Class == b.Class && a.Up == b.Up && + a.HardwarePort == b.HardwarePort && a.Service == b.Service && + slices.Equal(a.IPs, b.IPs) +} + +func sortedKeys(object map[string]any) []string { + keys := make([]string, 0, len(object)) + for key := range object { + keys = append(keys, key) + } + slices.Sort(keys) + return keys +} diff --git a/cmd/cli/network_transition.go b/cmd/cli/network_transition.go index 5638e2ce..711e592c 100644 --- a/cmd/cli/network_transition.go +++ b/cmd/cli/network_transition.go @@ -153,6 +153,13 @@ type recoveryDiagnostic struct { started time.Time firstFailure sync.Once failed atomic.Bool + + // The flow fills these while it runs. The end event reports them, so one + // journal line tells what the recovery found and what it changed. + recoveredUpstream string + dhcpServers []string + interceptTargetAction string + bypassActive bool } func recoveryReasonName(reason RecoveryReason) string { @@ -191,10 +198,33 @@ func (d *recoveryDiagnostic) failure(err error) { }) } +// The outcomes that one recovery pass ends with. +const ( + recoveryOutcomeCompleted = "completed" + recoveryOutcomeCanceled = "canceled" + recoveryOutcomeSuperseded = "superseded" +) + +// journalRecoveredUpstream bounds the upstream name of the end event. An empty +// name means that no upstream recovered, so it stays empty. +func journalRecoveredUpstream(upstream string) string { + if upstream == "" { + return "" + } + return journalUpstreamName(upstream) +} + func (d *recoveryDiagnostic) end(outcome string) { - e := mainLog.Load().Debug() - if outcome == "canceled" || d.failed.Load() { + e := mainLog.Load().Info() + if outcome == recoveryOutcomeCanceled || d.failed.Load() { e = mainLog.Load().Warn() } - d.event(e).Str("outcome", outcome).Bool("had_failure", d.failed.Load()).Dur("duration_ms", time.Since(d.started)).Msg("Recovery end") + journal(d.event(e)).Str("outcome", outcome).Bool("had_failure", d.failed.Load()). + Dur("duration_ms", time.Since(d.started)). + Str("recovered_upstream", journalRecoveredUpstream(d.recoveredUpstream)). + Strs("dhcp_servers", d.dhcpServers). + Int("dhcp_server_count", len(d.dhcpServers)). + Str("intercept_target_action", d.interceptTargetAction). + Bool("bypass_active", d.bypassActive). + Msg("Recovery end") } diff --git a/cmd/cli/network_transition_test.go b/cmd/cli/network_transition_test.go index eb68d7a8..716c619b 100644 --- a/cmd/cli/network_transition_test.go +++ b/cmd/cli/network_transition_test.go @@ -44,6 +44,10 @@ func captureTransitionLogs(t *testing.T) *syncBuffer { func sourceTestGlobals(t *testing.T) { t.Helper() + // A transition reads the hardware ports and the virtual adapters, and no + // test may start the commands behind them. + stubHeaderSnapshotSources(t) + stubSnapshotVirtualSet(t) v4, v6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() valid, route := networkChangeValidInterfacesFn, networkChangeDefaultRouteIPFn reconcile, ignored := networkChangeReconcileFn, networkChangeIgnoredInterceptFn @@ -189,6 +193,7 @@ func TestNetworkChangeSourceMovedToUpInterface(t *testing.T) { func TestRecoveryTransitionDiagnostics(t *testing.T) { logs := captureTransitionLogs(t) + stubHeaderSnapshotSources(t) original, intercept := recoveryResetDNSFn, dnsIntercept dnsIntercept = false t.Cleanup(func() { recoveryResetDNSFn = original; dnsIntercept = intercept }) @@ -289,7 +294,7 @@ func TestTransitionWarningsSurviveDebugRotation(t *testing.T) { t.Fatalf("debug buffer did not rotate %q", msg) } } - if strings.Count(text, "Recovery end") != 1 || !strings.Contains(text, `"outcome":"completed"`) { + if strings.Count(text, "Recovery end") != 2 || !strings.Contains(text, `"outcome":"completed"`) { t.Fatal("recovery completion must survive rotation beside its retained failure") } if strings.Contains(text, "private.customer") || strings.Contains(text, "secret payload") { diff --git a/cmd/cli/pf_state.go b/cmd/cli/pf_state.go new file mode 100644 index 00000000..2b10cc6b --- /dev/null +++ b/cmd/cli/pf_state.go @@ -0,0 +1,82 @@ +package cli + +import ( + "bufio" + "io" + "maps" + "slices" + "strings" +) + +// parsePFAnchorNames returns the anchor names that the running ruleset refers to, +// unique and sorted. It takes both pfctl show commands because each one holds half +// of the list: "pfctl -sr" the scrub and filter anchors, "pfctl -sn" the +// translation anchors. +func parsePFAnchorNames(rules, nat io.Reader) []string { + unique := make(map[string]struct{}) + // A read that stopped early gives a shorter list, and a shorter list reads + // as an anchor that went away. + if !collectPFAnchorNames(rules, unique) || !collectPFAnchorNames(nat, unique) { + return nil + } + return slices.Sorted(maps.Keys(unique)) +} + +// collectPFAnchorNames reports whether it read the whole output. +func collectPFAnchorNames(r io.Reader, into map[string]struct{}) bool { + if r == nil { + return true + } + scanner := bufio.NewScanner(r) + for scanner.Scan() { + name := pfAnchorNameOf(scanner.Text()) + if name == "" { + continue + } + into[name] = struct{}{} + } + return scanner.Err() == nil +} + +// pfAnchorNameOf returns the anchor name of one pfctl rule line. The rule verb is +// anchor, nat-anchor, rdr-anchor, scrub-anchor or dummynet-anchor, and the quoted +// name follows it. Every other line, including the merged ALTQ warnings, gives "". +func pfAnchorNameOf(line string) string { + verb, rest, ok := strings.Cut(strings.TrimSpace(line), " ") + if !ok || !strings.HasSuffix(verb, "anchor") { + return "" + } + _, quoted, ok := strings.Cut(rest, `"`) + if !ok { + return "" + } + name, _, ok := strings.Cut(quoted, `"`) + if !ok { + return "" + } + return name +} + +// parsePFStatus reads "pfctl -si" output and reports whether pf runs, with the +// uptime text that pfctl prints beside it. +func parsePFStatus(r io.Reader) (enabled bool, since string) { + if r == nil { + return false, "" + } + scanner := bufio.NewScanner(r) + for scanner.Scan() { + status, ok := strings.CutPrefix(strings.TrimSpace(scanner.Text()), "Status:") + if !ok { + continue + } + // pfctl pads the status into a column and prints the debug level next to it. + status, _, _ = strings.Cut(status, "Debug:") + state, uptime, _ := strings.Cut(strings.TrimSpace(status), " for ") + if state != "Enabled" { + return false, "" + } + return true, strings.TrimSpace(uptime) + } + // A read that stopped early may have stopped before the status line. + return false, "" +} diff --git a/cmd/cli/pf_state_darwin.go b/cmd/cli/pf_state_darwin.go new file mode 100644 index 00000000..b89dcfe6 --- /dev/null +++ b/cmd/cli/pf_state_darwin.go @@ -0,0 +1,119 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "os/exec" + "time" +) + +// pfStateReadTimeout bounds each state read. These reads run on the network change +// path, so a pfctl that waits on a busy kernel lock must not hold it. +const pfStateReadTimeout = 3 * time.Second + +// pfStateReadMaxBytes bounds the output of one state read. A ruleset or an +// interface dump of this size is not the state ctrld reads, and these reads run +// on the network change path. +const pfStateReadMaxBytes = 1 << 20 + +// errPFStateOutputTooLarge marks output that passed the bound. The bytes kept +// are a prefix of the real output, so the caller must not parse them. +var errPFStateOutputTooLarge = errors.New("pf state output passed its bound") + +// boundedBuffer keeps at most limit bytes and remembers that it dropped the +// rest. +type boundedBuffer struct { + bytes.Buffer + limit int + exceeded bool +} + +func (b *boundedBuffer) Write(p []byte) (int, error) { + room := b.limit - b.Len() + if len(p) > room { + b.exceeded = true + p = p[:max(room, 0)] + } + if _, err := b.Buffer.Write(p); err != nil { + return 0, err + } + return len(p), nil +} + +// pfStateRunCommand runs one read-only state command. A var so tests replace it +// instead of shelling out. +var pfStateRunCommand = func(name string, args ...string) ([]byte, error) { + ctx, cancel := context.WithTimeout(context.Background(), pfStateReadTimeout) + defer cancel() + out := &boundedBuffer{limit: pfStateReadMaxBytes} + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stdout = out + cmd.Stderr = out + err := cmd.Run() + if out.exceeded { + return nil, errPFStateOutputTooLarge + } + return out.Bytes(), err +} + +// pfRuleDump carries pfctl rule output that the caller already read. A nil +// field means that the caller holds no bytes for that flag, so the read runs +// here. +type pfRuleDump struct { + rules []byte // "pfctl -sr" + nat []byte // "pfctl -sn" +} + +// pfAnchorNames returns the anchor names of the running ruleset. A failed read +// gives no names, because a partial list reads as an anchor that went away. +func (p *prog) pfAnchorNames(dump pfRuleDump) []string { + rules, ok := p.pfRuleBytes(dump.rules, "-sr", "list filter anchors") + if !ok { + return nil + } + nat, ok := p.pfRuleBytes(dump.nat, "-sn", "list translation anchors") + if !ok { + return nil + } + return parsePFAnchorNames(bytes.NewReader(rules), bytes.NewReader(nat)) +} + +// pfRuleBytes returns the rule output of one pfctl flag. It starts pfctl only +// when the caller holds no bytes, and a failed read feeds the backoff that +// keeps ctrld off an exhausted host. +func (p *prog) pfRuleBytes(held []byte, flag, operation string) ([]byte, bool) { + if held != nil { + return held, true + } + out, err := pfStateRunCommand("pfctl", flag) + if err != nil { + p.pfBackoffResourceExhaustion(err, out, operation) + return nil, false + } + return out, true +} + +// pfStatus reports whether pf runs and for how long. A failed read gives false +// and "". +func (p *prog) pfStatus() (bool, string) { + out, err := pfStateRunCommand("pfctl", "-si") + if err != nil { + p.pfBackoffResourceExhaustion(err, out, "read pf status") + return false, "" + } + return parsePFStatus(bytes.NewReader(out)) +} + +// tunnelOwner returns the network extension that owns a tunnel interface, or "" +// when the read fails or no extension owns it. +func tunnelOwner(iface string) string { + if iface == "" { + return "" + } + out, err := pfStateRunCommand("ifconfig", "-v", iface) + if err != nil { + return "" + } + return parseIfconfigAgent(bytes.NewReader(out)) +} diff --git a/cmd/cli/pf_state_test.go b/cmd/cli/pf_state_test.go new file mode 100644 index 00000000..b7c47237 --- /dev/null +++ b/cmd/cli/pf_state_test.go @@ -0,0 +1,137 @@ +package cli + +import ( + "slices" + "strings" + "testing" +) + +// pfStatusEnabled is "pfctl -si" output from a Mac with pf enabled. +const pfStatusEnabled = `Status: Enabled for 0 days 02:11:05 Debug: Urgent +` + +// Captured from macOS 26.6 with ctrld installed. "pfctl -sr" prints the scrub and +// filter anchors, "pfctl -sn" prints the translation anchors, and both carry the +// ALTQ warnings because every reader here uses CombinedOutput. +const pfShowRules = altqNoise + `scrub-anchor "com.apple/*" all fragment reassemble +anchor "com.apple/*" all +anchor "com.controld.ctrld" all +pass in quick on lo0 reply-to lo0 inet proto udp from any to 127.0.0.1 port = 5354 +` + +const pfShowNAT = altqNoise + `nat-anchor "com.apple/*" all +rdr-anchor "com.apple/*" all +rdr-anchor "com.controld.ctrld" all +` + +func Test_parsePFAnchorNames(t *testing.T) { + tests := []struct { + name string + rules string + nat string + want []string + }{ + { + name: "a real ruleset with ctrld installed", + rules: pfShowRules, + nat: pfShowNAT, + want: []string{"com.apple/*", "com.controld.ctrld"}, + }, + { + // The names arrive out of order, so the result proves the sort. + name: "a VPN adds its own anchors", + rules: altqNoise + `anchor "windscribe" all +anchor "com.apple/*" all +`, + nat: altqNoise + `dummynet-anchor "com.apple/*" all +rdr-anchor "windscribe" all +`, + want: []string{"com.apple/*", "windscribe"}, + }, + { + name: "anchors in the translation output only", + rules: altqNoise, + nat: altqNoise + "rdr-anchor \"com.controld.ctrld\" all\n", + want: []string{"com.controld.ctrld"}, + }, + { + name: "a ruleset with no anchor", + rules: altqNoise + "block drop in all\n", + nat: altqNoise + "(no rules)\n", + want: nil, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := parsePFAnchorNames(strings.NewReader(tc.rules), strings.NewReader(tc.nat)) + if !slices.Equal(got, tc.want) { + t.Fatalf("parsePFAnchorNames() = %v, want %v", got, tc.want) + } + }) + } +} + +func Test_parsePFStatus(t *testing.T) { + tests := []struct { + name string + output string + wantEnabled bool + wantSince string + }{ + { + name: "enabled with an uptime", + output: altqNoise + `Status: Enabled for 3 days 02:11:05 Debug: Urgent + +Interface Stats for en0 IPv4 IPv6 + Bytes In 123456789 0 +`, + wantEnabled: true, + wantSince: "3 days 02:11:05", + }, + { + name: "disabled", + output: altqNoise + "Status: Disabled Debug: Urgent\n\n", + }, + { + name: "enabled without an uptime", + output: "Status: Enabled Debug: Urgent\n", + wantEnabled: true, + }, + { + name: "no status line", + output: altqNoise, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + enabled, since := parsePFStatus(strings.NewReader(tc.output)) + if enabled != tc.wantEnabled || since != tc.wantSince { + t.Fatalf("parsePFStatus() = %v, %q, want %v, %q", enabled, since, tc.wantEnabled, tc.wantSince) + } + }) + } +} + +// oversizedLine is longer than the largest line a scanner holds, so the read +// of it stops with an error. +var oversizedLine = strings.Repeat("a", 70<<10) + "\n" + +// Test_parsePFAnchorNamesRejectsAnOversizedLine covers a truncated read. The +// names collected before the error read as a ruleset that lost its anchors, +// and that is the state the watchdog acts on. +func Test_parsePFAnchorNamesRejectsAnOversizedLine(t *testing.T) { + if got := parsePFAnchorNames(strings.NewReader(pfShowRules+oversizedLine), strings.NewReader(pfShowNAT)); got != nil { + t.Fatalf("anchor names of a truncated rule list = %v, want none", got) + } + if got := parsePFAnchorNames(strings.NewReader(pfShowRules), strings.NewReader(pfShowNAT+oversizedLine)); got != nil { + t.Fatalf("anchor names of a truncated translation list = %v, want none", got) + } +} + +// Test_parsePFStatusRejectsAnOversizedLine covers a truncated status read. +func Test_parsePFStatusRejectsAnOversizedLine(t *testing.T) { + enabled, since := parsePFStatus(strings.NewReader(oversizedLine + pfStatusEnabled)) + if enabled || since != "" { + t.Fatalf("pf status of a truncated read = %v, %q, want false and an empty uptime", enabled, since) + } +} diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 487ca314..8cf4a2d8 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -147,6 +147,17 @@ type prog struct { hasLocalDNS bool runningOnDomainController bool + // The network journal state. The value fields here hold a mutex, so no + // code may copy a prog. Every user of a prog holds a pointer to it. + networkNoise noiseCoalescer + wake wakeReporter + repeats repeatLogger //lint:ignore U1000 used on darwin + dnsConfig *dnsConfigPoller + health *queryHealth + snapshotWrites snapshotLimiter + lastNAT64Mu sync.Mutex + lastNAT64Logged string + selfUninstallMu sync.Mutex refusedQueryCount int canSelfUninstall atomic.Bool @@ -170,6 +181,10 @@ type prog struct { networkSourceState *netmon.State networkSourceEpoch *netmon.State networkSourceReadFailed bool + // netmon hands a minor callback the cached major snapshot in Old, so the + // diff keeps the state that the callback before it reported. + networkDeltaState *netmon.State + networkDeltaEpoch *netmon.State // recoveryDebounceTimer coalesces rapid NetworkChange recovery triggers // into a single handleRecovery call. Only handleRecovery is debounced — @@ -192,15 +207,18 @@ type prog struct { // interceptDNSTargetSetValue records the exact value set. Both empty when // no target is set. Guarded by interceptDNSTargetMu. // - //lint:ignore U1000 used in Darwin code. interceptDNSTargetMu sync.Mutex //lint:ignore U1000 used in Darwin code. - interceptDNSTargetService string - //lint:ignore U1000 used in Darwin code. + interceptDNSTargetService string interceptDNSTargetSetValue string //lint:ignore U1000 used in Darwin code. interceptDNSTargetLoaded bool + // interceptTargetMirror holds the same value as interceptDNSTargetSetValue + // for readers that must not wait: the mutex above covers networksetup + // calls that take seconds. + interceptTargetMirror atomic.Value // holds string + // DNS intercept mode state (platform-specific). // On Windows: *wfpState, on macOS: *pfState, nil on other platforms. dnsInterceptState any @@ -224,7 +242,11 @@ type prog struct { // lastTunnelIfaces tracks the tunnel set included in the last successfully loaded // pf anchor. Pending tunnel state is kept separately so failed PF work is retried // instead of being mistaken for an applied update. Protected by mu. - lastTunnelIfaces []string //lint:ignore U1000 used on darwin + lastTunnelIfaces []string //lint:ignore U1000 used on darwin + // lastLoggedTunnelIfaces is the tunnel set of the last logged event. The + // reconcile retries a set until it succeeds, so a diff of the applied set + // repeats one event at every retry. + lastLoggedTunnelIfaces []string //lint:ignore U1000 used on darwin pendingTunnelIfaces []string //lint:ignore U1000 used on darwin hasPendingTunnelIfaces bool //lint:ignore U1000 used on darwin @@ -496,12 +518,11 @@ func (p *prog) postRun() { if !p.skipInitialDNSReset() { p.resetDNS(false, false) } - ns, systemNameservers := initializeOsResolverWithSystemNameserversFn(false) + ns, systemNameservers := initializeOsResolverWithSystemNameserversFn(false, osResolverReasonStart) mainLog.Load().Debug().Msgf("initialized OS resolver with nameservers: %v", ns) p.setDNS(systemNameservers) p.csSetDnsDone <- struct{}{} close(p.csSetDnsDone) - p.logInterfacesState() } } @@ -770,13 +791,7 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { } } - // The new monitor starts with every upstream up, so the old one closes its - // open outages first. Without this the journal holds a down event that no - // up event ever follows. - if p.um != nil { - p.um.retire() - } - p.um = newUpstreamMonitor(p.cfg) + p.replaceUpstreamMonitor() if !reload { p.sema = &chanSemaphore{ready: make(chan struct{}, defaultSemaphoreCap)} @@ -806,12 +821,8 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { } if !reload { - go func() { - // Start network monitoring - if err := p.monitorNetworkChanges(); err != nil { - mainLog.Load().Error().Err(err).Msg("Failed to start network monitoring") - } - }() + p.newNetworkJournal() + go p.startNetworkMonitorAndJournal() } for listenerNum := range p.cfg.Listener { @@ -884,6 +895,96 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { wg.Wait() } +// dnsConfigChangedMessage names the journal event of the resolver table of the +// host. A log tool selects every resolver change of a run by this message. +const dnsConfigChangedMessage = "DNS configuration changed" + +// runSCUtilDNSFn reads the resolver table of the host. A test replaces it, +// because the configuration daemon of the host is not a fixture. +var runSCUtilDNSFn = runSCUtilDNS + +// replaceUpstreamMonitor puts a new upstream monitor in place of the one of +// the run before. The new monitor starts with every upstream up, so the old one +// closes its open outages first. Without this the journal holds a down event +// that no up event ever follows. +func (p *prog) replaceUpstreamMonitor() { + p.mu.Lock() + defer p.mu.Unlock() + if p.um != nil { + p.um.retire() + } + p.um = newUpstreamMonitor(p.cfg) +} + +// upstreamMonitorNow reads the monitor of this run under the lock that a +// replacement takes, because a reload swaps it while the journal loops run. +func (p *prog) upstreamMonitorNow() *upstreamMonitor { + p.mu.Lock() + defer p.mu.Unlock() + return p.um +} + +// newNetworkJournal creates the trackers that follow the host network. The +// query path reads them without a lock, so they exist before a listener starts. +func (p *prog) newNetworkJournal() { + p.health = newQueryHealth() + p.querySampler.onFailure = p.health.countFailure + if runtime.GOOS == "darwin" { + p.dnsConfig = newDNSConfigPoller(runSCUtilDNSFn) + } +} + +// startNetworkMonitorAndJournal starts the network monitor and then opens the +// journal. The trackers do not depend on the monitor, so a monitor that cannot +// start leaves them running. It passes on the wait function of the journal. +func (p *prog) startNetworkMonitorAndJournal() func() { + if err := p.monitorNetworkChanges(); err != nil { + mainLog.Load().Error().Err(err).Msg("Failed to start network monitoring") + } + return p.startNetworkJournal() +} + +// startNetworkJournal runs the trackers and opens the journal of this run with +// one snapshot. Support reads the grade of the query path and the resolver +// table of each moment after an incident. It returns a function that waits for +// both loops, so a caller that closed the stop channel knows when they ended. +func (p *prog) startNetworkJournal() func() { + healthDone := p.health.startLoop(p.stopCh, func() (int, bool) { + return p.upstreamMonitorNow().countDownExcept(isInternalDomainUpstream), p.recoveryBypass.Load() + }) + var configDone <-chan struct{} + if p.dnsConfig != nil { + configDone = p.dnsConfig.startLoop(p.stopCh, p.logDNSConfigChanges) + } + p.logNetworkSnapshot("start") + return func() { + <-healthDone + if configDone != nil { + <-configDone + } + } +} + +// logDNSConfigChanges puts each changed resolver of the host in the journal. A +// resolver that vanished comes without a nameserver, so the reader sees the +// drop. The search suffixes name the organization of the endpoint, so the +// journal holds their number only. +func (p *prog) logDNSConfigChanges(entries []dnsResolverEntry) { + for _, entry := range entries { + journal(mainLog.Load().Info()). + Strs("nameservers", entry.Nameservers). + Int("if_index", entry.IfIndex). + Str("interface", entry.Interface). + Bool("scoped", entry.Scoped). + Str("action", entry.Action). + Str("flags", entry.Flags). + Int("search_domain_count", len(entry.SearchDomains)). + Int("order", entry.Order). + Str("reachable", entry.Reachable). + Msg(dnsConfigChangedMessage) + } +} + // setupClientInfoDiscover performs necessary works for running client info discover. func (p *prog) setupClientInfoDiscover(selfIP string) { p.ciTable = clientinfo.NewTable(&cfg, selfIP, cdUID, p.ptrNameservers) @@ -998,7 +1099,7 @@ var ( startDNSInterceptFn = (*prog).startDNSIntercept ensureInterceptDNSTargetFn = (*prog).ensureInterceptDNSTarget removeInterceptDNSTargetFn = (*prog).removeInterceptDNSTarget - initializeOsResolverWithSystemNameserversFn = ctrld.InitializeOsResolverWithSystemNameservers + initializeOsResolverWithSystemNameserversFn = ctrld.InitializeOsResolverWithSystemNameserversReason setDnsForRunningIfaceFn = (*prog).setDnsForRunningIface resetDNSFn = (*prog).resetDNS // refuseFallbackFatal reports a startup failure the interface-DNS fallback @@ -1463,28 +1564,6 @@ func (p *prog) resetDNSForRunningIface(isStart bool, restoreStatic bool) (runnin return } -func (p *prog) logInterfacesState() { - withEachPhysicalInterfaces("", "", func(i *net.Interface) error { - addrs, err := i.Addrs() - if err != nil { - mainLog.Load().Warn().Str("interface", i.Name).Err(err).Msg("failed to get addresses") - } - nss, err := currentStaticDNS(i) - if err != nil { - mainLog.Load().Warn().Str("interface", i.Name).Err(err).Msg("failed to get DNS") - } - if len(nss) == 0 { - nss = currentDNS(i) - } - mainLog.Load().Debug(). - Any("addrs", addrs). - Strs("nameservers", nss). - Int("index", i.Index). - Msgf("interface state: %s", i.Name) - return nil - }) -} - // findWorkingInterface looks for a network interface with a valid IP configuration func findWorkingInterface(currentIface string) string { // Helper to check if IP is valid (not link-local) diff --git a/cmd/cli/query_error_sampler.go b/cmd/cli/query_error_sampler.go index 78d017b4..af382f7c 100644 --- a/cmd/cli/query_error_sampler.go +++ b/cmd/cli/query_error_sampler.go @@ -12,6 +12,10 @@ const ( sampleClassResolveFailed = "resolve_failed" sampleClassAllEndpointsFailed = "all_endpoints_failed" sampleClassSendResponseFailed = "send_response_failed" + // sampleClassInternalDomain names a failure of an Internal Domain resolver. + // It stays apart from the other classes, because an endpoint away from the + // organization network cannot reach such a resolver. + sampleClassInternalDomain = "internal_domain" ) const ( @@ -56,6 +60,9 @@ type errorSampler struct { windows map[sampleKey]*sampleWindow now func() time.Time after func(time.Duration, func()) *time.Timer + // onFailure counts the event for the query health tracker. The daemon sets + // it once at start, before a listener serves a query. + onFailure func(class string) } // event returns the event a call site logs with. The first querySampleLimit @@ -83,6 +90,11 @@ func (s *errorSampler) event(class, upstream string) *zerolog.Event { } s.mu.Unlock() + // A lowered line keeps its place in the breakdown of the window. It is one + // event, not one failed query, so it does not change the grade. + if s.onFailure != nil { + s.onFailure(class) + } s.logSummaries(summaries) if suppressed { diff --git a/cmd/cli/query_health.go b/cmd/cli/query_health.go new file mode 100644 index 00000000..b28fa608 --- /dev/null +++ b/cmd/cli/query_health.go @@ -0,0 +1,284 @@ +package cli + +import ( + "maps" + "slices" + "sync" + "time" + + "github.com/rs/zerolog" +) + +// Health classes of the query path. The names go into the journal. +const ( + queryHealthHealthy = "healthy" + queryHealthDegraded = "degraded" + queryHealthFailing = "failing" +) + +const ( + // queryHealthWindow is the length of one counting window and the time + // between two heartbeats. + queryHealthWindow = 15 * time.Minute + // queryHealthFailingRatio and queryHealthDegradedRatio grade the failed + // share of a window. + queryHealthFailingRatio = 0.5 + queryHealthDegradedRatio = 0.05 + // queryHealthFailingQueries is the smallest sample that can leave the + // healthy class and the smallest sample that can report a class change. + queryHealthFailingQueries = 10 + // queryHealthClassTicks is the number of ticks in a row that a new class + // must hold before it reports. + queryHealthClassTicks = 2 + // queryHealthMessage names the journal event of the tracker. + queryHealthMessage = "Query health" +) + +// queryHealthTick is the time between two evaluations. A test shortens it. +var queryHealthTick = time.Minute + +// queryHealth grades the query path over a window. Support reads the grade +// after an incident, so the journal needs one class per window instead of a +// line per query. +type queryHealth struct { + mu sync.Mutex + window time.Duration + now func() time.Time + windowStart time.Time + queries uint64 + cacheHits uint64 + // failedQueries counts the client queries that ended without an answer. + // The grade follows this count, and failures holds the sampler classes + // behind it, which raise several events for one failed query. + failedQueries uint64 + failures map[string]uint64 + lastClass string + // pendingClass is the class that the last tick graded, and pendingTicks + // counts how many ticks in a row graded it. A class reports only after it + // held, so one noisy minute opens no event. + pendingClass string + pendingTicks int +} + +// queryHealthEvent holds the class and the counters of one window. +type queryHealthEvent struct { + Class string + WindowSeconds int + Queries uint64 + CacheHits uint64 + FailedQueries uint64 + Failures map[string]uint64 + UpstreamsDown int + BypassActive bool + CacheHitRatio float64 +} + +func newQueryHealth() *queryHealth { + return &queryHealth{ + window: queryHealthWindow, + now: time.Now, + failures: make(map[string]uint64), + } +} + +// A prog outside a run holds no tracker, so every counter of the query path +// and every loop accepts a nil receiver. +func (q *queryHealth) countQuery() { + if q == nil { + return + } + q.mu.Lock() + defer q.mu.Unlock() + q.queries++ +} + +func (q *queryHealth) countCacheHit() { + if q == nil { + return + } + q.mu.Lock() + defer q.mu.Unlock() + q.cacheHits++ +} + +// countFailedQuery counts one client query that ended without an answer. A +// query that failed on several upstreams counts once, because the grade +// measures the service the client got. +func (q *queryHealth) countFailedQuery() { + if q == nil { + return + } + q.mu.Lock() + defer q.mu.Unlock() + q.failedQueries++ +} + +func (q *queryHealth) countFailure(class string) { + if q == nil { + return + } + q.mu.Lock() + defer q.mu.Unlock() + if q.failures == nil { + q.failures = make(map[string]uint64) + } + q.failures[class]++ +} + +// evaluate grades the counters and reports whether the event goes out. The +// report covers the window that just closed, or the running one. The window +// starts at the first evaluate, so the caller owns the clock. +func (q *queryHealth) evaluate(now time.Time, upstreamsDown int, bypassActive bool) (queryHealthEvent, bool) { + if q == nil { + return queryHealthEvent{}, false + } + q.mu.Lock() + defer q.mu.Unlock() + + if q.windowStart.IsZero() { + q.windowStart = now + } + event := queryHealthEvent{ + Class: healthClass(q.queries, q.failedQueries), + WindowSeconds: int(q.window / time.Second), + Queries: q.queries, + CacheHits: q.cacheHits, + FailedQueries: q.failedQueries, + Failures: copyFailures(q.failures), + UpstreamsDown: upstreamsDown, + BypassActive: bypassActive, + CacheHitRatio: shareOf(q.cacheHits, q.queries), + } + // The end of a window always reports, so a quiet daemon stays visible and + // no window is lost after an early report. A class change waits for a + // sample and for a second tick. + windowEnded := !now.Before(q.windowStart.Add(q.window)) + emit := q.lastClass == "" || windowEnded || q.classChangeHolds(event.Class) + if emit { + q.lastClass = event.Class + q.pendingClass, q.pendingTicks = "", 0 + } + if windowEnded { + q.startWindow(now) + } + return event, emit +} + +// classChangeHolds reports whether a new class may open an event. It needs a +// window with enough queries, and the same class on two ticks in a row. The +// caller holds the lock. +func (q *queryHealth) classChangeHolds(class string) bool { + if class == q.lastClass || q.queries < queryHealthFailingQueries { + q.pendingClass, q.pendingTicks = "", 0 + return false + } + if class != q.pendingClass { + q.pendingClass, q.pendingTicks = class, 1 + return false + } + q.pendingTicks++ + return q.pendingTicks >= queryHealthClassTicks +} + +// startWindow drops the counters of the window that closed. Each report covers +// one window, so a quiet hour cannot hide inside the totals of a busy one. The +// caller holds the lock. +func (q *queryHealth) startWindow(now time.Time) { + q.windowStart = now + q.queries = 0 + q.cacheHits = 0 + q.failedQueries = 0 + q.failures = make(map[string]uint64) +} + +func (q *queryHealth) log(event queryHealthEvent) { + if q == nil { + return + } + journal(mainLog.Load().Info()). + Str("class", event.Class). + Int("window_s", event.WindowSeconds). + Uint64("queries", event.Queries). + Uint64("cache_hits", event.CacheHits). + Uint64("failed_queries", event.FailedQueries). + Dict("failures_by_class", failuresDict(event.Failures)). + Int("upstreams_down", event.UpstreamsDown). + Bool("bypass_active", event.BypassActive). + Float64("cache_hit_ratio", event.CacheHitRatio). + Msg(queryHealthMessage) +} + +// startLoop runs the grading loop and returns a channel that closes when the +// loop returns. A caller joins that channel before it reads the log. +func (q *queryHealth) startLoop(stop <-chan struct{}, inputs func() (upstreamsDown int, bypassActive bool)) <-chan struct{} { + done := make(chan struct{}) + go func() { + defer close(done) + q.loop(stop, inputs) + }() + return done +} + +// loop grades the counters on every tick until the caller stops it. +func (q *queryHealth) loop(stop <-chan struct{}, inputs func() (upstreamsDown int, bypassActive bool)) { + if q == nil { + return + } + ticker := time.NewTicker(queryHealthTick) + defer ticker.Stop() + for { + select { + case <-stop: + return + case <-ticker.C: + upstreamsDown, bypassActive := inputs() + event, emit := q.evaluate(q.now(), upstreamsDown, bypassActive) + if !emit { + continue + } + q.log(event) + } + } +} + +// healthClass grades the failed share of a window. A small sample stays +// healthy, because a few failures at a start are not an outage. +func healthClass(queries, failures uint64) string { + if queries < queryHealthFailingQueries { + return queryHealthHealthy + } + failed := shareOf(failures, queries) + if failed >= queryHealthFailingRatio { + return queryHealthFailing + } + if failed >= queryHealthDegradedRatio { + return queryHealthDegraded + } + return queryHealthHealthy +} + +// shareOf reports the part of the total, and 0 for an empty total. +func shareOf(part, total uint64) float64 { + if total == 0 { + return 0 + } + return float64(part) / float64(total) +} + +// copyFailures gives the report its own map, because the counters keep running +// while the caller holds the report. +func copyFailures(failures map[string]uint64) map[string]uint64 { + copied := make(map[string]uint64, len(failures)) + maps.Copy(copied, failures) + return copied +} + +// failuresDict renders the classes in a stable order, so two lines of the same +// window compare by eye. +func failuresDict(failures map[string]uint64) *zerolog.Event { + dict := zerolog.Dict() + for _, class := range slices.Sorted(maps.Keys(failures)) { + dict = dict.Uint64(class, failures[class]) + } + return dict +} diff --git a/cmd/cli/query_health_prog_test.go b/cmd/cli/query_health_prog_test.go new file mode 100644 index 00000000..cd0520db --- /dev/null +++ b/cmd/cli/query_health_prog_test.go @@ -0,0 +1,459 @@ +package cli + +import ( + "context" + "errors" + "runtime" + "sync/atomic" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/stretchr/testify/require" + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/dnscache" + "github.com/Control-D-Inc/ctrld/testhelper" +) + +// wantDNSConfigStrings compares one array field of a logged resolver entry. +func wantDNSConfigStrings(t *testing.T, event map[string]any, field string, want ...string) { + t.Helper() + values, ok := event[field].([]any) + if !ok { + t.Fatalf("field %q: got %T, want an array", field, event[field]) + } + if len(values) != len(want) { + t.Fatalf("field %q: got %v, want %v", field, values, want) + } + for i, value := range values { + if value != want[i] { + t.Fatalf("field %q at %d: got %v, want %v", field, i, value, want[i]) + } + } +} + +// Test_queryHealthCountsFailedProxyQueries drives the real query path against a +// refusing upstream: every sampled error reaches the tracker, so the window of +// the daemon grades as failing. +func Test_queryHealthCountsFailedProxyQueries(t *testing.T) { + captureDebugMainLog(t) + cfg := &ctrld.Config{} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg), health: newQueryHealth()} + // A new monitor holds back the failure counts for one second, and its + // timer would mark the upstream down after the test. + p.um.clearRecovered(upstreamOS) + p.um.after = func(time.Duration, func()) {} + p.querySampler.onFailure = p.health.countFailure + // The zero value sampler arms a real timer. Close its window after the + // assertions, so no late summary line reaches another test. + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + origOSUpstream := osUpstreamConfig + t.Cleanup(func() { osUpstreamConfig = origOSUpstream }) + osUpstreamConfig = &ctrld.UpstreamConfig{ + Name: "dead resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 500, + } + + const failedQueries = 12 + const servedQueries = 20 + ctx := context.Background() + for i := 0; i < failedQueries; i++ { + if rcode := askThroughProxy(t, p, ctx).Rcode; rcode != dns.RcodeServerFailure { + t.Fatalf("query %d: rcode %s, want SERVFAIL", i+1, dns.RcodeToString[rcode]) + } + } + // serveDNS counts every query it serves, and the eight answered ones never + // reach the sampler. + for i := 0; i < servedQueries; i++ { + p.health.countQuery() + } + + event, emit := p.health.evaluate(time.Now(), 0, false) + if !emit { + t.Fatal("the first evaluate did not emit") + } + if event.Class != queryHealthFailing { + t.Fatalf("class: got %q, want %q", event.Class, queryHealthFailing) + } + if event.Queries != servedQueries { + t.Fatalf("queries: got %d, want %d", event.Queries, servedQueries) + } + // The grade follows one failure per failed query. Each of those queries + // also raised two sampler events, and the breakdown still shows them. + if event.FailedQueries != failedQueries { + t.Fatalf("failed queries: got %d, want %d", event.FailedQueries, failedQueries) + } + if got := event.Failures[sampleClassResolveFailed]; got != failedQueries { + t.Fatalf("%s failures: got %d, want %d", sampleClassResolveFailed, got, failedQueries) + } + if got := event.Failures[sampleClassAllEndpointsFailed]; got != failedQueries { + t.Fatalf("%s failures: got %d, want %d", sampleClassAllEndpointsFailed, got, failedQueries) + } +} + +// Test_queryHealthKeepsInternalDomainFailuresOutOfTheGrade drives the same +// number of failed queries against an Internal Domain resolver. An endpoint +// away from the organization network cannot reach that resolver, so the class +// stays healthy, the failures keep their own name, and the down entry of that +// resolver stays out of upstreams_down. +func Test_queryHealthKeepsInternalDomainFailuresOutOfTheGrade(t *testing.T) { + captureDebugMainLog(t) + const internalUpstream = upstreamPrefix + internalDomainUpstreamPrefix + "1" + cfg := &ctrld.Config{} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + cfg.Upstream = map[string]*ctrld.UpstreamConfig{ + internalDomainUpstreamPrefix + "1": { + Name: "dead internal resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 500, + }, + } + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg), health: newQueryHealth()} + p.um.after = func(time.Duration, func()) {} + p.um.clearRecovered(internalUpstream) + p.querySampler.onFailure = p.health.countFailure + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + const failedQueries = 12 + const servedQueries = 20 + ctx := context.Background() + msg := new(dns.Msg) + msg.SetQuestion("internal.corp.example.", dns.TypeA) + for i := 0; i < failedQueries; i++ { + res := p.proxy(ctx, &proxyRequest{ + msg: msg, + ufr: &upstreamForResult{upstreams: []string{internalUpstream}, srcAddr: "192.168.0.1:1234"}, + }) + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeServerFailure { + t.Fatalf("query %d: the internal resolver answered", i+1) + } + } + for i := 0; i < servedQueries; i++ { + p.health.countQuery() + } + p.um.mu.Lock() + p.um.markDown(internalUpstream, 3, "immediate") + p.um.mu.Unlock() + + event, emit := p.health.evaluate(time.Now(), p.um.countDownExcept(isInternalDomainUpstream), false) + if !emit { + t.Fatal("the first evaluate did not emit") + } + if event.Class != queryHealthHealthy { + t.Fatalf("class: got %q, want %q", event.Class, queryHealthHealthy) + } + if event.FailedQueries != 0 { + t.Fatalf("failed queries: got %d, want 0", event.FailedQueries) + } + if got := event.Failures[sampleClassInternalDomain]; got != failedQueries { + t.Fatalf("%s failures: got %d, want %d", sampleClassInternalDomain, got, failedQueries) + } + if event.UpstreamsDown != 0 { + t.Fatalf("upstreams_down: got %d, want 0", event.UpstreamsDown) + } + if got := p.um.countDown(); got != 1 { + t.Fatalf("countDown: got %d, want 1", got) + } +} + +// Test_queryHealthCountsProxyCacheHits proves that an answer from the cache +// counts a cache hit, so the ratio of the journal follows the query path. +func Test_queryHealthCountsProxyCacheHits(t *testing.T) { + captureDebugMainLog(t) + p := &prog{cfg: testhelper.SampleConfig(t), health: newQueryHealth()} + cache, err := dnscache.NewLRUCache(16) + require.NoError(t, err) + p.cache = cache + + msg := new(dns.Msg) + msg.SetQuestion("cache.test.", dns.TypeA) + answer := new(dns.Msg) + answer.SetRcode(msg, dns.RcodeSuccess) + p.cache.Add(dnscache.NewKey(msg, "upstream.1"), dnscache.NewValue(answer, time.Now().Add(time.Minute))) + + res := p.proxy(context.Background(), &proxyRequest{ + msg: msg, + ufr: &upstreamForResult{upstreams: []string{"upstream.1"}}, + }) + require.NotNil(t, res) + require.True(t, res.cached, "the answer did not come from the cache") + + event, _ := p.health.evaluate(time.Now(), 0, false) + if event.CacheHits != 1 { + t.Fatalf("cache hits: got %d, want 1", event.CacheHits) + } +} + +// Test_logDNSConfigChangesLogsEveryEntry proves that each changed resolver of +// the host reaches the journal with its fields. A resolver that vanished keeps +// an empty nameserver list. +func Test_logDNSConfigChangesLogsEveryEntry(t *testing.T) { + logs := captureDebugMainLog(t) + p := &prog{} + + p.logDNSConfigChanges([]dnsResolverEntry{ + { + Order: 200000, + Nameservers: []string{"192.168.50.200", "192.168.50.1"}, + IfIndex: 16, + Interface: "en0", + Flags: "Request A records, Request AAAA records", + SearchDomains: []string{"corp.example"}, + Reachable: "Reachable,Directly Reachable Address", + }, + {Order: 101600, IfIndex: 21, Interface: "utun4", Flags: "Scoped"}, + }) + + events := jsonLogEvents(t, logs, dnsConfigChangedMessage) + if len(events) != 2 { + t.Fatalf("configuration lines: got %d, want 2", len(events)) + } + + wantField(t, events[0], "journal", true) + wantField(t, events[0], "level", "info") + wantField(t, events[0], "if_index", float64(16)) + wantField(t, events[0], "interface", "en0") + wantField(t, events[0], "flags", "Request A records, Request AAAA records") + wantField(t, events[0], "order", float64(200000)) + wantField(t, events[0], "reachable", "Reachable,Directly Reachable Address") + wantDNSConfigStrings(t, events[0], "nameservers", "192.168.50.200", "192.168.50.1") + // The suffixes of an organization stay out of the journal. Only their + // number goes in. + wantField(t, events[0], "search_domain_count", float64(1)) + if _, ok := events[0]["search_domains"]; ok { + t.Fatal("the journal holds the search domains") + } + + wantField(t, events[1], "journal", true) + wantField(t, events[1], "interface", "utun4") + wantDNSConfigStrings(t, events[1], "nameservers") + wantField(t, events[1], "search_domain_count", float64(0)) +} + +// Test_startNetworkJournalLogsTheStartSnapshot proves that the daemon opens the +// journal with one snapshot and runs the trackers that follow the query path +// and the DNS configuration of the host. +func Test_startNetworkJournalLogsTheStartSnapshot(t *testing.T) { + logs := captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + origTick := queryHealthTick + t.Cleanup(func() { queryHealthTick = origTick }) + queryHealthTick = time.Millisecond + + var polls atomic.Int64 + // The poll loop reads the delay on every pass, so it parks on a long delay + // before the test puts the package value back. + parked := make(chan struct{}, 1) + origRunner, origDelay := runSCUtilDNSFn, dnsConfigDelayFn + t.Cleanup(func() { runSCUtilDNSFn, dnsConfigDelayFn = origRunner, origDelay }) + movedResolver := wifiResolverEntry() + movedResolver.Nameservers = []string{"10.0.0.1"} + runSCUtilDNSFn = func() ([]dnsResolverEntry, error) { + if polls.Add(1) == 1 { + return []dnsResolverEntry{wifiResolverEntry()}, nil + } + return []dnsResolverEntry{movedResolver}, nil + } + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { + if polls.Load() < 2 { + return time.Millisecond + } + select { + case parked <- struct{}{}: + default: + } + return time.Hour + } + + cfg := &ctrld.Config{} + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg), stopCh: make(chan struct{})} + // The monitor arms a real timer for each upstream it resets. Nothing in + // this test waits for one. + p.um.after = func(time.Duration, func()) {} + p.lastNetworkState.Store(sourceTestState("en0", true, "192.0.2.9/24")) + p.recoveryBypass.Store(true) + p.um.mu.Lock() + p.um.markDown(upstreamOS, 3, "timer") + p.um.mu.Unlock() + + p.newNetworkJournal() + if p.health == nil || p.querySampler.onFailure == nil { + t.Fatal("the trackers do not exist before the loops start") + } + // A counter of the query path works as soon as the trackers exist. + p.health.countQuery() + waitForJournalLoops := p.startNetworkJournal() + t.Cleanup(func() { + close(p.stopCh) + waitForJournalLoops() + }) + + events := jsonLogEvents(t, logs, networkSnapshotMessage) + if len(events) != 1 { + t.Fatalf("snapshot lines: got %d, want 1", len(events)) + } + wantField(t, events[0], "journal", true) + wantField(t, events[0], "level", "info") + wantField(t, events[0], "trigger", "start") + + waitForLogLine(t, logs, queryHealthMessage) + health := jsonLogEvents(t, logs, queryHealthMessage) + wantField(t, health[0], "upstreams_down", float64(1)) + wantField(t, health[0], "bypass_active", true) + + if runtime.GOOS != "darwin" { + if p.dnsConfig != nil { + t.Fatal("the DNS configuration poller runs outside macOS") + } + return + } + if p.dnsConfig == nil { + t.Fatal("macOS starts no DNS configuration poller") + } + waitForLogLine(t, logs, dnsConfigChangedMessage) + select { + case <-parked: + case <-time.After(5 * time.Second): + t.Fatal("the poll loop did not park") + } + resolvers := jsonLogEvents(t, logs, dnsConfigChangedMessage) + if len(resolvers) != 1 { + t.Fatalf("configuration lines: got %d, want 1", len(resolvers)) + } + wantField(t, resolvers[0], "interface", "en0") + wantDNSConfigStrings(t, resolvers[0], "nameservers", "10.0.0.1") +} + +// Test_startNetworkJournalReadsTheMonitorAcrossAReload replaces the upstream +// monitor while the health loop runs. A reload builds a new monitor, and the +// loop read the field of the prog with no lock, which the race detector +// reports. +func Test_startNetworkJournalReadsTheMonitorAcrossAReload(t *testing.T) { + captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + origTick := queryHealthTick + t.Cleanup(func() { queryHealthTick = origTick }) + queryHealthTick = time.Millisecond + + cfg := &ctrld.Config{} + p := &prog{cfg: cfg, stopCh: make(chan struct{})} + p.lastNetworkState.Store(sourceTestState("en0", true, "192.0.2.9/24")) + p.replaceUpstreamMonitor() + p.health = newQueryHealth() + + waitForJournalLoops := p.startNetworkJournal() + for i := 0; i < 30; i++ { + p.replaceUpstreamMonitor() + time.Sleep(time.Millisecond) + } + close(p.stopCh) + waitForJournalLoops() +} + +// Test_startNetworkJournalRunsAfterAFailedMonitorStart proves that a network +// monitor that cannot start leaves the journal running. The error line goes +// out, the start snapshot opens the journal, and the health loop reports. +func Test_startNetworkJournalRunsAfterAFailedMonitorStart(t *testing.T) { + logs := captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + origTick := queryHealthTick + t.Cleanup(func() { queryHealthTick = origTick }) + queryHealthTick = time.Millisecond + + origMonitor := newNetworkChangeMonitorFn + t.Cleanup(func() { newNetworkChangeMonitorFn = origMonitor }) + newNetworkChangeMonitorFn = func(func(string, ...any)) (networkChangeMonitor, error) { + return nil, errors.New("no network monitor here") + } + origRunner, origDelay := runSCUtilDNSFn, dnsConfigDelayFn + t.Cleanup(func() { runSCUtilDNSFn, dnsConfigDelayFn = origRunner, origDelay }) + runSCUtilDNSFn = func() ([]dnsResolverEntry, error) { + return []dnsResolverEntry{wifiResolverEntry()}, nil + } + dnsConfigDelayFn = func(*dnsConfigPoller, time.Time) time.Duration { return time.Hour } + + cfg := &ctrld.Config{} + p := &prog{cfg: cfg, stopCh: make(chan struct{})} + p.lastNetworkState.Store(sourceTestState("en0", true, "192.0.2.9/24")) + p.replaceUpstreamMonitor() + p.upstreamMonitorNow().after = func(time.Duration, func()) {} + + p.newNetworkJournal() + p.health.countQuery() + p.health.mu.Lock() + queries := p.health.queries + p.health.mu.Unlock() + if queries != 1 { + t.Fatalf("queries after one count: got %d, want 1", queries) + } + + waitForJournalLoops := p.startNetworkMonitorAndJournal() + t.Cleanup(func() { + close(p.stopCh) + waitForJournalLoops() + }) + + failures := jsonLogEvents(t, logs, "Failed to start network monitoring") + if len(failures) != 1 { + t.Fatalf("monitor failure lines: got %d, want 1", len(failures)) + } + wantField(t, failures[0], "level", "error") + + snapshots := jsonLogEvents(t, logs, networkSnapshotMessage) + if len(snapshots) != 1 { + t.Fatalf("snapshot lines: got %d, want 1", len(snapshots)) + } + wantField(t, snapshots[0], "trigger", "start") + + waitForLogLine(t, logs, queryHealthMessage) +} + +// Test_queryHealthCountsNoFailureWhenAnUpstreamAnswers proves that a failure on +// one upstream does not grade the query. The client got an answer, so the +// failure stays in the breakdown only. The DNS64 companion lookup fails through +// the same call, so it cannot grade a query either. +func Test_queryHealthCountsNoFailureWhenAnUpstreamAnswers(t *testing.T) { + captureDebugMainLog(t) + fixture := startDNSFixture(t) + cfg := &ctrld.Config{} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + cfg.Upstream = map[string]*ctrld.UpstreamConfig{ + "0": {Name: "dead resolver", Type: ctrld.ResolverTypeLegacy, Endpoint: deadUpstreamEndpoint, Timeout: 500}, + "1": {Name: "live resolver", Type: ctrld.ResolverTypeLegacy, Endpoint: fixture.addr, Timeout: 2000}, + } + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg), health: newQueryHealth()} + p.um.after = func(time.Duration, func()) {} + p.um.clearRecovered(upstreamPrefix + "0") + p.querySampler.onFailure = p.health.countFailure + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + msg := new(dns.Msg) + msg.SetQuestion("failover.test.", dns.TypeA) + res := p.proxy(context.Background(), &proxyRequest{ + msg: msg, + ufr: &upstreamForResult{ + upstreams: []string{upstreamPrefix + "0", upstreamPrefix + "1"}, + srcAddr: "192.168.0.1:1234", + }, + }) + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Fatal("the second upstream did not answer") + } + p.health.countQuery() + + event, _ := p.health.evaluate(time.Now(), 0, false) + if event.FailedQueries != 0 { + t.Fatalf("failed queries: got %d, want 0", event.FailedQueries) + } + if got := event.Failures[sampleClassResolveFailed]; got != 1 { + t.Fatalf("%s failures: got %d, want 1", sampleClassResolveFailed, got) + } + if event.Class != queryHealthHealthy { + t.Fatalf("class: got %q, want %q", event.Class, queryHealthHealthy) + } +} diff --git a/cmd/cli/query_health_test.go b/cmd/cli/query_health_test.go new file mode 100644 index 00000000..88f120ce --- /dev/null +++ b/cmd/cli/query_health_test.go @@ -0,0 +1,424 @@ +package cli + +import ( + "fmt" + "reflect" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/rs/zerolog" +) + +// queryHealthBase is the start time of every tracker test. The tracker takes +// the time from the caller, so no test waits for a real clock. +var queryHealthBase = time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) + +// feedQueryHealth counts queries, and one failed query for each failure of a +// class. The grade follows the failed queries, so a test that wants a class +// gives the failures that produced it. +func feedQueryHealth(q *queryHealth, queries int, failures map[string]int) { + for i := 0; i < queries; i++ { + q.countQuery() + } + for class, count := range failures { + for i := 0; i < count; i++ { + q.countFailure(class) + q.countFailedQuery() + } + } +} + +// waitForLogLine polls the buffer, because the loop logs from its own goroutine. +func waitForLogLine(t *testing.T, logs *syncBuffer, message string) { + t.Helper() + waitForLogLineOrExplain(t, logs, message, nil) +} + +// waitForLogLineOrExplain is waitForLogLine with extra text for the failure. +func waitForLogLineOrExplain(t *testing.T, logs *syncBuffer, message string, explain func() string) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) { + if strings.Contains(logs.String(), message) { + return + } + time.Sleep(time.Millisecond) + } + extra := "" + if explain != nil { + extra = " " + explain() + } + t.Fatalf("no %q line before the deadline (global level %s, captured %q)%s", message, zerolog.GlobalLevel(), logs.String(), extra) +} + +// loggerSwapRecord is one change of the main logger or of the global level. +type loggerSwapRecord struct { + at time.Duration + level zerolog.Level + writer string +} + +// watchLoggerSwaps records every change of the main logger and of the global +// level while the test runs, so a missing line names what took the logger. +func watchLoggerSwaps(t *testing.T) func() string { + t.Helper() + start := time.Now() + stop := make(chan struct{}) + done := make(chan struct{}) + var mu sync.Mutex + var records []loggerSwapRecord + go func() { + defer close(done) + lastLogger, lastLevel := mainLog.Load(), zerolog.GlobalLevel() + for { + select { + case <-stop: + return + default: + } + logger, level := mainLog.Load(), zerolog.GlobalLevel() + if logger != lastLogger || level != lastLevel { + mu.Lock() + records = append(records, loggerSwapRecord{time.Since(start), level, loggerWriterType(logger)}) + mu.Unlock() + lastLogger, lastLevel = logger, level + } + time.Sleep(50 * time.Microsecond) + } + }() + t.Cleanup(func() { close(stop); <-done }) + return func() string { + mu.Lock() + defer mu.Unlock() + parts := make([]string, 0, len(records)) + for _, r := range slices.Clone(records) { + parts = append(parts, fmt.Sprintf("{at=%s level=%s writer=%s}", r.at, r.level, r.writer)) + } + return fmt.Sprintf("swaps=%d %s current=%s", len(records), strings.Join(parts, " "), loggerWriterType(mainLog.Load())) + } +} + +// loggerWriterType names the writer behind a logger through its unexported +// field, so a swapped logger tells which init function built it. +func loggerWriterType(l *zerolog.Logger) string { + if l == nil { + return "nil" + } + w := reflect.ValueOf(l).Elem().FieldByName("w") + if !w.IsValid() || w.IsNil() { + return "none" + } + inner := w.Elem() + name := inner.Type().String() + if inner.Kind() == reflect.Struct && inner.NumField() == 1 && inner.Field(0).Kind() == reflect.Interface && !inner.Field(0).IsNil() { + name += "(" + inner.Field(0).Elem().Type().String() + ")" + } + return name +} + +func Test_queryHealthClassThresholds(t *testing.T) { + // The class names go into the journal, so the test pins the text. + tests := []struct { + name string + queries int + failures int + want string + }{ + {"no queries", 0, 0, "healthy"}, + {"two percent", 100, 2, "healthy"}, + {"five percent", 100, 5, "degraded"}, + {"six percent", 100, 6, "degraded"}, + {"half of twenty", 20, 12, "failing"}, + {"half of ten", 10, 5, "failing"}, + {"small sample cannot fail", 9, 9, "healthy"}, + {"small sample cannot degrade", 9, 1, "healthy"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, tc.queries, map[string]int{sampleClassResolveFailed: tc.failures}) + + event, emit := q.evaluate(queryHealthBase, 0, false) + if !emit { + t.Fatal("the first evaluate did not emit") + } + if event.Class != tc.want { + t.Fatalf("class: got %q, want %q", event.Class, tc.want) + } + }) + } +} + +func Test_queryHealthEmitsHeartbeatWithoutClassChange(t *testing.T) { + q := newQueryHealth() + if _, emit := q.evaluate(queryHealthBase, 0, false); !emit { + t.Fatal("the first evaluate did not emit") + } + + for _, minutes := range []int{1, 7, 14} { + at := queryHealthBase.Add(time.Duration(minutes) * time.Minute) + if _, emit := q.evaluate(at, 0, false); emit { + t.Fatalf("emit at minute %d without a class change", minutes) + } + } + + event, emit := q.evaluate(queryHealthBase.Add(15*time.Minute), 0, false) + if !emit { + t.Fatal("no heartbeat at 15 minutes") + } + if event.Class != "healthy" { + t.Fatalf("heartbeat class: got %q, want %q", event.Class, "healthy") + } + if _, emit := q.evaluate(queryHealthBase.Add(16*time.Minute), 0, false); emit { + t.Fatal("a second emit follows the heartbeat") + } +} + +func Test_queryHealthEmitsOnClassChange(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, 100, nil) + event, emit := q.evaluate(queryHealthBase, 0, false) + if !emit || event.Class != "healthy" { + t.Fatalf("first report: emit %v, class %q", emit, event.Class) + } + + feedQueryHealth(q, 0, map[string]int{sampleClassResolveFailed: 50}) + if _, emit := q.evaluate(queryHealthBase.Add(time.Minute), 0, false); emit { + t.Fatal("emit on the first tick of the new class") + } + + event, emit = q.evaluate(queryHealthBase.Add(2*time.Minute), 0, false) + if !emit { + t.Fatal("no emit after the new class held two ticks") + } + if event.Class != "failing" { + t.Fatalf("class: got %q, want %q", event.Class, "failing") + } + + if _, emit := q.evaluate(queryHealthBase.Add(3*time.Minute), 0, false); emit { + t.Fatal("emit without a class change") + } +} + +// Test_queryHealthHoldsAClassChangeForTwoTicks proves that a class which lasts +// one tick reports nothing. A minute of noise must not open an event. +func Test_queryHealthHoldsAClassChangeForTwoTicks(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, 20, nil) + event, emit := q.evaluate(queryHealthBase, 0, false) + if !emit || event.Class != queryHealthHealthy { + t.Fatalf("first report: emit %v, class %q", emit, event.Class) + } + + feedQueryHealth(q, 0, map[string]int{sampleClassResolveFailed: 12}) + if event, emit := q.evaluate(queryHealthBase.Add(time.Minute), 0, false); emit { + t.Fatalf("emit after one tick of class %q", event.Class) + } + + // The next tick grades the same window as degraded, so the failing class + // never held two ticks and never reported. + feedQueryHealth(q, 80, nil) + if event, emit := q.evaluate(queryHealthBase.Add(2*time.Minute), 0, false); emit { + t.Fatalf("emit after the class moved again to %q", event.Class) + } + + event, emit = q.evaluate(queryHealthBase.Add(3*time.Minute), 0, false) + if !emit { + t.Fatal("no emit after the new class held two ticks") + } + if event.Class != queryHealthDegraded { + t.Fatalf("class: got %q, want %q", event.Class, queryHealthDegraded) + } +} + +// Test_queryHealthNeedsASampleBeforeAClassChange proves that the empty window +// which follows a heartbeat reports nothing. The counters start at zero, and +// zero queries say nothing about the query path. +func Test_queryHealthNeedsASampleBeforeAClassChange(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, 20, map[string]int{sampleClassResolveFailed: 12}) + event, emit := q.evaluate(queryHealthBase, 0, false) + if !emit || event.Class != queryHealthFailing { + t.Fatalf("first report: emit %v, class %q", emit, event.Class) + } + + event, emit = q.evaluate(queryHealthBase.Add(15*time.Minute), 0, false) + if !emit || event.Class != queryHealthFailing { + t.Fatalf("window end: emit %v, class %q", emit, event.Class) + } + + for _, minutes := range []int{16, 17} { + at := queryHealthBase.Add(time.Duration(minutes) * time.Minute) + if event, emit := q.evaluate(at, 0, false); emit { + t.Fatalf("emit at minute %d with %d queries and class %q", minutes, event.Queries, event.Class) + } + } +} + +func Test_queryHealthCacheHitRatio(t *testing.T) { + q := newQueryHealth() + event, _ := q.evaluate(queryHealthBase, 0, false) + if event.CacheHitRatio != 0 { + t.Fatalf("ratio without queries: got %v, want 0", event.CacheHitRatio) + } + + feedQueryHealth(q, 20, nil) + for i := 0; i < 5; i++ { + q.countCacheHit() + } + + event, _ = q.evaluate(queryHealthBase.Add(time.Minute), 0, false) + if event.Queries != 20 || event.CacheHits != 5 { + t.Fatalf("counters: got %d queries and %d cache hits, want 20 and 5", event.Queries, event.CacheHits) + } + if event.CacheHitRatio != 0.25 { + t.Fatalf("ratio: got %v, want 0.25", event.CacheHitRatio) + } +} + +func Test_queryHealthResetsCountersAtWindowEnd(t *testing.T) { + q := newQueryHealth() + if _, emit := q.evaluate(queryHealthBase, 0, false); !emit { + t.Fatal("the first evaluate did not emit") + } + feedQueryHealth(q, 20, map[string]int{sampleClassResolveFailed: 12}) + q.countCacheHit() + + closed, emit := q.evaluate(queryHealthBase.Add(15*time.Minute), 0, false) + if !emit { + t.Fatal("no emit at the window end") + } + if closed.Queries != 20 || closed.CacheHits != 1 || closed.Failures[sampleClassResolveFailed] != 12 { + t.Fatalf("the closed window lost its counters: %+v", closed) + } + + q.countFailure(sampleClassResolveFailed) + next, _ := q.evaluate(queryHealthBase.Add(16*time.Minute), 0, false) + if next.Queries != 0 || next.CacheHits != 0 { + t.Fatalf("the counters survived the window end: %+v", next) + } + if next.Failures[sampleClassResolveFailed] != 1 { + t.Fatalf("failures of the new window: got %d, want 1", next.Failures[sampleClassResolveFailed]) + } +} + +func Test_queryHealthReportKeepsItsOwnFailures(t *testing.T) { + q := newQueryHealth() + feedQueryHealth(q, 10, map[string]int{sampleClassResolveFailed: 1}) + + report, _ := q.evaluate(queryHealthBase, 0, false) + q.countFailure(sampleClassResolveFailed) + + if got := report.Failures[sampleClassResolveFailed]; got != 1 { + t.Fatalf("the report shares the live failure map: got %d, want 1", got) + } +} + +func Test_queryHealthLogsJournalEvent(t *testing.T) { + swaps := watchLoggerSwaps(t) + logs := captureDebugMainLog(t) + q := newQueryHealth() + feedQueryHealth(q, 20, map[string]int{ + sampleClassResolveFailed: 8, + sampleClassAllEndpointsFailed: 4, + }) + for i := 0; i < 5; i++ { + q.countCacheHit() + } + + event, emit := q.evaluate(queryHealthBase, 2, true) + if !emit { + t.Fatal("the first evaluate did not emit") + } + q.log(event) + + events := jsonLogEvents(t, logs, queryHealthMessage) + if len(events) != 1 { + t.Fatalf("health lines: got %d, want 1 (global level %s, captured %q) %s", len(events), zerolog.GlobalLevel(), logs.String(), swaps()) + } + logged := events[0] + wantField(t, logged, "journal", true) + wantField(t, logged, "level", "info") + wantField(t, logged, "class", "failing") + wantField(t, logged, "window_s", float64(900)) + wantField(t, logged, "queries", float64(20)) + wantField(t, logged, "cache_hits", float64(5)) + wantField(t, logged, "upstreams_down", float64(2)) + wantField(t, logged, "bypass_active", true) + wantField(t, logged, "cache_hit_ratio", 0.25) + + failures, ok := logged["failures_by_class"].(map[string]any) + if !ok { + t.Fatalf("failures_by_class: got %T, want an object", logged["failures_by_class"]) + } + if len(failures) != 2 { + t.Fatalf("failure classes: got %d, want 2", len(failures)) + } + wantField(t, failures, sampleClassResolveFailed, float64(8)) + wantField(t, failures, sampleClassAllEndpointsFailed, float64(4)) +} + +func Test_queryHealthLoopEmitsUntilStop(t *testing.T) { + swaps := watchLoggerSwaps(t) + logs := captureDebugMainLog(t) + tick := queryHealthTick + defer func() { queryHealthTick = tick }() + queryHealthTick = time.Millisecond + + q := newQueryHealth() + stop := make(chan struct{}) + done := make(chan struct{}) + var once sync.Once + stopLoop := func() { + once.Do(func() { + close(stop) + <-done + }) + } + defer stopLoop() + + go func() { + defer close(done) + q.loop(stop, func() (int, bool) { return 1, true }) + }() + + waitForLogLineOrExplain(t, logs, queryHealthMessage, swaps) + stopLoop() + + events := jsonLogEvents(t, logs, queryHealthMessage) + if len(events) == 0 { + t.Fatal("the loop logged no health event") + } + wantField(t, events[0], "journal", true) + wantField(t, events[0], "upstreams_down", float64(1)) + wantField(t, events[0], "bypass_active", true) +} + +// Test_queryHealthReportsEveryWindowAfterAnEarlyEmit proves that a class +// change in the middle of a window does not make the rest of the window +// vanish. The heartbeat follows the window, not the last report. +func Test_queryHealthReportsEveryWindowAfterAnEarlyEmit(t *testing.T) { + q := newQueryHealth() + if _, emit := q.evaluate(queryHealthBase, 0, false); !emit { + t.Fatal("the first evaluate did not emit") + } + feedQueryHealth(q, 20, map[string]int{sampleClassResolveFailed: 20}) + if _, emit := q.evaluate(queryHealthBase.Add(5*time.Minute), 0, false); emit { + t.Fatal("emit on the first tick of the new class") + } + event, emit := q.evaluate(queryHealthBase.Add(6*time.Minute), 0, false) + if !emit || event.Class != queryHealthFailing { + t.Fatalf("class change at 6 minutes: emit %v, class %q", emit, event.Class) + } + feedQueryHealth(q, 7, nil) + event, emit = q.evaluate(queryHealthBase.Add(15*time.Minute), 0, false) + if !emit { + t.Fatal("the end of the window did not emit") + } + if event.Queries != 27 { + t.Fatalf("queries in the window report: got %d, want 27", event.Queries) + } +} diff --git a/cmd/cli/recovery_events_test.go b/cmd/cli/recovery_events_test.go new file mode 100644 index 00000000..03ca0014 --- /dev/null +++ b/cmd/cli/recovery_events_test.go @@ -0,0 +1,349 @@ +package cli + +import ( + "context" + "reflect" + "strings" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// The two events that bound one recovery pass. +const ( + recoveryBeginMessage = "Recovery begin" + recoveryEndMessage = "Recovery end" +) + +// recoveredUpstreamName is the upstream that the stubbed probe reports. +const recoveredUpstreamName = upstreamPrefix + "0" + +// recoveryEventHarness runs one whole recovery pass. It stubs every seam of +// the pass, because a real pass probes an upstream, reads the host resolvers, +// and rewrites the PF anchor. +type recoveryEventHarness struct { + prog *prog + logs *syncBuffer + dhcpServers []string + target string + recovered string +} + +func newRecoveryEventHarness(t *testing.T) *recoveryEventHarness { + t.Helper() + h := &recoveryEventHarness{ + logs: captureDebugMainLog(t), + prog: &prog{cfg: &ctrld.Config{}, dnsInterceptState: &interceptStateStub{}}, + dhcpServers: []string{"192.168.10.1"}, + target: "127.0.0.53", + recovered: recoveredUpstreamName, + } + stubHeaderSnapshotSources(t) + h.prog.um = newUpstreamMonitor(&ctrld.Config{}) + h.prog.lastNetworkState.Store(sourceTestState("en0", true, "192.0.2.10/24")) + // A stabilizing host defers the DHCP exemption, so the pass rewrites no + // PF anchor. + h.prog.pfStabilizing.Store(true) + + intercept := dnsIntercept + discovery := initializeOsResolverWithSystemNameserversFn + ensure := ensureInterceptDNSTargetFn + wait := waitForUpstreamRecoveryFn + t.Cleanup(func() { + dnsIntercept = intercept + initializeOsResolverWithSystemNameserversFn = discovery + ensureInterceptDNSTargetFn = ensure + waitForUpstreamRecoveryFn = wait + }) + dnsIntercept = true + initializeOsResolverWithSystemNameserversFn = func(bool, string) ([]string, []string) { + return effectiveNameserversOf(h.dhcpServers), h.dhcpServers + } + ensureInterceptDNSTargetFn = func(p *prog, _ []string) { + p.interceptDNSTargetMu.Lock() + defer p.interceptDNSTargetMu.Unlock() + p.interceptDNSTargetSetValue = h.target + p.publishInterceptTarget(h.target) + } + waitForUpstreamRecoveryFn = func(*prog, context.Context, map[string]*ctrld.UpstreamConfig, *recoveryDiagnostic) (string, error) { + return h.recovered, nil + } + return h +} + +// effectiveNameserversOf models the OS resolver list that the root package +// builds: every discovered server with a port, and a synthetic public resolver +// when the network gave no public one. +func effectiveNameserversOf(discovered []string) []string { + effective := make([]string, 0, len(discovered)+1) + for _, server := range discovered { + effective = append(effective, server+":53") + } + return append(effective, "76.76.2.0:53") +} + +// oneRecoveryEvent returns the single event with the given message. +func oneRecoveryEvent(t *testing.T, logs *syncBuffer, message string) map[string]any { + t.Helper() + events := jsonLogEvents(t, logs, message) + if len(events) != 1 { + t.Fatalf("%q events: got %d, want 1", message, len(events)) + } + return events[0] +} + +// snapshotTriggers lists the trigger of every snapshot event, in order. +func snapshotTriggers(t *testing.T, logs *syncBuffer) []string { + t.Helper() + var triggers []string + for _, event := range jsonLogEvents(t, logs, networkSnapshotMessage) { + trigger, _ := event["trigger"].(string) + triggers = append(triggers, trigger) + } + return triggers +} + +func Test_recoveryCompletedJournalsBeginAndEnd(t *testing.T) { + h := newRecoveryEventHarness(t) + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + begin := oneRecoveryEvent(t, h.logs, recoveryBeginMessage) + wantField(t, begin, "journal", true) + wantField(t, begin, "level", "info") + + end := oneRecoveryEvent(t, h.logs, recoveryEndMessage) + wantField(t, end, "journal", true) + wantField(t, end, "outcome", "completed") + wantField(t, end, "recovered_upstream", recoveredUpstreamName) + wantField(t, end, "bypass_active", false) + wantField(t, end, "intercept_target_action", "set") + wantField(t, end, "dhcp_server_count", float64(1)) + if got, want := end["dhcp_servers"], []any{"192.168.10.1"}; !reflect.DeepEqual(got, want) { + t.Fatalf("dhcp_servers = %v, want %v", got, want) + } + + // The network of this recovery does not change, so the end snapshot equals + // the begin snapshot and the writer drops it. + triggers := snapshotTriggers(t, h.logs) + if want := []string{"recovery_begin"}; !reflect.DeepEqual(triggers, want) { + t.Fatalf("snapshot triggers = %v, want %v", triggers, want) + } +} + +// Test_recoveryEndReportsTheDiscoveredDHCPNameservers drives the real OS +// resolver initialization. The effective list carries a synthetic public +// resolver, and a reader who counts it reads a DHCP server that the network +// never gave. +func Test_recoveryEndReportsTheDiscoveredDHCPNameservers(t *testing.T) { + h := newRecoveryEventHarness(t) + previousRead := ctrld.NameserversFn + ctrld.NameserversFn = func() []string { return []string{"192.168.1.1"} } + t.Cleanup(func() { ctrld.NameserversFn = previousRead }) + initializeOsResolverWithSystemNameserversFn = ctrld.InitializeOsResolverWithSystemNameserversReason + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + end := oneRecoveryEvent(t, h.logs, recoveryEndMessage) + wantField(t, end, "dhcp_server_count", float64(1)) + if got, want := end["dhcp_servers"], []any{"192.168.1.1"}; !reflect.DeepEqual(got, want) { + t.Fatalf("dhcp_servers = %v, want %v", got, want) + } +} + +// Test_recoveryJournalsTheReinitializedOSResolver covers the resolver line of +// a recovered network. The line names the nameservers that the recovery +// installed, and an outage report needs them. +func Test_recoveryJournalsTheReinitializedOSResolver(t *testing.T) { + h := newRecoveryEventHarness(t) + previousRead := ctrld.NameserversFn + ctrld.NameserversFn = func() []string { return []string{"192.168.1.1"} } + t.Cleanup(func() { ctrld.NameserversFn = previousRead }) + + h.prog.handleRecoveryForTransition(RecoveryReasonNetworkChange, 0) + + event := oneRecoveryEvent(t, h.logs, "Reinitialized OS resolver with nameservers: [192.168.1.1:53 76.76.2.0:53]") + wantField(t, event, "journal", true) + wantField(t, event, "level", "info") +} + +// Test_recoveryEndSpeedsUpTheDNSConfigurationPoll covers the poll cadence +// after a recovery. The resolver table of the host settles late, and a poll +// every five minutes reports that change long after the outage. +func Test_recoveryEndSpeedsUpTheDNSConfigurationPoll(t *testing.T) { + h := newRecoveryEventHarness(t) + now := time.Date(2026, 9, 17, 9, 0, 0, 0, time.UTC) + previousNow := networkEventsNowFn + networkEventsNowFn = func() time.Time { return now } + t.Cleanup(func() { networkEventsNowFn = previousNow }) + h.prog.dnsConfig = newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return nil, nil }) + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + if got := h.prog.dnsConfig.nextDelay(now); got != dnsConfigFastInterval { + t.Fatalf("DNS configuration poll delay after a recovery = %s, want %s", got, dnsConfigFastInterval) + } +} + +func Test_recoveryJournalsTheRecoveredUpstream(t *testing.T) { + h := newRecoveryEventHarness(t) + h.prog.um.mu.Lock() + h.prog.um.markDown(recoveredUpstreamName, 7, "timer") + h.prog.um.mu.Unlock() + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + event := oneRecoveryEvent(t, h.logs, "Upstream \""+recoveredUpstreamName+"\" recovered; re-applying DNS settings") + wantField(t, event, "journal", true) + wantField(t, event, "upstream", recoveredUpstreamName) + if _, ok := event["down_for_ms"]; !ok { + t.Fatalf("event has no down_for_ms field: %v", event) + } +} + +// Test_recoveryBoundsTheRecoveredUpstreamName covers an upstream key that +// holds a token. The key is operator text, and the journal leaves the host +// for a support case, so no line may carry it. +func Test_recoveryBoundsTheRecoveredUpstreamName(t *testing.T) { + const secret = "SECRET0123" + h := newRecoveryEventHarness(t) + h.recovered = upstreamPrefix + "doh-" + secret + h.prog.um.mu.Lock() + h.prog.um.markDown(h.recovered, 1, "timer") + h.prog.um.mu.Unlock() + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + const boundedName = upstreamPrefix + "custom" + event := oneRecoveryEvent(t, h.logs, "Upstream \""+boundedName+"\" recovered; re-applying DNS settings") + wantField(t, event, "upstream", boundedName) + wantField(t, oneRecoveryEvent(t, h.logs, recoveryEndMessage), "recovered_upstream", boundedName) + if strings.Contains(h.logs.String(), secret) { + t.Fatalf("a log line holds the upstream key: %s", h.logs.String()) + } +} + +func Test_recoveryJournalsTheDHCPBypassLines(t *testing.T) { + h := newRecoveryEventHarness(t) + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + for _, message := range []string{ + "DNS intercept recovery: enabling DHCP bypass (filters stay active)", + "DNS intercept recovery: found DHCP nameservers: [192.168.10.1]", + "DNS intercept recovery complete: disabling DHCP bypass, resuming normal flow", + } { + wantField(t, oneRecoveryEvent(t, h.logs, message), "journal", true) + } +} + +func Test_recoveryJournalsTheMissingDHCPNameservers(t *testing.T) { + h := newRecoveryEventHarness(t) + h.dhcpServers = nil + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + event := oneRecoveryEvent(t, h.logs, "DNS intercept recovery: no DHCP nameservers found") + wantField(t, event, "journal", true) + wantField(t, oneRecoveryEvent(t, h.logs, recoveryEndMessage), "dhcp_server_count", float64(0)) +} + +// wantOneRecoveryEndPerGeneration asserts the two lines that bound one pass: +// one end event and one snapshot beside it. +func wantOneRecoveryEndPerGeneration(t *testing.T, h *recoveryEventHarness) map[string]any { + t.Helper() + end := oneRecoveryEvent(t, h.logs, recoveryEndMessage) + triggers := snapshotTriggers(t, h.logs) + // Nothing changes between the begin and the end of these passes, so the + // snapshot limiter drops the end snapshot as a repeat of the begin one. + if want := []string{"recovery_begin"}; !reflect.DeepEqual(triggers, want) { + t.Fatalf("snapshot triggers = %v, want %v", triggers, want) + } + return end +} + +// Test_recoveryCanceledEndReportsTheCanceledPass covers the canceled outcome. +// A canceled pass releases the bypass, and support reads the end event to tell +// a canceled pass from a pass that never ended. +func Test_recoveryCanceledEndReportsTheCanceledPass(t *testing.T) { + h := newRecoveryEventHarness(t) + waitForUpstreamRecoveryFn = func(*prog, context.Context, map[string]*ctrld.UpstreamConfig, *recoveryDiagnostic) (string, error) { + return "", context.Canceled + } + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + end := wantOneRecoveryEndPerGeneration(t, h) + wantField(t, end, "outcome", "canceled") + wantField(t, end, "bypass_active", false) + wantField(t, end, "recovered_upstream", "") +} + +// Test_recoverySupersededEndReportsNoChange covers the superseded outcome. The +// successor owns the target and the bypass flag, so the end event of the +// superseded pass must not report the state of its successor. +func Test_recoverySupersededEndReportsNoChange(t *testing.T) { + h := newRecoveryEventHarness(t) + waitForUpstreamRecoveryFn = func(p *prog, _ context.Context, _ map[string]*ctrld.UpstreamConfig, _ *recoveryDiagnostic) (string, error) { + // A network change took ownership while the probe ran. + p.recoveryGen.Add(1) + return recoveredUpstreamName, nil + } + + h.prog.handleRecoveryForTransition(RecoveryReasonRegularFailure, 0) + + end := wantOneRecoveryEndPerGeneration(t, h) + wantField(t, end, "outcome", "superseded") + wantField(t, end, "bypass_active", false) + wantField(t, end, "intercept_target_action", "unchanged") +} + +func Test_recoveryCanceledWithNoSuccessorIsJournaled(t *testing.T) { + logs := captureDebugMainLog(t) + p := &prog{} + setupInterceptRecovery(t, p) + + p.recoveryCanceledCleanup(p.recoveryGen.Add(1)) + + event := oneRecoveryEvent(t, logs, "Recovery canceled with no successor; cleared recovery state and DHCP bypass") + wantField(t, event, "journal", true) +} + +func Test_interceptTargetAction(t *testing.T) { + for _, tc := range []struct { + name string + before, after string + want string + }{ + {"no target at all", "", "", "unchanged"}, + {"target kept", "127.0.0.53", "127.0.0.53", "unchanged"}, + {"target written", "", "127.0.0.53", "set"}, + {"target moved", "127.0.0.53", "127.0.0.54", "set"}, + {"target dropped", "127.0.0.53", "", "removed"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := interceptTargetAction(tc.before, tc.after); got != tc.want { + t.Fatalf("interceptTargetAction(%q, %q) = %q, want %q", tc.before, tc.after, got, tc.want) + } + }) + } +} + +func Test_noteNAT64ResultReportsChangesOnly(t *testing.T) { + p := &prog{} + const prefix = "64:ff9b::/96" + + for _, tc := range []struct { + result string + want bool + }{ + {prefix, true}, + {prefix, false}, + {"", true}, + } { + if got := p.noteNAT64Result(tc.result); got != tc.want { + t.Fatalf("noteNAT64Result(%q) = %v, want %v", tc.result, got, tc.want) + } + } +} diff --git a/cmd/cli/recovery_state.go b/cmd/cli/recovery_state.go index fa2e9de4..0eae9cdc 100644 --- a/cmd/cli/recovery_state.go +++ b/cmd/cli/recovery_state.go @@ -32,7 +32,7 @@ func (p *prog) recoveryOwnsState(gen uint64) bool { } func systemNameserversForInterceptRetry() []string { - _, system := initializeOsResolverWithSystemNameserversFn(true) + _, system := initializeOsResolverWithSystemNameserversFn(true, recoveryResolverReason) if system == nil { return []string{} } @@ -71,5 +71,5 @@ func (p *prog) recoveryCanceledCleanup(gen uint64) { // Superseded: the newer recovery owns the shared state. return } - mainLog.Load().Info().Msg("Recovery canceled with no successor; cleared recovery state and DHCP bypass") + journal(mainLog.Load().Info()).Msg("Recovery canceled with no successor; cleared recovery state and DHCP bypass") } diff --git a/cmd/cli/recovery_state_test.go b/cmd/cli/recovery_state_test.go index 4e615ad0..4e821ebb 100644 --- a/cmd/cli/recovery_state_test.go +++ b/cmd/cli/recovery_state_test.go @@ -143,7 +143,7 @@ func TestRecoveryCleanupClearsBypassAfterInterceptStateDisappears(t *testing.T) func TestSystemNameserversForInterceptRetryNormalizesEmptyDiscovery(t *testing.T) { original := initializeOsResolverWithSystemNameserversFn called := false - initializeOsResolverWithSystemNameserversFn = func(guard bool) ([]string, []string) { + initializeOsResolverWithSystemNameserversFn = func(guard bool, _ string) ([]string, []string) { called = true if !guard { t.Error("intercept retry discovery did not guard the existing resolver") diff --git a/cmd/cli/route_darwin.go b/cmd/cli/route_darwin.go new file mode 100644 index 00000000..15d81db4 --- /dev/null +++ b/cmd/cli/route_darwin.go @@ -0,0 +1,158 @@ +package cli + +import ( + "net" + "net/netip" + "syscall" + + "golang.org/x/net/route" +) + +// routeInterfaceNameFn names the interface of one route table index. A test +// replaces it, because the interfaces of the host are not a fixture. +var routeInterfaceNameFn = interfaceNameByIndex + +// defaultRoutes reads the default route of each family. It reads the kernel +// route table, because a snapshot follows every network change and a +// subprocess per change costs too much on a busy host. A table that the kernel +// does not give leaves both routes empty. +func defaultRoutes() (v4, v6 defaultRoute) { + rib, err := route.FetchRIB(syscall.AF_UNSPEC, route.RIBTypeRoute, 0) + if err != nil { + return defaultRoute{}, defaultRoute{} + } + messages, err := route.ParseRIB(route.RIBTypeRoute, rib) + if err != nil { + return defaultRoute{}, defaultRoute{} + } + return defaultRoutesFromMessages(messages) +} + +// defaultRoutesFromMessages picks the first default route of each family. The +// next hop and the interface come from one message, so a reader sees the pair +// that the kernel holds. The kernel keeps one unscoped default per family, so +// the first match is the only one. +func defaultRoutesFromMessages(messages []route.Message) (v4, v6 defaultRoute) { + foundV4, foundV6 := false, false + for _, message := range messages { + match, ok := matchDefaultRoute(message) + if !ok { + continue + } + found := defaultRoute{Interface: routeInterfaceNameFn(match.index)} + if match.gateway.IsValid() { + found.Gateway = match.gateway.String() + } + if match.v4 && !foundV4 { + v4, foundV4 = found, true + } + if !match.v4 && !foundV6 { + v6, foundV6 = found, true + } + if foundV4 && foundV6 { + return v4, v6 + } + } + return v4, v6 +} + +// defaultRouteMatch is one default route of the table. A tunnel default has +// no next hop, so gateway can be invalid while the interface is set. +type defaultRouteMatch struct { + v4 bool + gateway netip.Addr + index int +} + +// defaultRouteGateway reports the next hop and the interface index of one +// default route, for the tests of the selector. See matchDefaultRoute. +func defaultRouteGateway(message route.Message) (netip.Addr, int, bool) { + match, ok := matchDefaultRoute(message) + return match.gateway, match.index, ok +} + +// matchDefaultRoute reports whether one message is the default route of the +// host for its family. A route to another destination or to a part of the +// address space gives nothing: a VPN splits the default into two /1 routes, +// and the zero address alone does not make a route the default. A route bound +// to one interface serves that interface only, a host route is no default, and +// a reject or blackhole route delivers nothing. With the gateway flag, the +// next hop must be a host of the same family. Without it, the link itself +// delivers, which is how a tunnel owns the default; the match then has no +// gateway and names the interface. +func matchDefaultRoute(message route.Message) (defaultRouteMatch, bool) { + routeMessage, ok := message.(*route.RouteMessage) + if !ok || len(routeMessage.Addrs) <= syscall.RTAX_NETMASK { + return defaultRouteMatch{}, false + } + if routeMessage.Flags&(syscall.RTF_IFSCOPE|syscall.RTF_HOST|syscall.RTF_REJECT|syscall.RTF_BLACKHOLE) != 0 { + return defaultRouteMatch{}, false + } + destination, ok := routeMessageAddr(routeMessage.Addrs[syscall.RTAX_DST]) + if !ok || !destination.IsUnspecified() || !zeroNetmask(routeMessage.Addrs[syscall.RTAX_NETMASK], destination.Is4()) { + return defaultRouteMatch{}, false + } + match := defaultRouteMatch{v4: destination.Is4(), index: routeMessage.Index} + if routeMessage.Flags&syscall.RTF_GATEWAY == 0 { + if _, link := routeMessage.Addrs[syscall.RTAX_GATEWAY].(*route.LinkAddr); !link { + return defaultRouteMatch{}, false + } + return match, true + } + gateway, ok := routeMessageAddr(routeMessage.Addrs[syscall.RTAX_GATEWAY]) + if !ok || gateway.Is4() != destination.Is4() { + return defaultRouteMatch{}, false + } + match.gateway = gateway + return match, true +} + +// zeroNetmask reports whether the netmask of a route covers the whole address +// space of its family, which only the default route does. +func zeroNetmask(addr route.Addr, v4 bool) bool { + switch mask := addr.(type) { + case *route.Inet4Addr: + return v4 && mask.IP == [4]byte{} + case *route.Inet6Addr: + return !v4 && mask.IP == [16]byte{} + } + return false +} + +func routeMessageAddr(addr route.Addr) (netip.Addr, bool) { + switch address := addr.(type) { + case *route.Inet4Addr: + return netip.AddrFrom4(address.IP), true + case *route.Inet6Addr: + return withInterfaceZone(netip.AddrFrom16(address.IP), address.ZoneID), true + } + return netip.Addr{}, false +} + +// withInterfaceZone names the interface of a link-local next hop. Every link +// repeats the same range, so the address alone names no host. A tunnel gives +// the bare address fe80:: and the interface is the only part that tells the +// links apart. +func withInterfaceZone(address netip.Addr, zoneID int) netip.Addr { + if zoneID == 0 || !address.IsLinkLocalUnicast() { + return address + } + name := routeInterfaceNameFn(zoneID) + if name == "" { + return address + } + return address.WithZone(name) +} + +// interfaceNameByIndex names one interface of the route table. An index that +// no interface holds gives an empty name. +func interfaceNameByIndex(index int) string { + if index == 0 { + return "" + } + iface, err := net.InterfaceByIndex(index) + if err != nil { + return "" + } + return iface.Name +} diff --git a/cmd/cli/route_darwin_test.go b/cmd/cli/route_darwin_test.go new file mode 100644 index 00000000..8384cf11 --- /dev/null +++ b/cmd/cli/route_darwin_test.go @@ -0,0 +1,291 @@ +package cli + +import ( + "net/netip" + "syscall" + "testing" + + "golang.org/x/net/route" +) + +// stubRouteInterfaceNames names the interface of each index, because the +// interfaces of the host are not a fixture. +func stubRouteInterfaceNames(t *testing.T, names map[int]string) { + t.Helper() + orig := routeInterfaceNameFn + t.Cleanup(func() { routeInterfaceNameFn = orig }) + routeInterfaceNameFn = func(index int) string { return names[index] } +} + +// routeAddrs puts one destination, one netmask, and one gateway at the places +// of the route message that the kernel uses. A nil netmask leaves the slot +// empty, the shape of a host route. +func routeAddrs(destination, netmask, gateway route.Addr) []route.Addr { + addrs := make([]route.Addr, syscall.RTAX_NETMASK+1) + addrs[syscall.RTAX_DST] = destination + addrs[syscall.RTAX_GATEWAY] = gateway + addrs[syscall.RTAX_NETMASK] = netmask + return addrs +} + +func inet4(a, b, c, d byte) *route.Inet4Addr { + return &route.Inet4Addr{IP: [4]byte{a, b, c, d}} +} + +func inet6(address string, zoneID int) *route.Inet6Addr { + addr := netip.MustParseAddr(address) + return &route.Inet6Addr{IP: addr.As16(), ZoneID: zoneID} +} + +func Test_defaultRouteGateway(t *testing.T) { + stubRouteInterfaceNames(t, map[int]string{8: "utun4"}) + + for _, tt := range []struct { + name string + message route.Message + wantGateway string + wantIndex int + wantOK bool + }{ + { + name: "v4_gateway", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 1, 1)), + }, + wantGateway: "192.168.1.1", + wantIndex: 5, + wantOK: true, + }, + { + name: "v6_link_local_gateway_with_a_zone", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet6("::", 0), inet6("::", 0), inet6("fe80::1", 8)), + }, + wantGateway: "fe80::1%utun4", + wantIndex: 8, + wantOK: true, + }, + { + name: "destination_is_not_the_default_route", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(192, 168, 1, 0), inet4(255, 255, 255, 0), inet4(192, 168, 1, 1)), + }, + }, + { + name: "gateway_names_a_link", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), &route.LinkAddr{Index: 5, Name: "en0"}), + }, + }, + { + name: "gateway_family_differs_from_the_destination", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet6("fe80::1", 8)), + }, + }, + { + name: "address_list_holds_no_gateway", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: []route.Addr{inet4(0, 0, 0, 0)}, + }, + }, + { + name: "message_is_not_a_route", + message: &route.InterfaceMessage{Index: 5}, + }, + { + name: "route_has_no_gateway_flag", + message: &route.RouteMessage{ + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 1, 1)), + }, + }, + { + name: "route_is_bound_to_one_interface", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY | syscall.RTF_IFSCOPE, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 1, 1)), + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + gateway, index, ok := defaultRouteGateway(tt.message) + if ok != tt.wantOK { + t.Fatalf("ok = %v, want %v", ok, tt.wantOK) + } + if !ok { + return + } + if gateway.String() != tt.wantGateway { + t.Errorf("gateway = %q, want %q", gateway.String(), tt.wantGateway) + } + if index != tt.wantIndex { + t.Errorf("index = %d, want %d", index, tt.wantIndex) + } + }) + } +} + +// Test_defaultRoutesFromMessages covers the table of a host with a v4 default +// route and a v6 default route on another interface. The v6 route must not +// take the interface of the v4 route. +func Test_defaultRoutesFromMessages(t *testing.T) { + stubRouteInterfaceNames(t, map[int]string{4: "en0", 8: "utun4"}) + + v4, v6 := defaultRoutesFromMessages([]route.Message{ + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY | syscall.RTF_IFSCOPE, + Index: 9, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(10, 0, 0, 1)), + }, + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 4, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 1, 1)), + }, + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet6("::", 0), inet6("::", 0), inet6("fe80::1", 8)), + }, + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 5, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(172, 16, 0, 1)), + }, + }) + + if want := (defaultRoute{Gateway: "192.168.1.1", Interface: "en0"}); v4 != want { + t.Errorf("v4 route = %+v, want %+v", v4, want) + } + if want := (defaultRoute{Gateway: "fe80::1%utun4", Interface: "utun4"}); v6 != want { + t.Errorf("v6 route = %+v, want %+v", v6, want) + } +} + +// Test_defaultRoutesFromMessagesWithoutADefaultRoute covers a table that holds +// no default route, for example a host with no link. +func Test_defaultRoutesFromMessagesWithoutADefaultRoute(t *testing.T) { + stubRouteInterfaceNames(t, map[int]string{4: "en0"}) + + v4, v6 := defaultRoutesFromMessages([]route.Message{ + &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 4, + Addrs: routeAddrs(inet4(192, 168, 1, 0), inet4(255, 255, 255, 0), inet4(192, 168, 1, 1)), + }, + }) + + if v4 != (defaultRoute{}) || v6 != (defaultRoute{}) { + t.Errorf("routes = %+v / %+v, want two empty routes", v4, v6) + } +} + +// Test_defaultRouteGatewayNeedsAZeroNetmask covers routes whose destination is +// the zero address but whose prefix is not /0. OpenVPN splits the default into +// two /1 routes, and a host route to 0.0.0.0 carries no mask at all. +func Test_defaultRouteGatewayNeedsAZeroNetmask(t *testing.T) { + for _, tt := range []struct { + name string + message route.Message + }{ + { + name: "half of the v4 space", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(128, 0, 0, 0), inet4(10, 8, 0, 1)), + }, + }, + { + name: "host route to the zero address", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY | syscall.RTF_HOST, + Index: 8, + Addrs: routeAddrs(inet4(0, 0, 0, 0), nil, inet4(10, 8, 0, 1)), + }, + }, + { + name: "one eighth of the v6 space", + message: &route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet6("::", 0), inet6("e000::", 0), inet6("fe80::1", 8)), + }, + }, + } { + t.Run(tt.name, func(t *testing.T) { + if gateway, _, ok := defaultRouteGateway(tt.message); ok { + t.Fatalf("a %s route counted as the default route with gateway %s", tt.name, gateway) + } + }) + } + v4, _ := defaultRoutesFromMessages([]route.Message{&route.RouteMessage{ + Flags: syscall.RTF_GATEWAY, + Index: 8, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(128, 0, 0, 0), inet4(10, 8, 0, 1)), + }}) + if v4.Gateway != "" || v4.Interface != "" { + t.Fatalf("a table with one /1 route reported the default route %+v, want none", v4) + } +} + +// Test_defaultRoutesTakeALinkDefault covers a VPN that owns the default +// route. Its default has no next hop: the gateway is the link itself and the +// gateway flag is off. The snapshot must name that interface, with an empty +// gateway, instead of the interface of the network monitor. +func Test_defaultRoutesTakeALinkDefault(t *testing.T) { + stubRouteInterfaceNames(t, map[int]string{28: "utun4", 16: "en0"}) + linkDefault := &route.RouteMessage{ + Flags: syscall.RTF_UP | syscall.RTF_STATIC, + Index: 28, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), &route.LinkAddr{Index: 28, Name: "utun4"}), + } + scopedEn0 := &route.RouteMessage{ + Flags: syscall.RTF_UP | syscall.RTF_GATEWAY | syscall.RTF_IFSCOPE, + Index: 16, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(192, 168, 50, 1)), + } + + v4, _ := defaultRoutesFromMessages([]route.Message{linkDefault, scopedEn0}) + if v4.Interface != "utun4" || v4.Gateway != "" { + t.Fatalf("default route = %+v, want utun4 with no gateway", v4) + } +} + +// Test_defaultRouteGatewaySkipsRejectAndBlackholeRoutes covers a kill switch +// that installs a default route to nowhere. Such a route carries the gateway +// flag, but it delivers nothing, so it is not the default route of the host. +func Test_defaultRouteGatewaySkipsRejectAndBlackholeRoutes(t *testing.T) { + for _, tt := range []struct { + name string + flags int + }{ + {"reject", syscall.RTF_UP | syscall.RTF_GATEWAY | syscall.RTF_REJECT}, + {"blackhole", syscall.RTF_UP | syscall.RTF_GATEWAY | syscall.RTF_BLACKHOLE}, + } { + t.Run(tt.name, func(t *testing.T) { + message := &route.RouteMessage{ + Flags: tt.flags, + Index: 8, + Addrs: routeAddrs(inet4(0, 0, 0, 0), inet4(0, 0, 0, 0), inet4(10, 8, 0, 1)), + } + if gateway, _, ok := defaultRouteGateway(message); ok { + t.Fatalf("a %s route counted as the default route with gateway %s", tt.name, gateway) + } + }) + } +} diff --git a/cmd/cli/route_others.go b/cmd/cli/route_others.go new file mode 100644 index 00000000..65f1330a --- /dev/null +++ b/cmd/cli/route_others.go @@ -0,0 +1,10 @@ +//go:build !darwin + +package cli + +// defaultRoutes gives no route. Only macOS reads the route table in this +// release, so the snapshot of another system holds no gateway and no default +// route of the v6 family. +func defaultRoutes() (v4, v6 defaultRoute) { + return defaultRoute{}, defaultRoute{} +} diff --git a/cmd/cli/tunnel_owner.go b/cmd/cli/tunnel_owner.go new file mode 100644 index 00000000..b57b0852 --- /dev/null +++ b/cmd/cli/tunnel_owner.go @@ -0,0 +1,35 @@ +package cli + +import ( + "bufio" + "io" + "strings" +) + +// agentDescriptionKey starts the quoted product name on an ifconfig agent line. +const agentDescriptionKey = `desc:"` + +// parseIfconfigAgent returns the owner of a tunnel from "ifconfig -v utunN" +// output. The owner is the description of the first agent line, and a tunnel that +// no network extension owns has no agent line, so the result is "". +func parseIfconfigAgent(r io.Reader) string { + if r == nil { + return "" + } + scanner := bufio.NewScanner(r) + for scanner.Scan() { + line := strings.TrimSpace(scanner.Text()) + if !strings.HasPrefix(line, "agent ") { + continue + } + _, quoted, ok := strings.Cut(line, agentDescriptionKey) + if !ok { + continue + } + if description, _, ok := strings.Cut(quoted, `"`); ok && description != "" { + return description + } + } + // An owner that the read did not reach is not an owner that does not exist. + return "" +} diff --git a/cmd/cli/tunnel_owner_test.go b/cmd/cli/tunnel_owner_test.go new file mode 100644 index 00000000..76ba6235 --- /dev/null +++ b/cmd/cli/tunnel_owner_test.go @@ -0,0 +1,83 @@ +package cli + +import ( + "strings" + "testing" +) + +// Captured from "ifconfig -v utunN" on macOS. A tunnel that a network extension +// owns carries an agent line with the product name in desc. +const ifconfigWindscribeTunnel = `utun4: flags=8051 mtu 1420 + index 22 + eflags=1002080 + options=6460 + inet 100.96.0.2 --> 100.96.0.2 netmask 0xffffffff + nd6 options=201 + agent domain:NetworkExtension type:NetworkExtension flags:0xf desc:"Windscribe VPN" + agent domain:Multipath type:Path Source flags:0x3 desc:"Multipath path manager" + netif: 6d1e1b2a-4c55-4f0b-9f0a-1d2e3f4a5b6c + scheduler: FQ_CODEL + link quality: 100 (good) +` + +const ifconfigTailscaleTunnel = `utun6: flags=8051 mtu 1280 + index 24 + agent domain:NetworkExtension type:NetworkExtension flags:0xf desc:"Tailscale Tunnel" + nd6 options=201 +` + +const ifconfigSystemTunnel = `utun0: flags=8051 mtu 1500 + index 16 + eflags=1002080 + inet6 fe80::e1a2:3b4c:5d6e:7f80%utun0 prefixlen 64 scopeid 0x10 + nd6 options=201 + scheduler: FQ_CODEL +` + +func Test_parseIfconfigAgent(t *testing.T) { + tests := []struct { + name string + output string + want string + }{ + { + name: "a VPN network extension owns the tunnel", + output: ifconfigWindscribeTunnel, + want: "Windscribe VPN", + }, + { + name: "a mesh client owns the tunnel", + output: ifconfigTailscaleTunnel, + want: "Tailscale Tunnel", + }, + { + name: "no agent line", + output: ifconfigSystemTunnel, + }, + { + name: "no output", + output: "", + }, + { + name: "an agent line without a description", + output: "\tagent domain:Multipath type:Path Source flags:0x3\n\tagent domain:NetworkExtension type:NetworkExtension flags:0xf desc:\"Windscribe VPN\"\n", + want: "Windscribe VPN", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + if got := parseIfconfigAgent(strings.NewReader(tc.output)); got != tc.want { + t.Fatalf("parseIfconfigAgent() = %q, want %q", got, tc.want) + } + }) + } +} + +// Test_parseIfconfigAgentRejectsAnOversizedLine covers a truncated read. An +// owner that the reader did not reach is not an owner that does not exist. +func Test_parseIfconfigAgentRejectsAnOversizedLine(t *testing.T) { + output := strings.Repeat("a", 70<<10) + "\n" + ifconfigWindscribeTunnel + if got := parseIfconfigAgent(strings.NewReader(output)); got != "" { + t.Fatalf("tunnel owner of a truncated read = %q, want an empty owner", got) + } +} diff --git a/cmd/cli/upstream_monitor.go b/cmd/cli/upstream_monitor.go index 432d6110..6d1e62f1 100644 --- a/cmd/cli/upstream_monitor.go +++ b/cmd/cli/upstream_monitor.go @@ -230,6 +230,44 @@ func (um *upstreamMonitor) noteSuccess(upstream string) { um.markUp(upstream, "recovered") } +// countDown reports how many upstreams are down at this time. +func (um *upstreamMonitor) countDown() int { + um.mu.RLock() + defer um.mu.RUnlock() + return len(um.down) +} + +// countDownExcept reports how many upstreams are down at this time, without +// the ones that skip accepts. The grade of the query path leaves out the +// resolvers that only an endpoint on the organization network can reach. +func (um *upstreamMonitor) countDownExcept(skip func(string) bool) int { + if skip == nil { + return um.countDown() + } + um.mu.RLock() + defer um.mu.RUnlock() + count := 0 + for upstream := range um.down { + if skip(upstream) { + continue + } + count++ + } + return count +} + +// downFor reports how long an upstream has been down. An upstream that is up +// gives zero, so a caller logs the field without a branch. +func (um *upstreamMonitor) downFor(upstream string) time.Duration { + um.mu.RLock() + defer um.mu.RUnlock() + state, down := um.down[upstream] + if !down { + return 0 + } + return um.now().Sub(state.since) +} + // isDown reports whether the given upstream is being marked as down. func (um *upstreamMonitor) isDown(upstream string) bool { um.mu.Lock() diff --git a/cmd/cli/upstream_monitor_test.go b/cmd/cli/upstream_monitor_test.go index c45b4e25..68780d86 100644 --- a/cmd/cli/upstream_monitor_test.go +++ b/cmd/cli/upstream_monitor_test.go @@ -229,3 +229,58 @@ func Test_upstreamMonitorKeepsACustomNameOutOfTheJournal(t *testing.T) { } } } + +func Test_upstreamMonitorCountDown(t *testing.T) { + captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}) + if got := um.countDown(); got != 0 { + t.Fatalf("countDown on a new monitor = %d, want 0", got) + } + um.mu.Lock() + um.markDown(upstreamPrefix+"0", 3, "immediate") + um.markDown(upstreamPrefix+"1", 3, "immediate") + um.mu.Unlock() + if got := um.countDown(); got != 2 { + t.Fatalf("countDown with two down upstreams = %d, want 2", got) + } +} + +func Test_upstreamMonitorDownFor(t *testing.T) { + const upstream = upstreamPrefix + "0" + um, _, now := downMonitor(t, upstream) + + *now = now.Add(2500 * time.Millisecond) + + if got := um.downFor(upstream); got != 2500*time.Millisecond { + t.Fatalf("downFor a down upstream = %s, want 2.5s", got) + } + if got := um.downFor(upstreamPrefix + "9"); got != 0 { + t.Fatalf("downFor an unknown upstream = %s, want 0", got) + } + + um.reset(upstream) + + if got := um.downFor(upstream); got != 0 { + t.Fatalf("downFor an upstream that is up = %s, want 0", got) + } +} + +func Test_upstreamMonitorCountDownExcept(t *testing.T) { + captureDebugMainLog(t) + um := newUpstreamMonitor(&ctrld.Config{}) + um.mu.Lock() + um.markDown(upstreamPrefix+"0", 3, "immediate") + um.markDown(upstreamPrefix+internalDomainUpstreamPrefix+"1", 3, "immediate") + um.markDown(upstreamPrefix+internalDomainUpstreamPrefix+"2", 3, "immediate") + um.mu.Unlock() + + if got := um.countDown(); got != 3 { + t.Fatalf("countDown = %d, want 3", got) + } + if got := um.countDownExcept(isInternalDomainUpstream); got != 1 { + t.Fatalf("countDownExcept(isInternalDomainUpstream) = %d, want 1", got) + } + if got := um.countDownExcept(nil); got != 3 { + t.Fatalf("countDownExcept(nil) = %d, want 3", got) + } +} diff --git a/cmd/cli/vpn_dns.go b/cmd/cli/vpn_dns.go index 691cf173..2c8516ae 100644 --- a/cmd/cli/vpn_dns.go +++ b/cmd/cli/vpn_dns.go @@ -2,6 +2,7 @@ package cli import ( "context" + "fmt" "net" "runtime" "strings" @@ -60,10 +61,16 @@ type vpnDNSManager struct { refreshRunning bool refreshPending bool discoveryMu sync.Mutex + // repeats holds the last refresh summary. The manager has no prog, and a + // network change storm runs one refresh per delta. + repeats repeatLogger // Called when VPN DNS server list changes, to update intercept exemptions. onServersChanged vpnDNSExemptFunc } +// vpnDNSRefreshKey names the repeat counter of the refresh summary. +const vpnDNSRefreshKey = "vpn_dns_refresh" + // newVPNDNSManager creates a new manager. Only call when dnsIntercept is active. // exemptFunc is called whenever VPN DNS servers are discovered/changed, to update // the OS-level intercept rules to permit ctrld's outbound queries to those IPs. @@ -220,8 +227,11 @@ func (m *vpnDNSManager) refreshOnce(guardAgainstNoNameservers bool) { } m.domainlessServers = domainlessServers - logger.Debug().Msgf("VPN DNS refresh completed: %d configs, %d routes, %d domainless servers, %d unique exemptions", + summary := fmt.Sprintf("%d configs, %d routes, %d domainless servers, %d unique exemptions", len(m.configs), len(m.routes), len(m.domainlessServers), len(exemptions)) + if changed, repeats := m.repeats.changed(vpnDNSRefreshKey, summary); changed { + logger.Debug().Uint64("repeats", repeats).Msgf("VPN DNS refresh completed: %s", summary) + } // Update intercept rules only when desired exemptions differ from the last // successfully applied set. Failed PF/WFP callbacks remain retryable on the diff --git a/cmd/cli/vpn_dns_test.go b/cmd/cli/vpn_dns_test.go index 94cf9bf3..ebe004ee 100644 --- a/cmd/cli/vpn_dns_test.go +++ b/cmd/cli/vpn_dns_test.go @@ -3,6 +3,7 @@ package cli import ( "context" "errors" + "strings" "sync" "sync/atomic" "testing" @@ -296,3 +297,53 @@ func TestVPNDNSFullAndRouteOnlyDiscoveryAreSerialized(t *testing.T) { t.Fatalf("serialized exemption updates = %v, want old then new", exemptionUpdates) } } + +// eventsWithPrefix returns the captured events whose message starts with the +// prefix. Several lines of this module carry a value in the message, so no +// constant matches the whole of it. +func eventsWithPrefix(t *testing.T, logs *syncBuffer, prefix string) []map[string]any { + t.Helper() + var matched []map[string]any + for _, event := range jsonLogEvents(t, logs, "") { + message, _ := event["message"].(string) + if strings.HasPrefix(message, prefix) { + matched = append(matched, event) + } + } + return matched +} + +// TestVPNDNSRefreshSummaryLogsOnChangeOnly covers the refresh summary. A network +// change storm runs the refresh every few seconds, and the same summary in every +// one of those lines buries the refresh that changed something. +func TestVPNDNSRefreshSummaryLogsOnChangeOnly(t *testing.T) { + logs := captureDebugMainLog(t) + m := newVPNDNSManager(nil) + configs := []ctrld.VPNDNSConfig{{ + InterfaceName: "utun4", + Servers: []string{"10.0.0.1"}, + Domains: []string{"corp.internal"}, + }} + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { return configs } + + m.Refresh(true) + m.Refresh(true) + + summaries := eventsWithPrefix(t, logs, "VPN DNS refresh completed") + if len(summaries) != 1 { + t.Fatalf("got %d refresh summaries for two identical results, want 1", len(summaries)) + } + wantField(t, summaries[0], "repeats", float64(0)) + + configs = append(configs, ctrld.VPNDNSConfig{ + InterfaceName: "utun6", + Servers: []string{"10.0.0.2"}, + }) + m.Refresh(true) + + summaries = eventsWithPrefix(t, logs, "VPN DNS refresh completed") + if len(summaries) != 2 { + t.Fatalf("got %d refresh summaries after the result changed, want 2", len(summaries)) + } + wantField(t, summaries[1], "repeats", float64(1)) +} diff --git a/docs/dns-intercept-mode.md b/docs/dns-intercept-mode.md index 29463def..cb362f05 100644 --- a/docs/dns-intercept-mode.md +++ b/docs/dns-intercept-mode.md @@ -235,6 +235,10 @@ DNS intercept: pf anchor "com.controld.ctrld" active with 3 rules DNS intercept: pf redirect active — all outbound DNS (port 53) redirected to 127.0.0.1:53 ``` +DNS intercept mode on macOS also runs the wake detector. That detector reports a `Host woke` journal event with the source `detector` and the length of the sleep. +Outside DNS intercept mode the netmon delta is the only wake source, and it reports no length of its own. +The events of the journal are listed in [Network-recovery diagnostics](network-recovery-diagnostics.md#journal-events). + ## Troubleshooting ### Windows @@ -306,12 +310,14 @@ VPN apps commonly add rules like `pass out quick on ipsec0 inet all` that match ### 3. Dynamic Tunnel Interface Detection -The network change monitor (`validInterfacesMap()`) only tracks physical hardware ports (en0, bridge0, etc.) — it doesn't see tunnel interfaces (utun*, ipsec*, etc.) created by VPN software. When a VPN connects and creates a new interface (e.g., utun420 for WireGuard), ctrld detects this through a separate tunnel interface change check and rebuilds the pf anchor to include explicit intercept rules for the new interface. This runs on every network change event, even if no physical interface changed. +The network change monitor (`validInterfacesMap()`) only tracks physical hardware ports (en0, bridge0, etc.) — it doesn't see tunnel interfaces (utun*, ipsec*, etc.) created by VPN software. When a VPN connects and creates a new interface (e.g., utun420 for WireGuard), ctrld detects this through a separate tunnel interface change check and rebuilds the pf anchor to include explicit intercept rules for the new interface. This runs on every network change event that is not noise, even if no physical interface changed. A delta that touches AirDrop or virtual adapters alone is noise, and ctrld skips the handler for it, so no tunnel discovery runs. ### 4. pf Watchdog + Network Change Hooks A background watchdog (30s interval) plus immediate checks on network change events detect when another program replaces the entire pf ruleset (e.g., Windscribe's `pfctl -f /etc/pf.conf`). When detected, ctrld rebuilds its anchor with up-to-date tunnel interface rules and re-injects the anchor reference at the top of the ruleset. A 2-second delayed re-check catches race conditions where the other program clears rules slightly after the network event. +A noise delta starts no immediate check and no delayed re-check. The watchdog keeps its own 30 s cadence, so it still restores a missing anchor during a delta storm. + ### 4a. Active Interception Probe (pf Translation State Corruption) Programs like Parallels Desktop reload `/etc/pf.conf` when creating/destroying virtual network interfaces (bridge100, vmenet0). This can corrupt pf's internal translation engine — rdr rules survive in text form but stop evaluating, causing DNS interception to silently fail while the watchdog reports "intact." @@ -639,6 +645,8 @@ Network change detected (netmon callback) └─ (same as 2s — catches slower VPN teardowns) ``` +A noise delta skips this whole chain. ctrld runs no pf read, no tunnel discovery, no OS resolver read, and no VPN DNS refresh for it. The noise class holds the AirDrop interfaces (`awdl*`, `llw*`) and the virtual adapters. The pf watchdog keeps its own 30 s cadence, so it still tests the rules during a delta storm. + ### VPN Connect Sequence 1. VPN creates tunnel interface (e.g., utun420) diff --git a/docs/network-recovery-diagnostics.md b/docs/network-recovery-diagnostics.md index 856c679d..d676ae1c 100644 --- a/docs/network-recovery-diagnostics.md +++ b/docs/network-recovery-diagnostics.md @@ -8,10 +8,139 @@ Every `ctrld log send` upload holds the whole journal, so a reproduction does no Match timestamps and IDs. The last retained error does not describe the current network in every case. Each upload starts with a header line that names the build, the intercept mode, and the network at send time. +## Journal events + +These events describe the host network. Each one carries `journal=true`, so it stays on disk after a rotation and after a restart. +netmon reports every network change as a delta, which holds the state before the change and the state after it. +The table gives the trigger, the level, and the fields of each event. + +| Event | Trigger | Level | Fields | +|---|---|---|---| +| `Network snapshot` | Start, an accepted transition, a recovery begin, and a recovery end. Two limits hold some of them back | info | `trigger` (`start`, `transition`, `recovery_begin`, `recovery_end`), `network` | +| `Network interface changed` | One event per changed interface of a delta that is not noise. A mixed delta gives an event for the interfaces outside the noise class only | info | `transition_id`, `interface`, `action`, `class`, `hardware_port`, `service`, `ips_before`, `ips_after`, `flags`, `mtu`, `is_default_route` | +| `Network transition` | The end of every network callback that is not noise | info, debug for the outcomes `ignored` and `snapshot_superseded` | `transition_id`, `outcome`, `is_major_change`, `changed`, `interface`, `changed_interfaces`, `time_jumped`, `default_route_before`, `default_route_after`, `have_v4_before`, `have_v4_after`, `have_v6_before`, `have_v6_after`, `source_ipv4_before`, `source_ipv4_after`, `source_ipv6_before`, `source_ipv6_after`, `source_snapshot_available` | +| `Network delta noise` | Each noise delta, and one journal line per 10 minutes of a storm | debug, info for the journal line | `interfaces`, `count`, `first_at`, `last_at` | +| `Host woke` | One event per wake: a time jump in a netmon delta, or the macOS wake detector, which runs in DNS intercept mode only | info | `gap_known`, `gap_ms` (only when `gap_known` is true), `source` (`netmon`, `detector`), `default_route`, `interfaces_up` | +| `OS resolver set changed` | A nameserver read that differs from the read before | info | `source` (`scutil`, `dhcp`, `resolv.conf`, `unknown`), `before`, `after`, `default_route`, `reason` (`start`, `recovery`, `delayed_recheck`, `wake_probe`, `vpn_settle`, `unspecified`) | +| `Reinitialized OS resolver with nameservers: …` | ctrld read the nameservers of the system again and built the OS resolver again | info | none, the nameservers are part of the message | +| `DNS configuration changed` | A `scutil --dns` poll that finds a changed resolver, one event per resolver (macOS) | info | `nameservers`, `if_index`, `interface`, `scoped`, `action` (`added`, `changed`, `removed`), `flags`, `search_domain_count`, `order`, `reachable` | +| `Recovery begin` | The start of every recovery | info | `transition_id`, `recovery_generation`, `recovery_reason`, `intercept` | +| `Recovery end` | The end of every recovery | info, warn for a canceled recovery and for a recovery with a failure | `transition_id`, `recovery_generation`, `recovery_reason`, `outcome`, `had_failure`, `duration_ms`, `recovered_upstream`, `dhcp_servers`, `dhcp_server_count`, `intercept_target_action`, `bypass_active` | +| `Upstream … recovered …` | An upstream that was down answers again | info | `down_for_ms` | +| `Recovery canceled with no successor …` | A recovery stops and no other recovery follows it | info | none | +| `DNS intercept recovery: enabling DHCP bypass …` | ctrld sends the queries of a recovery to the OS resolver | info | none | +| `DNS intercept recovery complete: disabling DHCP bypass …` | The recovery ends and the normal query path returns | info | none | +| `DNS intercept recovery: found DHCP nameservers: …` | ctrld read the DHCP nameservers of the service | info | `dhcp_servers` | +| `DNS intercept recovery: no DHCP nameservers found` | The service gave no DHCP nameserver | warn | none | +| `intercept DNS target: service … set …` | ctrld wrote the loopback target into a network service | warn | `service`, `target`, `reason` | +| `intercept DNS target: removed …` | ctrld took the loopback target out of the service | info | `service`, `target`, `reason` | +| `intercept DNS target: … DNS changed externally …` | Another program changed the DNS of the service, so the target of ctrld is gone (macOS) | info | `service`, `target`, `reason` (`external_change`) | +| `dns64: discovered NAT64 prefix …` | A NAT64 prefix that differs from the result before | info | none | +| `dns64: no NAT64 prefix present …` | A no-prefix result that differs from the result before | info | none | +| `DNS intercept: interface appeared/disappeared …` | An interface appeared or disappeared, and the probe monitor starts | info | `interface`, `class`, `action` | +| `PF anchor list changed` | A missing anchor, a restored anchor, a stabilization start, and a stabilization end (macOS) | warn for the reason `missing`, info for the rest | `reason`, `anchors`, `anchors_known`, `pf_enabled`, `pf_since` | +| `Tunnel interface changed` | The set of tunnel interfaces changes (macOS). A retry of the same change writes no second event | info | `added`, `removed`, `owner` | +| `DNS intercept: VPN connecting …` | A new tunnel interface starts the stabilization mode (macOS) | info | none | +| `DNS intercept: pf stable for …` | The pf ruleset stops changing and stabilization ends (macOS) | info | none | +| `Query health` | A change of the health class that holds for two evaluations, and a heartbeat every 15 minutes | info | `class`, `window_s`, `queries`, `failed_queries`, `cache_hits`, `failures_by_class`, `upstreams_down`, `bypass_active`, `cache_hit_ratio` | + +No event in this catalog holds a query name, an upstream URL, or a token. +The journal does hold network addresses. It holds the LAN and VPN resolver addresses, the gateways, the addresses of the host, and the DHCP servers of a service. +It holds no search suffix. `DNS configuration changed` counts the suffixes in `search_domain_count` and logs the count alone. +The field `owner` of `Tunnel interface changed` names the VPN product that owns the tunnel. On Windows the field `hardware_port` names the VPN product of an adapter in the same way. +The Wi-Fi network name is never logged. On macOS 14 and later, only a process with the Location Services permission can read it, and a daemon cannot request that permission. + +### The network object + +The `network` object describes the host network at one moment. +The `Network snapshot` event and the header line of every log file render the same object, with these fields: + +- Routes: `default_route_v4`, `default_route_v6`, `gateway_v4`, `gateway_v6` +- Families: `have_v4`, `have_v6` +- Interfaces: `interfaces`, an array of `name`, `class`, `up`, `ips`, `hardware_port`, `service`, and the count `interfaces_omitted` +- Resolvers: `resolvers`, the host and the port of each OS nameserver +- Source addresses: `source_ipv4`, `source_ipv6` +- Intercept state: `intercept_target`, `dns_less_target`, `bypass_active`, `recovery_running`, `pf_stabilizing` +- Link: `link_type`, `tethered`, `clat_present`, `nat64_prefix` + +The field `default_route_interface` is gone. Read `default_route_v4` and `default_route_v6` in its place. +The two routes and the two gateways come from the route table of macOS, one route per address family. A tunnel that owns the default route has no next hop, so its route names the tunnel and its gateway stays empty. +Windows and Linux read no route table in this release. They fill `default_route_v4` from the network monitor and leave `default_route_v6`, `gateway_v4`, and `gateway_v6` empty. + +`link_type` is one of `wifi`, `ethernet`, `usb_tether`, `tunnel`, and `unknown`. +macOS reads it from the name of the hardware port. Linux reads `wifi`, `ethernet`, `usb_tether`, and `tunnel` from the kernel. Windows names a tunnel and leaves the other links unknown. +The field `service` names a macOS network service. It stays empty on Windows and on Linux. +`tethered` is true on every platform when the interface of the default route holds an address of the range 172.20.10.0/28. +It is also true when the default gateway is in that range, and when the hardware port of the default route is `iPhone USB`. + +The list `interfaces` holds the interfaces that are up or hold an address, and it stops at 32 entries. +`interfaces_omitted` counts the interfaces that the list leaves out, the ones over the limit included. +A container host holds hundreds of interfaces, and a list of all of them fills the journal. + +ctrld does not write a snapshot for every trigger. A snapshot equal to the snapshot written last reaches no journal. +Every other snapshot waits 60 s after the snapshot written last. A new value of `default_route_v4`, `default_route_v6`, or `resolvers` passes that wait. +The trigger `start` always writes. + +### Noise class + +A noise delta changes interfaces of the noise class only, and it does not move the default route. +The noise class holds `awdl*` and `llw*` on macOS. +On Linux it holds the names `docker*`, `veth*`, `virbr*`, and `br-*`, plus the devices of the tun and tap types. +On Windows it holds the virtual adapters without a connector. A Windows adapter whose description names TAP, Wintun, WireGuard, or VPN is a tunnel, not a noise interface. +On Linux and on Windows, an interface that holds a global address on either side of the delta is never noise. +A delta is also not noise when it flips `have_v4` or `have_v6`, or when it carries a time jump. +A noise delta runs no reconcile work. +ctrld starts no `scutil` process, reads no PF rule, discovers no tunnel, refreshes no VPN DNS, and schedules no delayed recheck for it. +A delta that touches a hardware port and a noise interface in the same callback is not noise. +The PF watchdog keeps its own 30 s timer, so it still tests the PF rules during a delta storm. + +Each noise delta logs one debug line. The first noise delta of a storm also writes a journal line with `count` 1. +While the storm lasts, one journal line of the same name arrives every 10 minutes, with the count of that window. +AirDrop in the Finder is the common source of such a storm on macOS. + +### DNS configuration poll (macOS) + +ctrld polls `scutil --dns` on macOS only. The first poll runs at start. +The poll runs every 15 s for 10 minutes after a network change, a wake, or the end of a recovery. +Each of these three events starts the fast cadence at once, even while the poll waits. +At rest the poll runs every 5 minutes. Each changed resolver of a poll gives one `DNS configuration changed` event. + +### Change-only lines + +These lines log at debug level on change only. Each one carries `repeats`, the number of equal results before it: + +- `Got system nameservers` +- `Final available nameservers` +- `pf anchor intact` +- `discovered active tunnel interfaces` +- `VPN DNS refresh completed` + +### PF anchor list (macOS) + +A `PF anchor list changed` event reaches the journal when its reason, its anchor list, or `pf_enabled` differs from the event before it. +The field `pf_since` is no part of that test, because the uptime of pf grows between two reads of one state. +A failed read of the anchor names gives `anchors_known` false and no anchor list. + +### Host woke + +ctrld writes one `Host woke` event per wake. Two sources that report the same wake inside 30 s give one event. +The macOS wake detector runs in DNS intercept mode only. Outside that mode, and on Windows and Linux, the netmon delta is the only source. +The netmon source knows no gap, so its report waits 8 s for the detector. A detector report inside that hold gives the event, with `gap_known` true and the length of the sleep in `gap_ms`. Without a detector report, the netmon report goes out with `gap_known` false and no `gap_ms`. Every later report of the same wake is dropped. + +### Query health + +ctrld grades the query path over a window of 15 minutes and reads the window every minute. +Each failed client query counts one failure, whatever number of upstreams that query tried. +`failures_by_class` keeps one count per failure class. The class `internal_domain` holds the failures of an Internal Domain resolver. +The grade leaves the class `internal_domain` out, and `upstreams_down` counts no Internal Domain resolver. +A laptop away from the company network cannot reach these resolvers. Without this rule the grade stays degraded for the whole day. +A change of the class reports after it holds for two reads of the window. + ## Transitions and recovery -`Network transition` debug events include a `transition_id`, affected interface, default route, and stored source addresses before and after the callback. -The outcome identifies an accepted, ignored, superseded, or unusable change. +`Network transition` events include a `transition_id`, affected interface, default route, and stored source addresses before and after the callback. +The outcome identifies an accepted, ignored, superseded, snapshot_superseded, or unusable change. +A callback whose state read a newer snapshot gives the outcome `snapshot_superseded` at debug level with `transition_id` 0, and it gives no interface event. Newer accepted changes supersede pending recovery. Ignored changes do not cancel accepted recovery. Source commits validate addresses against the current network state. Netmon does not order minor callbacks within one major cache epoch. @@ -22,8 +151,9 @@ This read does not change host DNS. A failed read defers source writes, but reco The warning gives the reason. Every warning enters the journal, so it stays on disk after a debug rotation and after a restart. `Recovery begin` and `Recovery end` connect the transition to a `recovery_generation`, reason, outcome, and elapsed time. Only the first upstream failure in each recovery produces a bounded summary. -If a recovery encounters a failure, its final outcome also enters the journal. -Normal successful recovery stays at debug level. +Every recovery outcome enters the journal. +A canceled recovery and a recovery with a failure log at warn level. +For the outcome `superseded`, `intercept_target_action` is `unchanged` and `bypass_active` is false. The recovery that follows owns both of them. These summaries exclude upstream URLs and queried names. Each PF probe records the recovery generation at its start. diff --git a/docs/pf-dns-intercept.md b/docs/pf-dns-intercept.md index 488f56e4..c241d5e6 100644 --- a/docs/pf-dns-intercept.md +++ b/docs/pf-dns-intercept.md @@ -289,6 +289,13 @@ The monitor probes with exponential backoff (0, 0.5, 1, 2, 4s) to win the race against async pf reloads. Only one monitor runs at a time (singleton). The watchdog also runs the probe every 30s as a safety net. +A network delta that touches AirDrop or virtual adapters alone is noise. The +handler returns at once for it, so no probe monitor starts and no pf read runs. +A delta that moves the default route, that flips the v4 or the v6 family, or +that carries a time jump is not noise, whatever its interfaces are. +The watchdog keeps its own 30 s cadence, so it still tests the rules during a +delta storm. + The full pf reload is VPN-safe: it reassembles from `pfctl -sr` + `pfctl -sn` (the current running state), preserving all existing anchors and rules. diff --git a/docs/runtime-internal-logging.md b/docs/runtime-internal-logging.md index 8a52cadc..26fad074 100644 --- a/docs/runtime-internal-logging.md +++ b/docs/runtime-internal-logging.md @@ -45,12 +45,25 @@ The message of the line is `Log header`. The line carries these fields: - Mode: `intercept_mode`, `listeners`, `upstream_count`, and `upstream_types` - Identity: `resolver_uid` - Files: `log_files` -- Network: `network` with `default_route_interface`, `have_v4`, `have_v6`, and `interfaces` +- Network: `network`, the network snapshot of the host The header holds no provisioning token, no upstream endpoint URL, and no query name. `upstream_types` holds the type words, for example `doh` or `os`. `listeners` holds the listen addresses. +The `network` object holds the routes, the gateways, the interfaces, the resolvers, the link type, and the intercept state. It has no `default_route_interface` field. The runbook [Network-recovery diagnostics](network-recovery-diagnostics.md) lists every field of the object. + After each rotation ctrld logs one `Log rotated` event at info level, with the fields `file`, `bytes_written`, `first_event_at`, `last_event_at`, and `backups`. This event is a journal event, so it lands in the debug stream and in the journal. +## Network state events + +The journal holds the events that describe the host network. `Network snapshot` renders the `network` object at start, at each accepted transition, and at each begin and end of a recovery. +A snapshot equal to the snapshot written last reaches no journal, and one snapshot per 60 s is the limit. A new default route and a new resolver set pass that limit. `Network interface changed`, `Network transition`, `Host woke`, `OS resolver set changed`, and `DNS configuration changed` name each change of the network. `Recovery begin`, `Recovery end`, `PF anchor list changed`, and `Tunnel interface changed` name the repair work. + +AirDrop and virtual adapters produce a storm of network callbacks. ctrld puts these interfaces in a noise class. A delta that touches noise interfaces alone skips the handler. The first delta of a storm writes one `Network delta noise` journal line, and one more line covers each 10 minutes of the storm. + +`Query health` grades the query path over a window of 15 minutes. It logs at each change between the classes `healthy`, `degraded`, and `failing`, and once every 15 minutes as a heartbeat. A class change reports after it holds for two reads of the window. + +The runbook [Network-recovery diagnostics](network-recovery-diagnostics.md) lists every event with its trigger, its level, and its fields. + ## Sampling of per-query errors A broken network produces one error line per query. ctrld bounds this flood for each error class and each upstream. The first 5 lines in a 60 s window stay at error level. Later lines in the same window drop to debug level, so the debug stream stays complete. A run with `-v` holds every stream at info, and the debug stream then loses these lines. @@ -79,7 +92,7 @@ Both keys also apply to the `log_path` file, which rotates the same way. The jou - The marker `=== LOG_END ===` - Every journal file, oldest first -The `network` object of the send-time header comes from a fresh read of the interfaces and the default route. `ctrld log send --full` sends every debug file in place of the newest 10 MB. The reported size equals the number of bytes uploaded. +The `network` object of the send-time header comes from a fresh read of the interfaces and of the route table. The resolvers come from the list that ctrld stored at its last resolver initialization, not from a fresh read. `ctrld log send --full` sends every debug file in place of the newest 10 MB. The reported size equals the number of bytes uploaded. `ctrld log view` prints the same composition on the terminal and takes the same `--full` flag. ctrld accepts one upload per minute. The body is plain text without gzip. diff --git a/log.go b/log.go index 14c82e8a..28ac7704 100644 --- a/log.go +++ b/log.go @@ -32,3 +32,12 @@ func Log(ctx context.Context, e *zerolog.Event, format string, v ...any) { return fmt.Sprintf("[%s] %s", id, fmt.Sprintf(format, v...)) }) } + +// JournalField marks an event for the retained journal stream. +const JournalField = "journal" + +// Journal marks an event for the retained journal stream. The journal writer +// keeps every line that carries the field, whatever the level of the line is. +func Journal(e *zerolog.Event) *zerolog.Event { + return e.Bool(JournalField, true) +} diff --git a/net.go b/net.go index 7bbf54bb..81a75579 100644 --- a/net.go +++ b/net.go @@ -6,8 +6,6 @@ import ( "sync/atomic" "time" - "tailscale.com/net/netmon" - ctrldnet "github.com/Control-D-Inc/ctrld/internal/net" ) @@ -25,26 +23,16 @@ func HasIPv6() bool { val := ctrldnet.IPv6Available(ctx) ipv6Available.Store(val) ProxyLogger.Load().Debug().Msgf("ipv6 availability: %v", val) - mon, err := netmon.New(func(format string, args ...any) {}) - if err != nil { - ProxyLogger.Load().Debug().Err(err).Msg("failed to monitor IPv6 state") - return - } - mon.RegisterChangeCallback(func(delta *netmon.ChangeDelta) { - old := ipv6Available.Load() - cur := delta.Monitor.InterfaceState().HaveV6 - if old != cur { - ProxyLogger.Load().Warn().Msgf("ipv6 availability changed, old: %v, new: %v", old, cur) - } else { - ProxyLogger.Load().Debug().Msg("ipv6 availability does not changed") - } - ipv6Available.Store(cur) - }) - mon.Start() }) return ipv6Available.Load() } +// SetIPv6Available stores the IPv6 state that the network monitor reports, so +// one monitor serves the whole process. +func SetIPv6Available(v bool) { + ipv6Available.Store(v) +} + // DisableIPv6 marks IPv6 as unavailable if enabled. func DisableIPv6() { if ipv6Available.CompareAndSwap(true, false) { diff --git a/net_ipv6_test.go b/net_ipv6_test.go new file mode 100644 index 00000000..37ca2ea3 --- /dev/null +++ b/net_ipv6_test.go @@ -0,0 +1,37 @@ +package ctrld + +import ( + "sync" + "testing" +) + +// resetIPv6Probe clears the one-time IPv6 probe. Test only: the process runs +// the probe once, so a test cannot see it without this reset. +func resetIPv6Probe(t *testing.T) { + t.Helper() + reset := func() { + hasIPv6Once = sync.Once{} + ipv6Available.Store(false) + } + reset() + t.Cleanup(reset) +} + +func TestHasIPv6KeepsTheProbeThenTakesTheStoredFlag(t *testing.T) { + resetIPv6Probe(t) + + probed := HasIPv6() + if got := HasIPv6(); got != probed { + t.Fatalf("HasIPv6 = %v on the second call, want the probe value %v", got, probed) + } + + SetIPv6Available(!probed) + if got := HasIPv6(); got == probed { + t.Fatalf("HasIPv6 = %v after SetIPv6Available(%v), want the stored value", got, !probed) + } + + SetIPv6Available(false) + if HasIPv6() { + t.Fatal("HasIPv6 = true after SetIPv6Available(false), want false") + } +} diff --git a/resolver.go b/resolver.go index 08353a35..959d57a9 100644 --- a/resolver.go +++ b/resolver.go @@ -65,12 +65,64 @@ func init() { } var ( - resolverMutex sync.Mutex - or *osResolver + resolverMutex sync.Mutex + or *osResolver + // osResolverPtr holds the same resolver as or, for readers alone. An + // initialization holds resolverMutex across a scutil read of several + // seconds, and a log header render must not wait for it. + osResolverPtr atomic.Pointer[osResolver] defaultLocalIPv4 atomic.Value // holds net.IP (IPv4) defaultLocalIPv6 atomic.Value // holds net.IP (IPv6) ) +// storeOsResolver keeps the resolver of the process and the copy that readers +// take. The caller holds resolverMutex. +func storeOsResolver(resolver *osResolver) { + or = resolver + osResolverPtr.Store(resolver) +} + +// NameserversFn reads the nameservers of the system. A test of this package +// or of a package that drives the OS resolver replaces it to stay off the host. +var NameserversFn = nameservers + +// osResolverReasonUnspecified marks a resolver read that no caller explained. +const osResolverReasonUnspecified = "unspecified" + +// nameserverReads remembers the last nameserver list of one read. The system +// reports the same list many times, and only a change is news. +type nameserverReads struct { + seen bool + last []string + repeats int +} + +// record stores a new read. It reports whether the list changed, how many +// reads repeated the list before it, and the list that this read replaces. +func (n *nameserverReads) record(list []string) (changed bool, repeats int, before []string) { + if n.seen && slices.Equal(n.last, list) { + n.repeats++ + return false, n.repeats, n.last + } + before, repeats = n.last, n.repeats + n.seen = true + n.last = slices.Clone(list) + n.repeats = 0 + return true, repeats, before +} + +// osResolverLogState keeps the last nameserver reads and the reason of the +// initialization that runs. The read sits deep in the call chain, so the +// reason cannot travel as a parameter. +type osResolverLogState struct { + mu sync.Mutex + system nameserverReads + final nameserverReads + reason string +} + +var osResolverLog = osResolverLogState{reason: osResolverReasonUnspecified} + func newLocalResolver() Resolver { var nss []string for _, addr := range Rfc1918Addresses() { @@ -117,9 +169,8 @@ func availableNameservers() []string { "Added local IP to OS resolverexclusion map: %s", ipStr) } - systemNameservers := nameservers() - Log(context.Background(), logger.Debug(), - "Got system nameservers: %v", systemNameservers) + systemNameservers := NameserversFn() + logSystemNameservers(systemNameservers) for _, ns := range systemNameservers { if _, ok := machineIPsMap[ns]; ok { @@ -132,19 +183,89 @@ func availableNameservers() []string { "Added non-local nameserver: %s", ns) } - Log(context.Background(), logger.Debug(), - "Final available nameservers: %v", nss) + logFinalNameservers(nss) return nss } +// logSystemNameservers logs the system read at debug level on change only. +func logSystemNameservers(list []string) { + osResolverLog.mu.Lock() + changed, repeats, _ := osResolverLog.system.record(list) + osResolverLog.mu.Unlock() + if !changed { + return + } + logger := *ProxyLogger.Load() + Log(context.Background(), logger.Debug().Int("repeats", repeats), + "Got system nameservers: %v", list) +} + +// logFinalNameservers logs the effective read at debug level on change only, +// and sends each change to the journal, where an outage report needs it. +func logFinalNameservers(list []string) { + osResolverLog.mu.Lock() + changed, repeats, before := osResolverLog.final.record(list) + reason := osResolverLog.reason + osResolverLog.mu.Unlock() + if !changed { + return + } + logger := *ProxyLogger.Load() + Log(context.Background(), logger.Debug().Int("repeats", repeats), + "Final available nameservers: %v", list) + Journal(logger.Info()). + Str("source", osResolverSource()). + Strs("before", before). + Strs("after", list). + Str("default_route", defaultRouteInterfaceName()). + Str("reason", reason). + Msg("OS resolver set changed") +} + +// osResolverSource names the place where the platform holds the nameservers. +func osResolverSource() string { + switch runtime.GOOS { + case "darwin": + return "scutil" + case "windows": + return "dhcp" + case "linux", "dragonfly", "freebsd", "netbsd", "openbsd": + return "resolv.conf" + } + return "unknown" +} + +// defaultRouteInterfaceName returns the interface of the default route, and an +// empty string when the route table does not answer. +func defaultRouteInterfaceName() string { + name, err := netmon.DefaultRouteInterface() + if err != nil { + return "" + } + return name +} + +// setOsResolverReason keeps the reason of the initialization that runs. +func setOsResolverReason(reason string) { + osResolverLog.mu.Lock() + defer osResolverLog.mu.Unlock() + osResolverLog.reason = reason +} + // InitializeOsResolver initializes OS resolver using the current system DNS settings. // It returns the nameservers that is going to be used by the OS resolver. // // It's the caller's responsibility to ensure the system DNS is in a clean state before // calling this function. func InitializeOsResolver(guardAgainstNoNameservers bool) []string { - ns, _ := InitializeOsResolverWithSystemNameservers(guardAgainstNoNameservers) - return ns + return InitializeOsResolverWithReason(guardAgainstNoNameservers, osResolverReasonUnspecified) +} + +// InitializeOsResolverWithReason is InitializeOsResolver with the reason that +// the journal reports when the resolver set changes. +func InitializeOsResolverWithReason(guardAgainstNoNameservers bool, reason string) []string { + effective, _ := InitializeOsResolverWithSystemNameserversReason(guardAgainstNoNameservers, reason) + return effective } // InitializeOsResolverWithSystemNameservers initializes the OS resolver and @@ -152,9 +273,19 @@ func InitializeOsResolver(guardAgainstNoNameservers bool) []string { // discovered from the system. The latter deliberately excludes synthetic // fallbacks added by initializeOsResolver. func InitializeOsResolverWithSystemNameservers(guardAgainstNoNameservers bool) (effective, system []string) { + return InitializeOsResolverWithSystemNameserversReason(guardAgainstNoNameservers, osResolverReasonUnspecified) +} + +// InitializeOsResolverWithSystemNameserversReason is +// InitializeOsResolverWithSystemNameservers with the reason that the journal +// reports when the resolver set changes. +func InitializeOsResolverWithSystemNameserversReason(guardAgainstNoNameservers bool, reason string) (effective, system []string) { resolverMutex.Lock() defer resolverMutex.Unlock() + setOsResolverReason(reason) + defer setOsResolverReason(osResolverReasonUnspecified) + system = availableNameservers() if system == nil { // A non-nil empty slice means discovery completed and found no DNS. @@ -165,7 +296,7 @@ func InitializeOsResolverWithSystemNameservers(guardAgainstNoNameservers bool) ( if skip { return effective, system } - or = newResolverWithNameserver(effective) + storeOsResolver(newResolverWithNameserver(effective)) return effective, system } @@ -230,7 +361,7 @@ func NewResolver(uc *UpstreamConfig) (Resolver, error) { resolverMutex.Lock() if or == nil { ProxyLogger.Load().Debug().Msgf("Initialize new OS resolver") - or = newResolverWithNameserver(defaultNameservers()) + storeOsResolver(newResolverWithNameserver(defaultNameservers())) } resolverMutex.Unlock() return or, nil @@ -269,10 +400,10 @@ type publicResponse struct { // OsResolverNameservers returns the current OS resolver nameservers (host:port format). // Returns nil if the OS resolver has not been initialized. +// The read takes no lock, so a log header render never waits for a resolver +// initialization. func OsResolverNameservers() []string { - resolverMutex.Lock() - r := or - resolverMutex.Unlock() + r := osResolverPtr.Load() if r == nil { return nil } @@ -727,7 +858,7 @@ func initDefaultOsResolver() []string { defer resolverMutex.Unlock() if or == nil { ProxyLogger.Load().Debug().Msgf("Initialize new OS resolver with default nameservers") - or = newResolverWithNameserver(defaultNameservers()) + storeOsResolver(newResolverWithNameserver(defaultNameservers())) } nss := *or.lanServers.Load() nss = append(nss, *or.publicServers.Load()...) diff --git a/resolver_test.go b/resolver_test.go index 1840cb3b..674bef37 100644 --- a/resolver_test.go +++ b/resolver_test.go @@ -4,9 +4,13 @@ import ( "context" "crypto/rand" "encoding/hex" + "encoding/json" "errors" "net" "os" + "runtime" + "slices" + "strings" "sync" "sync/atomic" "syscall" @@ -14,6 +18,7 @@ import ( "time" "github.com/miekg/dns" + "github.com/rs/zerolog" ) func Test_osResolver_Resolve(t *testing.T) { @@ -859,3 +864,231 @@ func generateEdns0ServerCookie(clientCookie string) string { } return clientCookie + hex.EncodeToString(cookie) } + +// syncLogBuffer collects the log lines that any goroutine writes. +type syncLogBuffer struct { + mu sync.Mutex + sb strings.Builder +} + +func (b *syncLogBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + return b.sb.Write(p) +} + +func (b *syncLogBuffer) String() string { + b.mu.Lock() + defer b.mu.Unlock() + return b.sb.String() +} + +// captureProxyLog sends the proxy log to a buffer at debug level. It puts the +// logger and the global level back when the test ends. +func captureProxyLog(t *testing.T) *syncLogBuffer { + t.Helper() + buf := &syncLogBuffer{} + logger := zerolog.New(buf).Level(zerolog.DebugLevel) + previousLogger := ProxyLogger.Load() + previousLevel := zerolog.GlobalLevel() + ProxyLogger.Store(&logger) + zerolog.SetGlobalLevel(zerolog.DebugLevel) + t.Cleanup(func() { + ProxyLogger.Store(previousLogger) + zerolog.SetGlobalLevel(previousLevel) + }) + return buf +} + +// logEventsWithPrefix returns the log events whose message starts with prefix. +func logEventsWithPrefix(t *testing.T, logs, prefix string) []map[string]any { + t.Helper() + var events []map[string]any + for _, line := range strings.Split(logs, "\n") { + if strings.TrimSpace(line) == "" { + continue + } + event := map[string]any{} + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatalf("log line is not JSON: %q: %v", line, err) + } + message, _ := event["message"].(string) + if !strings.HasPrefix(message, prefix) { + continue + } + events = append(events, event) + } + return events +} + +// wantLogField fails the test when an event misses a field or holds another value. +func wantLogField(t *testing.T, event map[string]any, field string, want any) { + t.Helper() + got, ok := event[field] + if !ok { + t.Fatalf("log event has no field %q: %v", field, event) + } + if got != want { + t.Fatalf("log event field %q = %v, want %v", field, got, want) + } +} + +// wantLogStrings fails the test when a list field does not hold want. +func wantLogStrings(t *testing.T, event map[string]any, field string, want []string) { + t.Helper() + values, ok := event[field].([]any) + if !ok { + t.Fatalf("log event field %q is not a list: %v", field, event[field]) + } + got := make([]string, 0, len(values)) + for _, value := range values { + text, ok := value.(string) + if !ok { + t.Fatalf("log event field %q holds a value that is not a string: %v", field, value) + } + got = append(got, text) + } + if !slices.Equal(got, want) { + t.Fatalf("log event field %q = %v, want %v", field, got, want) + } +} + +// stubNameservers answers each system nameserver read from lists, in order. +// The last list answers every read after it. +func stubNameservers(t *testing.T, lists ...[]string) { + t.Helper() + previous := NameserversFn + reads := 0 + NameserversFn = func() []string { + list := lists[min(reads, len(lists)-1)] + reads++ + return slices.Clone(list) + } + t.Cleanup(func() { NameserversFn = previous }) +} + +// resetOsResolverLog clears the change-only state, so one test does not see +// the nameserver reads of another. +func resetOsResolverLog(t *testing.T) { + t.Helper() + reset := func() { + osResolverLog.mu.Lock() + defer osResolverLog.mu.Unlock() + osResolverLog.system = nameserverReads{} + osResolverLog.final = nameserverReads{} + osResolverLog.reason = osResolverReasonUnspecified + } + reset() + t.Cleanup(reset) +} + +// keepOsResolver puts back the OS resolver that the process had before the test. +func keepOsResolver(t *testing.T) { + t.Helper() + resolverMutex.Lock() + previous := or + resolverMutex.Unlock() + t.Cleanup(func() { + resolverMutex.Lock() + storeOsResolver(previous) + resolverMutex.Unlock() + }) +} + +// TestOsResolverNameserversReadsWithoutTheResolverLock covers a log header +// render during a resolver initialization. That initialization holds +// resolverMutex across a scutil read of several seconds, and the header must +// not wait for it. +func TestOsResolverNameserversReadsWithoutTheResolverLock(t *testing.T) { + keepOsResolver(t) + resolverMutex.Lock() + storeOsResolver(newResolverWithNameserver([]string{"192.0.2.1:53"})) + + read := make(chan []string, 1) + go func() { read <- OsResolverNameservers() }() + + select { + case nameservers := <-read: + resolverMutex.Unlock() + if want := []string{"192.0.2.1:53"}; !slices.Equal(nameservers, want) { + t.Fatalf("nameservers = %v, want %v", nameservers, want) + } + case <-time.After(5 * time.Second): + resolverMutex.Unlock() + t.Fatal("OsResolverNameservers waited for resolverMutex") + } +} + +func TestJournalMarksTheEventAndKeepsItsLevel(t *testing.T) { + logs := captureProxyLog(t) + + Journal(ProxyLogger.Load().Info()).Msg("Journal test event") + + events := logEventsWithPrefix(t, logs.String(), "Journal test event") + if len(events) != 1 { + t.Fatalf("got %d events, want 1: %s", len(events), logs.String()) + } + wantLogField(t, events[0], JournalField, true) + wantLogField(t, events[0], "level", "info") +} + +func TestOsResolverNameserverReadsLogOnChange(t *testing.T) { + logs := captureProxyLog(t) + resetOsResolverLog(t) + keepOsResolver(t) + first := []string{"192.0.2.1", "192.0.2.2"} + second := []string{"192.0.2.3"} + stubNameservers(t, first, first, second) + + for range 3 { + InitializeOsResolverWithReason(false, "transition") + } + + systemReads := logEventsWithPrefix(t, logs.String(), "Got system nameservers") + if len(systemReads) != 2 { + t.Fatalf("got %d system nameserver lines, want 2: %s", len(systemReads), logs.String()) + } + wantLogField(t, systemReads[0], "repeats", float64(0)) + wantLogField(t, systemReads[1], "repeats", float64(1)) + wantLogField(t, systemReads[1], "message", "Got system nameservers: [192.0.2.3]") + + finalReads := logEventsWithPrefix(t, logs.String(), "Final available nameservers") + if len(finalReads) != 2 { + t.Fatalf("got %d final nameserver lines, want 2: %s", len(finalReads), logs.String()) + } + wantLogField(t, finalReads[1], "repeats", float64(1)) + + changes := logEventsWithPrefix(t, logs.String(), "OS resolver set changed") + if len(changes) != 2 { + t.Fatalf("got %d resolver change events, want one for each changed list: %s", len(changes), logs.String()) + } + last := changes[1] + wantLogStrings(t, last, "before", first) + wantLogStrings(t, last, "after", second) + wantLogField(t, last, JournalField, true) + wantLogField(t, last, "level", "info") + wantLogField(t, last, "reason", "transition") + if _, ok := last["default_route"]; !ok { + t.Fatalf("resolver change event has no default_route field: %v", last) + } + wantSource := map[string]string{"darwin": "scutil", "windows": "dhcp", "linux": "resolv.conf"}[runtime.GOOS] + if wantSource != "" { + wantLogField(t, last, "source", wantSource) + } +} + +func TestInitializeOsResolverReportsTheUnspecifiedReason(t *testing.T) { + logs := captureProxyLog(t) + resetOsResolverLog(t) + keepOsResolver(t) + stubNameservers(t, []string{"192.0.2.10"}) + + InitializeOsResolver(false) + + changes := logEventsWithPrefix(t, logs.String(), "OS resolver set changed") + if len(changes) != 1 { + t.Fatalf("got %d resolver change events, want 1: %s", len(changes), logs.String()) + } + wantLogField(t, changes[0], "reason", "unspecified") + wantLogStrings(t, changes[0], "after", []string{"192.0.2.10"}) +} From f8239779929f6d58d4d4131785ae4b689281efde Mon Sep 17 00:00:00 2001 From: Dev Scribe Date: Fri, 18 Sep 2026 12:04:24 -0400 Subject: [PATCH 11/29] fix(net): close losing dials and retire replaced transports Co-authored-by: Ginder Singh --- config.go | 42 +++- config_quic.go | 65 ++++-- dot.go | 170 ++++++++------ internal/net/net.go | 28 ++- transport_replacement_test.go | 329 ++++++++++++++++++++++++++ transport_retirement_test.go | 420 ++++++++++++++++++++++++++++++++++ 6 files changed, 943 insertions(+), 111 deletions(-) create mode 100644 transport_replacement_test.go create mode 100644 transport_retirement_test.go diff --git a/config.go b/config.go index 90363259..befa7b59 100644 --- a/config.go +++ b/config.go @@ -534,11 +534,36 @@ func (uc *UpstreamConfig) ForceReBootstrap() { uc.rebootstrap.Store(rebootstrapNotStarted) } +// CloseTransports retires the upstream's transports, including active DoT and +// HTTP/3 connections. Callers must stop publishing this upstream before calling +// it and must not reuse it without calling SetupTransport. +func (uc *UpstreamConfig) CloseTransports() { + uc.retireTransports() +} + +// retireTransports permanently closes DoT pools and HTTP/3 transports before +// their owner discards them. Like closeTransports, it does not acquire an +// upstream lock: callers that serialize slot access must keep that serialization +// across retirement and replacement. +func (uc *UpstreamConfig) retireTransports() { + uc.closeTransports() + for _, p := range []*dotConnPool{uc.dotClientPool, uc.dotClientPool4, uc.dotClientPool6} { + if p != nil { + p.Close() + } + } + // Unlike CloseIdleConnections, Close also cancels pending dials and active + // HTTP/3 requests, and prevents late requests from reviving the transport. + for _, rt := range []http.RoundTripper{uc.http3RoundTripper, uc.http3RoundTripper4, uc.http3RoundTripper6} { + if c, ok := rt.(interface{ Close() error }); ok { + _ = c.Close() + } + } +} + // closeTransports closes idle connections on all existing transports. -// This is called before creating new transports during re-bootstrap to -// force in-flight requests on stale connections to fail quickly, rather -// than waiting for the full context deadline (e.g. 5s) after a firewall -// state table flush kills the underlying TCP/QUIC connections. +// It does not retire DoT pools or HTTP/3 transports: active work may continue +// and the resources remain usable. Use retireTransports before replacing them. func (uc *UpstreamConfig) closeTransports() { if t := uc.transport; t != nil { t.CloseIdleConnections() @@ -588,11 +613,10 @@ func (uc *UpstreamConfig) SetupTransport() { return } - // Close existing transport connections before creating new ones. - // This forces in-flight requests on stale connections (e.g. after a - // firewall state table flush) to fail fast instead of waiting for - // the full context deadline timeout. - uc.closeTransports() + // Retire old DoT/HTTP3 resources before replacing their slots. Idle-only + // cleanup would leave active queries and late work owning orphaned resources. + // DoH and DoQ retain their existing cleanup semantics. + uc.retireTransports() ips := uc.bootstrapIPs switch uc.IPStack { diff --git a/config_quic.go b/config_quic.go index 4c35fa8f..ab2778d0 100644 --- a/config_quic.go +++ b/config_quic.go @@ -31,9 +31,16 @@ func (uc *UpstreamConfig) newDOH3Transport(addrs []string) http.RoundTripper { } remoteAddr, err := net.ResolveUDPAddr("udp", addr) if err != nil { + udpConn.Close() return nil, err } - return quic.DialEarly(ctx, udpConn, remoteAddr, tlsCfg, cfg) + conn, err := quic.DialEarly(ctx, udpConn, remoteAddr, tlsCfg, cfg) + if err != nil { + udpConn.Close() + return nil, err + } + closeUDPConnWhenDone(conn, udpConn) + return conn, nil } dialAddrs := make([]string, len(addrs)) for i := range addrs { @@ -73,18 +80,25 @@ func (uc *UpstreamConfig) dotTransport(dnsType uint16) *dotConnPool { // - quic dialer is different with net.Dialer // - simplification for quic free version type parallelDialerResult struct { - conn *quic.Conn - err error + conn *quic.Conn + udpConn *net.UDPConn + err error +} + +// closeUDPConnWhenDone closes udpConn once conn terminates. quic.DialEarly does +// not take ownership of the socket it is handed, so the caller must. +func closeUDPConnWhenDone(conn *quic.Conn, udpConn *net.UDPConn) { + go func() { + <-conn.Context().Done() + _ = udpConn.Close() + }() } // quicParallelDialer races DialEarly across a list of remote addresses and -// returns the first successful connection. When transport is non-nil, all -// dials share that transport's UDP socket, which removes both the per-dial -// socket allocation and the winner-path socket leak that an owner-of-the-conn -// receiver cannot clean up. When transport is nil, the dialer falls back to a -// fresh UDP socket per attempt (compat path used where no shared transport is -// available yet); the loser paths close their sockets, and the winner path's -// socket is owned by quic.DialEarly's internal transport. +// returns the first successful connection. When transport is non-nil, all dials +// share that transport's UDP socket. When transport is nil, the dialer falls +// back to a fresh UDP socket per attempt; losing sockets are closed as their +// dials settle, and the winning one is closed with the connection it carries. type quicParallelDialer struct { transport *quic.Transport } @@ -97,8 +111,6 @@ func (d *quicParallelDialer) Dial(ctx context.Context, addrs []string, tlsCfg *t ctx, cancel := context.WithCancel(ctx) defer cancel() - done := make(chan struct{}) - defer close(done) ch := make(chan *parallelDialerResult, len(addrs)) var wg sync.WaitGroup wg.Add(len(addrs)) @@ -133,16 +145,7 @@ func (d *quicParallelDialer) Dial(ctx context.Context, addrs []string, tlsCfg *t udpConn = nil } } - select { - case ch <- ¶llelDialerResult{conn: conn, err: err}: - case <-done: - if conn != nil { - conn.CloseWithError(quic.ApplicationErrorCode(http3.ErrCodeNoError), "") - } - if udpConn != nil { - udpConn.Close() - } - } + ch <- ¶llelDialerResult{conn: conn, udpConn: udpConn, err: err} }(addr) } @@ -150,7 +153,11 @@ func (d *quicParallelDialer) Dial(ctx context.Context, addrs []string, tlsCfg *t for res := range ch { if res.err == nil { cancel() - return res.conn, res.err + if res.udpConn != nil { + closeUDPConnWhenDone(res.conn, res.udpConn) + } + go closeLosingQuicConns(ch) + return res.conn, nil } errs = append(errs, res.err) } @@ -158,6 +165,18 @@ func (d *quicParallelDialer) Dial(ctx context.Context, addrs []string, tlsCfg *t return nil, errors.Join(errs...) } +// closeLosingQuicConns drains ch and releases everything left in it. +func closeLosingQuicConns(ch <-chan *parallelDialerResult) { + for res := range ch { + if res.conn != nil { + res.conn.CloseWithError(quic.ApplicationErrorCode(http3.ErrCodeNoError), "") + } + if res.udpConn != nil { + _ = res.udpConn.Close() + } + } +} + func (uc *UpstreamConfig) newDOQConnPool(addrs []string) *doqConnPool { if uc.Type != ResolverTypeDOQ { return nil diff --git a/dot.go b/dot.go index ff0aac27..ee1dce60 100644 --- a/dot.go +++ b/dot.go @@ -7,6 +7,7 @@ import ( "io" "net" "runtime" + "sync" "time" "github.com/miekg/dns" @@ -36,7 +37,8 @@ func (r *dotResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, erro const dotPoolSize = 16 -// dotConnPool manages a pool of TCP/TLS connections for DoT queries using a buffered channel. +// dotConnPool manages a pool of TCP/TLS connections for DoT queries. +// mu serializes retirement, ownership and idle queue operations. type dotConnPool struct { uc *UpstreamConfig addrs []string @@ -44,6 +46,12 @@ type dotConnPool struct { tlsConfig *tls.Config dialer *net.Dialer conns chan *dotConn + + mu sync.Mutex + closed bool + owned map[*tls.Conn]struct{} // handshaking, checked-out and idle connections + ctx context.Context + cancel context.CancelFunc } type dotConn struct { @@ -72,6 +80,7 @@ func newDOTClientPool(uc *UpstreamConfig, addrs []string) *dotConnPool { tlsConfig.ServerName = uc.Domain } + ctx, cancel := context.WithCancel(context.Background()) pool := &dotConnPool{ uc: uc, addrs: addrs, @@ -79,13 +88,16 @@ func newDOTClientPool(uc *UpstreamConfig, addrs []string) *dotConnPool { tlsConfig: tlsConfig, dialer: dialer, conns: make(chan *dotConn, dotPoolSize), + owned: make(map[*tls.Conn]struct{}), + ctx: ctx, + cancel: cancel, } // Use SetFinalizer here because we need to call a method on the pool itself. // AddCleanup would require passing the pool as arg (which panics) or capturing // it in a closure (which prevents GC). SetFinalizer is appropriate for this case. runtime.SetFinalizer(pool, func(p *dotConnPool) { - p.CloseIdleConnections() + p.Close() }) return pool @@ -115,18 +127,28 @@ func (p *dotConnPool) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, erro } // getConn gets a TCP/TLS connection from the pool or creates a new one. -// A connection is taken from the channel while in use; putConn returns it. +// Checked-out connections remain owned until putConn discards them or Close runs. func (p *dotConnPool) getConn(ctx context.Context) (net.Conn, error) { for { + if err := ctx.Err(); err != nil { + return nil, err + } + p.mu.Lock() + if p.closed { + p.mu.Unlock() + return nil, net.ErrClosed + } select { case dc := <-p.conns: + p.mu.Unlock() if dc.conn != nil && isAlive(dc.conn) { return dc.conn, nil } if dc.conn != nil { - dc.conn.Close() + p.putConn(dc.conn, false) } default: + p.mu.Unlock() _, conn, err := p.dialConn(ctx) if err != nil { return nil, err @@ -136,87 +158,66 @@ func (p *dotConnPool) getConn(ctx context.Context) (net.Conn, error) { } } -// putConn returns a connection to the pool for reuse by other goroutines. +// putConn cannot repopulate a retired pool, even for a successful late response. func (p *dotConnPool) putConn(conn net.Conn, isGood bool) { - if !isGood || conn == nil { - if conn != nil { - conn.Close() - } + if conn == nil { return } dc := &dotConn{conn: conn.(*tls.Conn)} - select { - case p.conns <- dc: - default: - // Channel full, close the connection - dc.conn.Close() + p.mu.Lock() + if isGood && !p.closed { + select { + case p.conns <- dc: + p.mu.Unlock() + return + default: + } } + delete(p.owned, dc.conn) + p.mu.Unlock() + _ = dc.conn.Close() } -// dialConn creates a new TCP/TLS connection. +// dialConn creates a new TCP/TLS connection. Retirement cancels pending TCP +// dials and handshakes; finishing a parallel race cancels and closes its losers. func (p *dotConnPool) dialConn(ctx context.Context) (string, *tls.Conn, error) { + ctx, cancel := context.WithCancel(ctx) + stop := context.AfterFunc(p.ctx, cancel) + defer stop() + defer cancel() + if p.ctx.Err() != nil { + return "", nil, net.ErrClosed + } logger := ProxyLogger.Load() - var endpoint string - if p.uc.BootstrapIP != "" { - endpoint = net.JoinHostPort(p.uc.BootstrapIP, p.port) + endpoint := net.JoinHostPort(p.uc.BootstrapIP, p.port) Log(ctx, logger.Debug(), "Sending DoT request to: %s", endpoint) - conn, err := p.dialer.DialContext(ctx, "tcp", endpoint) - if err != nil { - return "", nil, err - } - tlsConn := tls.Client(conn, p.tlsConfig) - if err := tlsConn.HandshakeContext(ctx); err != nil { - conn.Close() - return "", nil, err - } - return endpoint, tlsConn, nil + conn, err := p.dialTLSConn(ctx, endpoint, p.tlsConfig) + return endpoint, conn, err } - - // Try bootstrap IPs in parallel if len(p.addrs) > 0 { type result struct { conn *tls.Conn addr string err error } - - ch := make(chan result, len(p.addrs)) - done := make(chan struct{}) - defer close(done) - + // An unbuffered handoff leaves every losing connection with its dialer. + ch := make(chan result) for _, addr := range p.addrs { go func(addr string) { endpoint := net.JoinHostPort(addr, p.port) - conn, err := p.dialer.DialContext(ctx, "tcp", endpoint) - if err != nil { - select { - case ch <- result{conn: nil, addr: endpoint, err: err}: - case <-done: - } - return - } tlsConfig := p.tlsConfig.Clone() tlsConfig.ServerName = p.uc.Domain - tlsConn := tls.Client(conn, tlsConfig) - if err := tlsConn.HandshakeContext(ctx); err != nil { - conn.Close() - select { - case ch <- result{conn: nil, addr: endpoint, err: err}: - case <-done: - } - return - } + conn, err := p.dialTLSConn(ctx, endpoint, tlsConfig) select { - case ch <- result{conn: tlsConn, addr: endpoint, err: nil}: - case <-done: + case ch <- result{conn: conn, addr: endpoint, err: err}: + case <-ctx.Done(): if conn != nil { - conn.Close() + p.putConn(conn, false) } } }(addr) } - errs := make([]error, 0, len(p.addrs)) for range len(p.addrs) { select { @@ -232,32 +233,47 @@ func (p *dotConnPool) dialConn(ctx context.Context) (string, *tls.Conn, error) { return "", nil, ctx.Err() } } - return "", nil, errors.Join(errs...) } - - // Fallback to endpoint resolution - endpoint = p.uc.Endpoint + endpoint := p.uc.Endpoint Log(ctx, logger.Debug(), "Sending DoT request to: %s", endpoint) + conn, err := p.dialTLSConn(ctx, endpoint, p.tlsConfig) + return endpoint, conn, err +} + +// Register ownership before the TLS handshake so Close interrupts stalled peers. +// A TCP dial racing retirement cannot publish a socket into the closed pool. +func (p *dotConnPool) dialTLSConn(ctx context.Context, endpoint string, cfg *tls.Config) (*tls.Conn, error) { conn, err := p.dialer.DialContext(ctx, "tcp", endpoint) if err != nil { - return "", nil, err + return nil, err } - tlsConn := tls.Client(conn, p.tlsConfig) + tlsConn := tls.Client(conn, cfg) + p.mu.Lock() + if p.closed { + p.mu.Unlock() + _ = conn.Close() + return nil, net.ErrClosed + } + p.owned[tlsConn] = struct{}{} + p.mu.Unlock() if err := tlsConn.HandshakeContext(ctx); err != nil { - conn.Close() - return "", nil, err + p.putConn(tlsConn, false) + return nil, err } - return endpoint, tlsConn, nil + return tlsConn, nil } -// CloseIdleConnections closes all connections in the pool. -// Connections currently checked out (in use) are not closed. +// CloseIdleConnections closes only queued connections. The pool remains usable +// and outstanding queries may return their connections. Replacement uses Close. func (p *dotConnPool) CloseIdleConnections() { + p.mu.Lock() + defer p.mu.Unlock() for { select { case dc := <-p.conns: if dc.conn != nil { + delete(p.owned, dc.conn) dc.conn.Close() } default: @@ -266,6 +282,26 @@ func (p *dotConnPool) CloseIdleConnections() { } } +// Close permanently retires this pool. Closing the underlying TCP sockets also +// interrupts TLS reads/writes without waiting to send a close_notify. +// Keep the channel open: outstanding borrowers may still call putConn. +func (p *dotConnPool) Close() { + p.mu.Lock() + defer p.mu.Unlock() + if p.closed { + return + } + p.closed = true + p.cancel() + for conn := range p.owned { + _ = conn.NetConn().Close() + delete(p.owned, conn) + } + for len(p.conns) > 0 { + <-p.conns + } +} + func isAlive(c *tls.Conn) bool { // Set a very short deadline for the read c.SetReadDeadline(time.Now().Add(1 * time.Millisecond)) diff --git a/internal/net/net.go b/internal/net/net.go index 677e90dc..cb703212 100644 --- a/internal/net/net.go +++ b/internal/net/net.go @@ -295,8 +295,6 @@ func (d *ParallelDialer) DialContext(ctx context.Context, network string, addrs ctx, cancel := context.WithCancel(ctx) defer cancel() - done := make(chan struct{}) - defer close(done) ch := make(chan *parallelDialerResult, len(live)) var wg sync.WaitGroup wg.Add(len(live)) @@ -318,25 +316,31 @@ func (d *ParallelDialer) DialContext(ctx context.Context, network string, addrs } else { unreachable.markReachable(addr) } - select { - case ch <- ¶llelDialerResult{conn: conn, err: err}: - case <-done: - if conn != nil { - logger.Debug().Msgf("connection closed: %s", conn.RemoteAddr()) - conn.Close() - } - } + ch <- ¶llelDialerResult{conn: conn, err: err} }(addr) } - errs := make([]error, 0, len(addrs)) + errs := make([]error, 0, len(live)) for res := range ch { if res.err == nil { cancel() logger.Debug().Msgf("connected to %s", res.conn.RemoteAddr()) - return res.conn, res.err + // Losing dials may already have connected, or may still connect + // after cancel; close whatever they hand back. + go closeLosingConns(ch, logger) + return res.conn, nil } errs = append(errs, res.err) } return nil, errors.Join(errs...) } + +// closeLosingConns drains ch and closes every connection left in it. +func closeLosingConns(ch <-chan *parallelDialerResult, logger *zerolog.Logger) { + for res := range ch { + if res.conn != nil { + logger.Debug().Msgf("connection closed: %s", res.conn.RemoteAddr()) + _ = res.conn.Close() + } + } +} diff --git a/transport_replacement_test.go b/transport_replacement_test.go new file mode 100644 index 00000000..f93cd0fe --- /dev/null +++ b/transport_replacement_test.go @@ -0,0 +1,329 @@ +package ctrld + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "errors" + "io" + "net" + "net/http" + "net/url" + "sync" + "syscall" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/quic-go/quic-go" + "github.com/quic-go/quic-go/http3" +) + +var transportReplacements = []struct { + name string + run func(*UpstreamConfig) +}{ + {"SetupTransport", (*UpstreamConfig).SetupTransport}, + {"ForceReBootstrap", (*UpstreamConfig).ForceReBootstrap}, +} + +// Accept multiple connections so a replacement must answer on the same real +// endpoint, while the original query is still held by the server. +func replacementDOTServer(t *testing.T) (*UpstreamConfig, <-chan struct{}, func(), <-chan struct{}) { + t.Helper() + cert := generateTestCertificate(t) + ln, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert.tlsCert}}) + if err != nil { + t.Fatal(err) + } + entered, release, peerClosed, done := make(chan struct{}), make(chan struct{}), make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + var mu sync.Mutex + var conns []net.Conn + var wg sync.WaitGroup + go func() { + defer close(done) + for { + c, err := ln.Accept() + if err != nil { + return + } + mu.Lock() + conns = append(conns, c) + mu.Unlock() + wg.Add(1) + go func() { + defer wg.Done() + defer c.Close() + dc := &dns.Conn{Conn: c} + for { + q, err := dc.ReadMsg() + if err != nil { + return + } + held := q.Question[0].Name == "held.example." + if held { + close(entered) + defer close(peerClosed) + <-release + } + a := new(dns.Msg) + a.SetReply(q) + if err := dc.WriteMsg(a); err != nil { + return + } + } + }() + } + }() + roots := x509.NewCertPool() + roots.AddCert(cert.cert) + uc := &UpstreamConfig{Type: ResolverTypeDOT, Endpoint: ln.Addr().String(), BootstrapIP: "127.0.0.1", Domain: "localhost", certPool: roots} + uc.ensureSetupTransport() + t.Cleanup(func() { + unblock() + uc.CloseTransports() + _ = ln.Close() + retirementWait(t, done, "DoT accept loop cleanup") // No more wg.Add after this. + mu.Lock() + for _, c := range conns { + _ = c.Close() + } + mu.Unlock() + wg.Wait() + }) + return uc, entered, unblock, peerClosed +} + +func replacementAnswer(t *testing.T, resolve func(context.Context, *dns.Msg) (*dns.Msg, error)) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + q := retirementQuery() + a, err := resolve(ctx, q) + if err != nil { + t.Fatalf("replacement query failed: %v", err) + } + if a == nil || !a.Response || a.Id != q.Id || len(a.Question) != 1 || a.Question[0] != q.Question[0] { + t.Fatalf("replacement returned wrong DNS answer: %v", a) + } +} + +func replacementResult(t *testing.T, result <-chan error) error { + t.Helper() + select { + case err := <-result: + return err + case <-time.After(3 * time.Second): + t.Fatal("old query survived replacement") + return nil + } +} + +func TestReplaceTransportsDoTActive(t *testing.T) { + for _, replace := range transportReplacements { + t.Run(replace.name, func(t *testing.T) { + uc, entered, release, peerClosed := replacementDOTServer(t) + old := uc.dotClientPool + selected := make(chan struct{}) + old.dialer.ControlContext = func(context.Context, string, string, syscall.RawConn) error { + close(selected) + return nil + } + r := &dotResolver{uc: uc} + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + q := new(dns.Msg) + q.SetQuestion("held.example.", dns.TypeA) + result := make(chan error, 1) + go func() { _, err := r.Resolve(ctx, q); result <- err }() + retirementWait(t, selected, "DoT transport selection") + retirementWait(t, entered, "active DoT query") + replace.run(uc) + if uc.dotClientPool == old { + t.Fatal("DoT pool was not replaced") + } + if err := replacementResult(t, result); !errors.Is(err, net.ErrClosed) { + t.Fatalf("old query did not fail from socket retirement: %v", err) + } + // This answer must arrive before releasing the old server handler. + replacementAnswer(t, r.Resolve) + release() + retirementWait(t, peerClosed, "old DoT peer closure") + if _, err := old.Resolve(ctx, retirementQuery()); !errors.Is(err, net.ErrClosed) { + t.Fatalf("late query revived replaced DoT pool: %v", err) + } + replacementAnswer(t, r.Resolve) + }) + } +} + +func TestReplaceTransportsDoTLateReturn(t *testing.T) { + for _, replace := range transportReplacements { + for _, slot := range []string{"default", "v4", "v6", "aliased"} { + t.Run(replace.name+"/"+slot, func(t *testing.T) { + uc, _, _, _ := replacementDOTServer(t) + old := uc.dotClientPool + r := &dotResolver{uc: uc} + replacementAnswer(t, r.Resolve) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + conn, err := old.getConn(ctx) + if err != nil { + t.Fatal(err) + } + // Model each old slot, including the IPv6-to-IPv4 alias, without + // requiring an external IPv6 capability probe in this fixture. + switch slot { + case "v4": + uc.dotClientPool, uc.dotClientPool4 = nil, old + case "v6": + uc.dotClientPool, uc.dotClientPool6 = nil, old + case "aliased": + uc.dotClientPool4, uc.dotClientPool6 = old, old + } + replace.run(uc) + // A successful borrower returning after the slots change must not + // put its connection back into the now-orphaned pool. + old.putConn(conn, true) + if len(old.conns) != 0 { + t.Fatal("late return repopulated replaced DoT pool") + } + if _, err := old.getConn(ctx); !errors.Is(err, net.ErrClosed) { + t.Fatalf("replaced DoT pool accepted late work: %v", err) + } + replacementAnswer(t, r.Resolve) + }) + } + } +} + +func TestReplaceTransportsDoH3IdleAndLateWork(t *testing.T) { + for _, replace := range transportReplacements { + t.Run(replace.name, func(t *testing.T) { + srv := newTestHTTP3Server(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + roots := x509.NewCertPool() + roots.AddCert(srv.cert) + uc := &UpstreamConfig{Type: ResolverTypeDOH3, Endpoint: "https://" + srv.addr, BootstrapIP: "127.0.0.1", certPool: roots} + defer uc.CloseTransports() + // The real warmup path initializes the transport and leaves it idle. + if err := uc.ErrorPing(); err != nil { + t.Fatal(err) + } + old := uc.http3RoundTripper + defer old.(io.Closer).Close() + replace.run(uc) + ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second) + defer cancel() + req, _ := http.NewRequestWithContext(ctx, "HEAD", uc.Endpoint, nil) + resp, err := old.RoundTrip(req) + if resp != nil { + resp.Body.Close() + } + if err == nil || ctx.Err() != nil { + t.Fatalf("late work revived idle replaced HTTP/3 transport: %v (context: %v)", err, ctx.Err()) + } + if err := uc.ErrorPing(); err != nil { + t.Fatalf("replacement HTTP/3 transport is not usable: %v", err) + } + }) + } +} + +func TestReplaceTransportsDoH3ActiveAndLateWork(t *testing.T) { + for _, replace := range transportReplacements { + for _, slot := range []string{"default", "v4", "v6", "aliased"} { + t.Run(replace.name+"/"+slot, func(t *testing.T) { + entered, release, peerCanceled := make(chan struct{}), make(chan struct{}), make(chan struct{}) + srv := newTestHTTP3Server(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/late" { + w.WriteHeader(http.StatusOK) + return + } + data, err := base64.RawURLEncoding.DecodeString(r.URL.Query().Get("dns")) + q := new(dns.Msg) + if err != nil || q.Unpack(data) != nil || len(q.Question) != 1 { + http.Error(w, "bad DNS query", http.StatusBadRequest) + return + } + if q.Question[0].Name == "held.example." { + close(entered) + select { + case <-r.Context().Done(): + close(peerCanceled) + case <-release: + } + return + } + a := new(dns.Msg) + a.SetReply(q) + data, _ = a.Pack() + w.Header().Set("Content-Type", headerApplicationDNS) + _, _ = w.Write(data) + })) + defer close(release) + roots := x509.NewCertPool() + roots.AddCert(srv.cert) + u, err := url.Parse("https://" + srv.addr + "/dns-query") + if err != nil { + t.Fatal(err) + } + uc := &UpstreamConfig{Type: ResolverTypeDOH3, Endpoint: u.String(), u: u, BootstrapIP: "127.0.0.1", certPool: roots} + uc.ensureSetupTransport() + defer uc.CloseTransports() + old := uc.http3RoundTripper + // Clean the old transport even if a regression leaves it orphaned. + defer old.(io.Closer).Close() + selected := make(chan struct{}) + rt := old.(*http3.Transport) + dial := rt.Dial + var selectedOnce sync.Once + rt.Dial = func(ctx context.Context, addr string, tlsCfg *tls.Config, cfg *quic.Config) (*quic.Conn, error) { + selectedOnce.Do(func() { close(selected) }) + return dial(ctx, addr, tlsCfg, cfg) + } + r := newDohResolver(uc) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + q := new(dns.Msg) + q.SetQuestion("held.example.", dns.TypeA) + result := make(chan error, 1) + go func() { _, err := r.Resolve(ctx, q); result <- err }() + // A network exchange alone isn't a Go happens-before edge. Observe + // selection before writing slots; still execute the production dial. + retirementWait(t, selected, "HTTP/3 transport selection") + retirementWait(t, entered, "active HTTP/3 DNS query") + switch slot { + case "v4": + uc.http3RoundTripper, uc.http3RoundTripper4 = nil, old + case "v6": + uc.http3RoundTripper, uc.http3RoundTripper6 = nil, old + case "aliased": + uc.http3RoundTripper4, uc.http3RoundTripper6 = old, old + } + replace.run(uc) + if uc.http3RoundTripper == old { + t.Fatal("HTTP/3 transport was not replaced") + } + retirementWait(t, peerCanceled, "old HTTP/3 request cancellation") + if err := replacementResult(t, result); err == nil || ctx.Err() != nil { + t.Fatalf("old HTTP/3 query not aborted by replacement: %v (context: %v)", err, ctx.Err()) + } + replacementAnswer(t, r.Resolve) + req, _ := http.NewRequestWithContext(ctx, "GET", "https://"+srv.addr+"/late", nil) + resp, err := old.RoundTrip(req) + if resp != nil { + resp.Body.Close() + } + if err == nil || ctx.Err() != nil { + t.Fatalf("late request revived replaced HTTP/3 transport: %v (context: %v)", err, ctx.Err()) + } + replacementAnswer(t, r.Resolve) + }) + } + } +} diff --git a/transport_retirement_test.go b/transport_retirement_test.go new file mode 100644 index 00000000..a26ea124 --- /dev/null +++ b/transport_retirement_test.go @@ -0,0 +1,420 @@ +package ctrld + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "io" + "net" + "net/http" + "sync" + "syscall" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/quic-go/quic-go/http3" +) + +func retirementWait(t *testing.T, ch <-chan struct{}, what string) { + t.Helper() + select { + case <-ch: + case <-time.After(3 * time.Second): + t.Fatalf("timed out waiting for %s", what) + } +} + +// One real TLS connection, with the response optionally held until release. +// Cleanup joins the server goroutine and never touches host DNS or global state. +func retirementDOTServer(t *testing.T, hold bool) (*UpstreamConfig, <-chan struct{}, func(), <-chan struct{}) { + t.Helper() + cert := generateTestCertificate(t) + ln, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert.tlsCert}}) + if err != nil { + t.Fatal(err) + } + entered, release, peerClosed, done := make(chan struct{}), make(chan struct{}), make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + if !hold { + unblock() + } + var mu sync.Mutex + var accepted net.Conn + go func() { + defer close(done) + c, err := ln.Accept() + if err != nil { + return + } + mu.Lock() + accepted = c + mu.Unlock() + defer c.Close() + dc := &dns.Conn{Conn: c} + q, err := dc.ReadMsg() + if err != nil { + return + } + close(entered) + <-release + a := new(dns.Msg) + a.SetReply(q) + _ = dc.WriteMsg(a) + // No more queries are sent on this fixture; EOF proves peer retirement. + _, _ = dc.ReadMsg() + close(peerClosed) + }() + roots := x509.NewCertPool() + roots.AddCert(cert.cert) + uc := &UpstreamConfig{Type: ResolverTypeDOT, Endpoint: ln.Addr().String(), BootstrapIP: "127.0.0.1", Domain: "localhost", certPool: roots} + uc.SetupTransport() + uc.transportOnce.Do(func() {}) // SetupTransport above is the resolver's initial setup. + t.Cleanup(func() { + unblock() + uc.CloseTransports() + _ = ln.Close() + mu.Lock() + if accepted != nil { + _ = accepted.Close() + } + mu.Unlock() + retirementWait(t, done, "DoT server cleanup") + }) + return uc, entered, unblock, peerClosed +} + +func retirementQuery() *dns.Msg { + q := new(dns.Msg) + q.SetQuestion("retirement.example.", dns.TypeA) + return q +} + +func TestCloseTransportsDoTActive(t *testing.T) { + uc, entered, release, peerClosed := retirementDOTServer(t, true) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + result := make(chan error, 1) + go func() { _, err := (&dotResolver{uc: uc}).Resolve(ctx, retirementQuery()); result <- err }() + retirementWait(t, entered, "active DoT query") + uc.CloseTransports() + select { + case err := <-result: + if !errors.Is(err, net.ErrClosed) { + // miekg/dns has its own two-second read timeout, independent of + // the longer context deadline. A timeout is not retirement proof. + t.Fatalf("active DoT query was not closed by retirement: %v", err) + } + case <-time.After(3 * time.Second): + t.Fatal("active DoT query survived retirement until its own deadline") + } + release() + retirementWait(t, peerClosed, "DoT peer socket closure") + if _, err := (&dotResolver{uc: uc}).Resolve(ctx, retirementQuery()); !errors.Is(err, net.ErrClosed) { + t.Fatalf("retired resolver accepted another query: %v", err) + } + if n := len(uc.dotClientPool.conns); n != 0 { + t.Fatalf("late return repopulated pool: %d", n) + } +} + +func TestCloseTransportsDoTIdleAndLateReturn(t *testing.T) { + for _, slot := range []string{"default", "v4", "v6", "aliased"} { + for _, checkedOut := range []bool{false, true} { + t.Run(slot+map[bool]string{false: "/idle", true: "/checked-out"}[checkedOut], func(t *testing.T) { + uc, _, _, peerClosed := retirementDOTServer(t, false) + p := uc.dotClientPool + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if _, err := p.Resolve(ctx, retirementQuery()); err != nil { + t.Fatal(err) + } + var conn net.Conn + if checkedOut { + var err error + conn, err = p.getConn(ctx) + if err != nil { + t.Fatal(err) + } + } + switch slot { + case "v4": + uc.dotClientPool = nil + uc.dotClientPool4 = p + case "v6": + uc.dotClientPool = nil + uc.dotClientPool6 = p + case "aliased": + uc.dotClientPool4 = p + uc.dotClientPool6 = p + } + uc.CloseTransports() + retirementWait(t, peerClosed, "idle/checked-out DoT socket closure") + // Simulate a response that completed just before retirement and returns late. + if conn != nil { + p.putConn(conn, true) + } + uc.CloseTransports() // aliased slots and repeated retirement are safe. + if n := len(p.conns); n != 0 { + t.Fatalf("retired pool has %d connections", n) + } + if _, err := p.getConn(ctx); !errors.Is(err, net.ErrClosed) { + t.Fatalf("get after close: %v", err) + } + }) + } + } +} + +func TestCloseTransportsDoTHandshake(t *testing.T) { + for _, mode := range []string{"bootstrap", "parallel", "fallback"} { + t.Run(mode, func(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + entered, peerClosed := make(chan struct{}), make(chan struct{}) + go func() { + c, err := ln.Accept() + if err != nil { + return + } + defer c.Close() + var b [1]byte + if _, err := c.Read(b[:]); err != nil { + return + } + close(entered) // ClientHello was received, but no TLS response is sent. + _, _ = io.Copy(io.Discard, c) + close(peerClosed) + }() + uc := &UpstreamConfig{Type: ResolverTypeDOT, Endpoint: ln.Addr().String(), BootstrapIP: "127.0.0.1", Domain: "localhost"} + var addrs []string + if mode != "bootstrap" { + uc.BootstrapIP = "" + } + if mode == "parallel" { + addrs = []string{"127.0.0.1"} + } + p := newDOTClientPool(uc, addrs) + // The raw fallback fixture has no bootstrap hostname to infer SNI from. + p.tlsConfig.ServerName = "localhost" + uc.dotClientPool = p + defer uc.CloseTransports() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + result := make(chan error, 1) + go func() { _, err := p.Resolve(ctx, retirementQuery()); result <- err }() + retirementWait(t, entered, "stalled TLS handshake") + uc.CloseTransports() + retirementWait(t, peerClosed, "handshaking socket closure") + select { + case err := <-result: + if err == nil { + t.Fatal("handshake succeeded after retirement") + } + case <-time.After(3 * time.Second): + t.Fatal("handshake was not canceled") + } + }) + } +} + +func TestCloseTransportsDoTPendingDial(t *testing.T) { + // Per-pool dialer seam holds a real TCP dial before connect; no global hooks. + uc := &UpstreamConfig{Type: ResolverTypeDOT, Endpoint: "127.0.0.1:853", BootstrapIP: "127.0.0.1"} + p := newDOTClientPool(uc, nil) + uc.dotClientPool = p + defer uc.CloseTransports() + entered, canceled := make(chan struct{}), make(chan struct{}) + p.dialer.ControlContext = func(ctx context.Context, _, _ string, _ syscall.RawConn) error { + close(entered) + <-ctx.Done() + close(canceled) + return ctx.Err() + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + result := make(chan error, 1) + go func() { _, err := p.Resolve(ctx, retirementQuery()); result <- err }() + retirementWait(t, entered, "pending TCP dial") + uc.CloseTransports() + retirementWait(t, canceled, "TCP dial cancellation") + select { + case err := <-result: + if err == nil { + t.Fatal("dial succeeded after close") + } + case <-time.After(3 * time.Second): + t.Fatal("dial did not return") + } +} + +func TestDoTParallelDialClosesLosers(t *testing.T) { + const attempts = 4 + cert := generateTestCertificate(t) + ln, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{cert.tlsCert}}) + if err != nil { + t.Fatal(err) + } + defer ln.Close() + allAccepted, release := make(chan struct{}), make(chan struct{}) + closed, received := make(chan struct{}, attempts), make(chan struct{}, attempts) + var unblock sync.Once + releaseAll := func() { unblock.Do(func() { close(release) }) } + defer releaseAll() + var wg sync.WaitGroup + go func() { + for i := 0; i < attempts; i++ { + c, err := ln.Accept() + if err != nil { + return + } + wg.Add(1) + go func() { + defer wg.Done() + defer c.Close() + defer func() { closed <- struct{}{} }() + _ = c.SetDeadline(time.Now().Add(5 * time.Second)) + <-release + var b [1]byte + if _, err := c.Read(b[:]); err == nil { + received <- struct{}{} + _, _ = io.Copy(io.Discard, c) + } + }() + } + close(allAccepted) + }() + roots := x509.NewCertPool() + roots.AddCert(cert.cert) + uc := &UpstreamConfig{Type: ResolverTypeDOT, Endpoint: ln.Addr().String(), Domain: "localhost", certPool: roots} + addrs := make([]string, attempts) + for i := range addrs { + addrs[i] = "127.0.0.1" + } + p := newDOTClientPool(uc, addrs) + uc.dotClientPool = p + defer uc.CloseTransports() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + type result struct { + conn net.Conn + err error + } + resultCh := make(chan result, 1) + go func() { conn, err := p.getConn(ctx); resultCh <- result{conn, err} }() + retirementWait(t, allAccepted, "all parallel TCP connections") + releaseAll() + var conn net.Conn + select { + case res := <-resultCh: + if res.err != nil { + t.Fatal(res.err) + } + conn = res.conn + case <-time.After(3 * time.Second): + t.Fatal("parallel dial failed to return a winner") + } + // A dial-context cancellation must close losers, not the winning connection. + if _, err := conn.Write([]byte{42}); err != nil { + t.Fatal(err) + } + retirementWait(t, received, "winner data") + for i := 0; i < attempts-1; i++ { + retirementWait(t, closed, "losing parallel socket closure") + } + uc.CloseTransports() + retirementWait(t, closed, "winning socket retirement") + p.putConn(conn, true) + wg.Wait() +} + +func TestDoTIdleCleanupPreservesActive(t *testing.T) { + uc, entered, release, _ := retirementDOTServer(t, true) + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + result := make(chan error, 1) + go func() { _, err := uc.dotClientPool.Resolve(ctx, retirementQuery()); result <- err }() + retirementWait(t, entered, "active query") + uc.closeTransports() // Idle-only cleanup, not transport replacement. + release() + select { + case err := <-result: + if err != nil { + t.Fatalf("idle cleanup aborted active query: %v", err) + } + case <-time.After(3 * time.Second): + t.Fatal("query did not finish") + } + conn, err := uc.dotClientPool.getConn(ctx) + if err != nil { + t.Fatalf("idle cleanup retired pool: %v", err) + } + uc.dotClientPool.putConn(conn, true) +} + +func TestCloseTransportsDoH3Active(t *testing.T) { + for _, slot := range []string{"default", "v4", "v6", "aliased"} { + t.Run(slot, func(t *testing.T) { + entered, release, peerCanceled := make(chan struct{}), make(chan struct{}), make(chan struct{}) + srv := newTestHTTP3Server(t, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + close(entered) + select { + case <-r.Context().Done(): + close(peerCanceled) + case <-release: + } + })) + defer close(release) + roots := x509.NewCertPool() + roots.AddCert(srv.cert) + uc := &UpstreamConfig{Type: ResolverTypeDOH3, BootstrapIP: "127.0.0.1", certPool: roots} + rt := uc.newDOH3Transport(nil).(*http3.Transport) + defer rt.Close() + switch slot { + case "default": + uc.http3RoundTripper = rt + case "v4": + uc.http3RoundTripper4 = rt + case "v6": + uc.http3RoundTripper6 = rt + case "aliased": + uc.http3RoundTripper = rt + uc.http3RoundTripper4 = rt + uc.http3RoundTripper6 = rt + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + request := func() error { + req, _ := http.NewRequestWithContext(ctx, "GET", "https://"+srv.addr, nil) + resp, err := rt.RoundTrip(req) + if err == nil { + defer resp.Body.Close() + _, err = io.ReadAll(resp.Body) + } + return err + } + result := make(chan error, 1) + go func() { result <- request() }() + retirementWait(t, entered, "active HTTP/3 request") + uc.CloseTransports() + retirementWait(t, peerCanceled, "HTTP/3 server request cancellation") + select { + case err := <-result: + if err == nil { + t.Fatal("active HTTP/3 request survived close") + } + case <-time.After(3 * time.Second): + t.Fatal("HTTP/3 request was not aborted") + } + if err := request(); err == nil { + t.Fatal("retired HTTP/3 transport accepted another request") + } + uc.CloseTransports() + }) + } +} From 3127212eca63b830c95f5289c93285fb66f44114 Mon Sep 17 00:00:00 2001 From: Dev Scribe Date: Fri, 18 Sep 2026 12:04:24 -0400 Subject: [PATCH 12/29] fix(cli): release runtime resources across startup and shutdown Co-authored-by: Ginder Singh --- cmd/cli/api_reload_shutdown_test.go | 157 ++++++++++ cmd/cli/cli.go | 31 +- cmd/cli/control_server.go | 1 - cmd/cli/dns_intercept_darwin.go | 23 +- cmd/cli/dns_intercept_windows.go | 4 + cmd/cli/dns_proxy.go | 101 ++++++- cmd/cli/force_reload_lifetime_test.go | 48 +++ cmd/cli/internal_log_shutdown_test.go | 50 ++++ cmd/cli/listener_shutdown_test.go | 293 ++++++++++++++++++ cmd/cli/network_monitor_test.go | 2 + cmd/cli/network_shutdown_test.go | 322 ++++++++++++++++++++ cmd/cli/network_transition.go | 1 + cmd/cli/prog.go | 412 ++++++++++++++++++++++---- cmd/cli/recovery_state.go | 8 +- cmd/cli/reload_others.go | 4 + cmd/cli/reload_windows.go | 2 + cmd/cli/restore_lock_test.go | 51 ++++ cmd/cli/shutdown_test.go | 164 ++++++++++ cmd/cli/upstream_retirement_test.go | 101 +++++++ 19 files changed, 1701 insertions(+), 74 deletions(-) create mode 100644 cmd/cli/api_reload_shutdown_test.go create mode 100644 cmd/cli/force_reload_lifetime_test.go create mode 100644 cmd/cli/internal_log_shutdown_test.go create mode 100644 cmd/cli/listener_shutdown_test.go create mode 100644 cmd/cli/network_shutdown_test.go create mode 100644 cmd/cli/restore_lock_test.go create mode 100644 cmd/cli/shutdown_test.go create mode 100644 cmd/cli/upstream_retirement_test.go diff --git a/cmd/cli/api_reload_shutdown_test.go b/cmd/cli/api_reload_shutdown_test.go new file mode 100644 index 00000000..c52e6d53 --- /dev/null +++ b/cmd/cli/api_reload_shutdown_test.go @@ -0,0 +1,157 @@ +package cli + +import ( + "context" + "encoding/base64" + "net" + "os" + "os/exec" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +func TestAPIConfigReloadShutdown(t *testing.T) { + const env = "CTRLD_TEST_API_RELOAD_SHUTDOWN" + mode := os.Getenv(env) + if mode == "" { + for _, mode := range []string{"stop-exclude", "abort-exclude", "stop-custom", "abort-custom", "stop-internal", "abort-internal", "stop-fetch", "abort-fetch", "live-exclude", "live-custom", "live-internal"} { + t.Run(mode, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestAPIConfigReloadShutdown$", "-test.v") + cmd.Env = append(os.Environ(), env+"="+mode) + out, err := cmd.CombinedOutput() + require.NoError(t, err, "%s", out) + }) + } + return + } + cdUID = "testuid" + version = "dev" + p := &prog{ + cfg: &ctrld.Config{}, rc: &controld.ResolverConfig{}, + stopCh: make(chan struct{}), runAbortCh: make(chan struct{}), + apiForceReloadCh: make(chan struct{}), apiReloadCh: make(chan *ctrld.Config), + } + rc := &controld.ResolverConfig{Exclude: []string{"internal.test"}} + if strings.HasSuffix(mode, "internal") { + rc.Exclude = nil + rc.SplitDNS = []controld.SplitDNS{{Domain: "internal.test", Mode: controld.SplitDNSModeOS}} + } + if strings.HasSuffix(mode, "custom") { + rc.Ctrld.CustomConfig = base64.StdEncoding.EncodeToString([]byte(`[listener.0] +ip = "127.0.0.1" +port = 5354 +[network.0] +cidrs = ["127.0.0.0/8"] +[upstream.0] +type = "legacy" +endpoint = "127.0.0.1" +`)) + } + entered, canceled := make(chan struct{}), make(chan struct{}) + fetchResolverConfigForReload = func(ctx context.Context, _ *controld.ResolverConfigRequest, _ bool) (*controld.ResolverConfig, error) { + close(entered) + if strings.HasSuffix(mode, "fetch") { + <-ctx.Done() + close(canceled) + // A late successful response must not schedule a reload after stop. + return rc, nil + } + return rc, nil + } + done := make(chan struct{}) + go func() { p.apiConfigReload(); close(done) }() + p.apiForceReloadCh <- struct{}{} + awaitShutdown(t, entered) + if strings.HasPrefix(mode, "live-") { + select { + case got := <-p.apiReloadCh: + if strings.HasSuffix(mode, "custom") { + require.NotNil(t, got) + require.Equal(t, "127.0.0.1", got.Listener["0"].IP) + } else { + require.Nil(t, got) + } + case <-time.After(2 * time.Second): + t.Fatal("running API loop did not deliver its reload") + } + } else if !strings.HasSuffix(mode, "fetch") { + marker := "exclude list changes detected, reloading" + if strings.HasSuffix(mode, "custom") { + marker = "custom config changes detected, reloading" + } else if strings.HasSuffix(mode, "internal") { + marker = "internal domain changes detected, reloading" + } + require.Eventually(t, func() bool { return strings.Contains(logOutput.String(), marker) }, time.Second, time.Millisecond) + } + if strings.HasPrefix(mode, "abort-") { + close(p.runAbortCh) + } else { + close(p.stopCh) + } + awaitShutdown(t, done) + if strings.HasSuffix(mode, "fetch") { + awaitShutdown(t, canceled) + require.Empty(t, p.rc.Exclude, "late fetch must not update run state") + } +} + +func TestForceFetchingAPIStopsWithoutReceiver(t *testing.T) { + old := cdUID + cdUID = "testuid" + t.Cleanup(func() { cdUID = old }) + for _, abort := range []bool{false, true} { + for _, delivered := range []bool{false, true} { + p := &prog{cfg: &ctrld.Config{}, stopCh: make(chan struct{}), runAbortCh: make(chan struct{}), apiForceReloadCh: make(chan struct{})} + p.forceFetchingAPI("testuid.verify.controld.com") + done := p.apiForceReloadGroup.DoChan("force_sync_api", func() (interface{}, error) { t.Error("lost active force-reload worker"); return nil, nil }) + if delivered { + select { + case <-p.apiForceReloadCh: + case <-time.After(time.Second): + t.Fatal("no force-reload signal") + } + } + if abort { + close(p.runAbortCh) + } else { + close(p.stopCh) + } + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("force-reload send/cooldown survived stop") + } + } + } +} + +type closeCountingLogConn struct { + net.Conn + closes atomic.Int32 +} + +func (c *closeCountingLogConn) Close() error { c.closes.Add(1); return nil } +func (*closeCountingLogConn) Write(b []byte) (int, error) { return len(b), nil } + +func TestStartupLogConnectionCloseOwnership(t *testing.T) { + c := &closeCountingLogConn{} + p := &prog{cfg: &ctrld.Config{}, dnsWatcherStopCh: make(chan struct{}), logConn: c} + var wg sync.WaitGroup + for i := 0; i < 20; i++ { + wg.Add(1) + go func() { defer wg.Done(); p.notifyExitToLogServer(); p.closeLogConn() }() + } + wg.Wait() + require.NoError(t, p.shutdown()) + require.EqualValues(t, 1, c.closes.Load()) + require.Nil(t, p.logConn) +} diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 7e86915c..8679f406 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -50,6 +50,7 @@ const ( windowsForwardersFilename = ".forwarders.txt" oldBinSuffix = "_previous" msgExit = "$$EXIT$$" + shutdownTimeout = 10 * time.Second ) var ( @@ -144,10 +145,15 @@ func initCLI() { } // isMobile reports whether the current OS is a mobile platform. -func isMobile() bool { +var isMobile = func() bool { return runtime.GOOS == "android" || runtime.GOOS == "ios" } +var ( + networkUp = ctrldnet.Up + newRouter = router.New +) + func updateConfigInterceptMode(cfg *ctrld.Config, mode string) bool { desired := "" switch mode { @@ -241,6 +247,8 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { dnsWatcherStopCh: make(chan struct{}), apiReloadCh: make(chan *ctrld.Config), apiForceReloadCh: make(chan struct{}), + runDone: make(chan struct{}), + runAbortCh: make(chan struct{}), cfg: &cfg, appCallback: appCallback, } @@ -273,7 +281,15 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { return } - if !daemon { + switch { + case isMobile(): + defer p.finishRun() + // There is no OS service manager here, and s.Run parks a goroutine on a + // signal that never arrives, leaking one per start/stop cycle. + if err := p.Start(nil); err != nil { + mainLog.Load().Fatal().Err(err).Msg("failed to start ctrld") + } + case !daemon: // We need to call s.Run() as soon as possible to response to the OS manager, so it // can see ctrld is running and don't mark ctrld as failed service. go func() { @@ -316,12 +332,12 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { mainLog.Load().Info().Msgf("os: %s", osVersion()) // Wait for network up. - if !ctrldnet.Up() { + if !networkUp() { failRunUnclassified(mainLog.Load().Error(), "network is not up yet", p.notifyExitToLogServer) return } - p.router = router.New(&cfg, cdUID != "") + p.router = newRouter(&cfg, cdUID != "") cs, err := newControlServer(filepath.Join(sockDir, ControlSocketName())) if err != nil { mainLog.Load().Warn().Err(err).Msg("could not create control server") @@ -492,6 +508,9 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { close(waitCh) <-stopCh + if !isMobile() { + p.finishRun() + } } func writeConfigFile(cfg *ctrld.Config) error { @@ -2200,8 +2219,9 @@ func newSocketControlClientMobile(dir string, stopCh chan struct{}) *controlClie case <-stopCh: return nil default: - _, err := cc.post("/", nil) + resp, err := cc.post("/", nil) if err == nil { + resp.Body.Close() return cc } else { bo.BackOff(ctx, err) @@ -2339,6 +2359,7 @@ func checkDeactivationPin(s service.Service, stopCh chan struct{}) error { resp, err := cc.post(deactivationPath, bytes.NewReader(data)) mainLog.Load().Debug().Msg("Posting deactivation request done") if resp != nil { + defer resp.Body.Close() switch resp.StatusCode { case http.StatusBadRequest: mainLog.Load().Error().Msg(errRequiredDeactivationPin.Error()) diff --git a/cmd/cli/control_server.go b/cmd/cli/control_server.go index c6ba7f93..3e544de5 100644 --- a/cmd/cli/control_server.go +++ b/cmd/cli/control_server.go @@ -76,7 +76,6 @@ func (s *controlServer) start() error { } func (s *controlServer) stop() error { - _ = os.Remove(s.addr) ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) defer cancel() return s.server.Shutdown(ctx) diff --git a/cmd/cli/dns_intercept_darwin.go b/cmd/cli/dns_intercept_darwin.go index 7336b84d..2402f3e8 100644 --- a/cmd/cli/dns_intercept_darwin.go +++ b/cmd/cli/dns_intercept_darwin.go @@ -1480,6 +1480,10 @@ func (p *prog) pfStabilizationLoop(ctx context.Context, stableRequired time.Dura func (p *prog) pfStabilizationLoopWithMaxWait(ctx context.Context, stableRequired, maxWaitDuration time.Duration) { defer p.pfStabilizing.Store(false) + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() pollInterval := 1500 * time.Millisecond pollTicker := time.NewTicker(pollInterval) @@ -1504,6 +1508,9 @@ func (p *prog) pfStabilizationLoopWithMaxWait(ctx context.Context, stableRequire case <-pollTicker.C: } + if p.networkActivityClosed() { + return + } if p.pfExecBackoffActive() { continue } @@ -2075,6 +2082,10 @@ func (p *prog) scheduleDNSAfterVPNSettleRefresh(reason string, delay time.Durati p.pfSettleFollowupTimer.Stop() } p.pfSettleFollowupTimer = time.AfterFunc(delay, func() { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() if p.dnsInterceptState == nil { return } @@ -2083,7 +2094,7 @@ func (p *prog) scheduleDNSAfterVPNSettleRefresh(reason string, delay time.Durati } // stopPFSettleFollowup cancels a pending post-settle refresh. A callback that already -// fired still checks dnsInterceptState, so either teardown order is safe. +// fired is fenced and joined by closeNetMonitor during program shutdown. func (p *prog) stopPFSettleFollowup() { p.pfDelayedRecheckMu.Lock() defer p.pfDelayedRecheckMu.Unlock() @@ -2161,6 +2172,10 @@ func (p *prog) scheduleDelayedRechecks() { for _, delay := range []time.Duration{pfAnchorRecheckDelay, pfAnchorRecheckDelayLong} { delay := delay timer := time.AfterFunc(delay, func() { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() if p.dnsInterceptState == nil || p.pfStabilizing.Load() { return } @@ -2471,6 +2486,10 @@ var pfInterceptMonitorDelays = []time.Duration{0, 500 * time.Millisecond, time.S // The backoff schedule provides both fast detection (immediate + 500ms) and extended // coverage (up to ~8s) to win the race against async pf reloads by hypervisors. func (p *prog) pfInterceptMonitor() { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() // Eligibility first, ownership second. A monitor that is about to stand down must not // take the flag on its way out: the post-stabilization verifier reads that flag as // "another prober is working" and would step aside for a prober that never probes. @@ -2494,7 +2513,7 @@ func (p *prog) pfInterceptMonitor() { if delay > 0 { time.Sleep(delay) } - if !p.interceptProbeMonitorAllowed() { + if p.networkActivityClosed() || !p.interceptProbeMonitorAllowed() { mainLog.Load().Debug().Msg("DNS intercept monitor: aborting — intercept disabled or stabilizing") return } diff --git a/cmd/cli/dns_intercept_windows.go b/cmd/cli/dns_intercept_windows.go index 4dd87d25..ce965361 100644 --- a/cmd/cli/dns_intercept_windows.go +++ b/cmd/cli/dns_intercept_windows.go @@ -2744,6 +2744,10 @@ func (p *prog) healBlockedLoopbackDNS(state *wfpState, reason string) bool { func (p *prog) scheduleDelayedRechecks() { for _, delay := range []time.Duration{pfAnchorRecheckDelay, pfAnchorRecheckDelayLong} { time.AfterFunc(delay, func() { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() if p.dnsInterceptState == nil { return } diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index bb6b0f58..4d3ca6f5 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -285,6 +285,7 @@ func (p *prog) serveDNS(listenerNum string) error { defer s.Shutdown() select { case <-p.stopCh: + case <-p.runAbortCh: case <-ctx.Done(): case err := <-errCh: // Local ipv6 listener should not terminate ctrld. @@ -305,6 +306,7 @@ func (p *prog) serveDNS(listenerNum string) error { defer s.Shutdown() select { case <-p.stopCh: + case <-p.runAbortCh: case <-ctx.Done(): case err := <-errCh: // RFC1918 listener should not terminate ctrld. @@ -321,10 +323,19 @@ func (p *prog) serveDNS(listenerNum string) error { s, errCh := runDNSServer(addr, proto, handler) defer s.Shutdown() - p.started <- struct{}{} + select { + case p.started <- struct{}{}: + case <-p.stopCh: + return nil + case <-p.runAbortCh: + return nil + case <-ctx.Done(): + return nil + } select { case <-p.stopCh: + case <-p.runAbortCh: case <-ctx.Done(): case err := <-errCh: return err @@ -1279,7 +1290,7 @@ func runDNSServer(addr, network string, handler dns.Handler) (*dns.Server, <-cha startedCh := make(chan struct{}) s.NotifyStartedFunc = func() { sync.OnceFunc(func() { close(startedCh) })() } - errCh := make(chan error) + errCh := make(chan error, 1) go func() { defer close(errCh) if err := s.ListenAndServe(); err != nil { @@ -1440,6 +1451,10 @@ func (p *prog) selfUninstallCoolOfPeriod() { // forceFetchingAPI sends signal to force syncing API config if run in cd mode, // and the domain == "cdUID.verify.controld.com" func (p *prog) forceFetchingAPI(domain string) { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() if cdUID == "" { return } @@ -1456,9 +1471,32 @@ func (p *prog) forceFetchingAPI(domain string) { return } _ = p.apiForceReloadGroup.DoChan("force_sync_api", func() (interface{}, error) { - p.apiForceReloadCh <- struct{}{} + if !p.beginNetworkActivity() { + return nil, nil + } + defer p.netMonitorWG.Done() + shutdown := p.networkActivityDone() + select { + case <-shutdown: + return nil, nil + case p.apiForceReloadCh <- struct{}{}: + case <-p.stopCh: + return nil, nil + case <-p.runAbortCh: + return nil, nil + } // Wait here to prevent abusing API if we are flooded. - time.Sleep(timeDurationOrDefault(p.cfg.Service.ForceRefetchWaitTime, 30) * time.Second) + p.mu.Lock() + wait := timeDurationOrDefault(p.cfg.Service.ForceRefetchWaitTime, 30) * time.Second + p.mu.Unlock() + timer := time.NewTimer(wait) + defer timer.Stop() + select { + case <-timer.C: + case <-shutdown: + case <-p.stopCh: + case <-p.runAbortCh: + } return nil, nil }) } @@ -1670,20 +1708,35 @@ func FlushDNSCache() error { return nil } -// monitorNetworkChanges starts monitoring for network interface changes +// networkChangeCallback fences even callbacks already dispatched by netmon +// when Close begins. The monitor does not track those callback goroutines. +func (p *prog) networkChangeCallback(fn netmon.ChangeFunc) netmon.ChangeFunc { + return func(delta *netmon.ChangeDelta) { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() + fn(delta) + } +} + +// monitorNetworkChanges starts monitoring for network interface changes. func (p *prog) monitorNetworkChanges() error { mon, err := newNetworkChangeMonitorFn(func(format string, args ...any) { - // Always fetch the latest logger (and inject the prefix) + // Always fetch the latest logger (and inject the prefix). mainLog.Load().Printf("netmon: "+format, args...) }) if err != nil { return fmt.Errorf("creating network monitor: %w", err) } - - mon.RegisterChangeCallback(func(delta *netmon.ChangeDelta) { + mon.RegisterChangeCallback(p.networkChangeCallback(func(delta *netmon.ChangeDelta) { p.handleNetworkChange(delta, mon.IsMajorChangeFrom(delta.Old, delta.New)) - }) - + })) + if !p.setNetMonitor(mon) { + _ = mon.Close() + mainLog.Load().Debug().Msg("network monitor discarded, ctrld is stopping") + return nil + } mon.Start() mainLog.Load().Debug().Msg("Network monitor started") return nil @@ -2222,12 +2275,19 @@ func (p *prog) debounceRecovery(transitionID uint64) { } p.recoveryDebounceMu.Lock() defer p.recoveryDebounceMu.Unlock() + if p.networkActivityClosed() { + return + } if p.recoveryDebounceTimer != nil { p.recoveryDebounceTimer.Stop() mainLog.Load().Debug().Msg("Recovery debounce: resetting timer (rapid network change)") } p.recoveryDebounceTimer = time.AfterFunc(recoveryDebounceWindow, func() { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() p.networkSourceMu.Lock() if p.networkAcceptedGen.Load() != transitionID { p.networkSourceMu.Unlock() @@ -2293,6 +2353,10 @@ func (p *prog) handleRecovery(reason RecoveryReason) { } func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID uint64) { + if !p.beginNetworkActivity() { + return + } + defer p.netMonitorWG.Done() p.networkSourceMu.Lock() if transitionID != 0 && p.networkAcceptedGen.Load() != transitionID { p.networkSourceMu.Unlock() @@ -2313,6 +2377,15 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u targetBefore := p.interceptTargetSnapshot() outcome := recoveryOutcomeSuperseded defer func() { p.endRecovery(&diagnostic, outcome, targetBefore) }() + // Clean every exit, including shutdown before the first DNS mutation. + // Run cleanup before the end snapshot so it reports the final state. + defer p.recoveryCanceledCleanup(gen) + if recoveryCtx.Err() != nil { + if p.recoveryGen.Load() == gen { + outcome = recoveryOutcomeCanceled + } + return + } if reason == RecoveryReasonNetworkChange { mainLog.Load().Debug().Msg("Network change recovery now owns shared recovery state") } @@ -2399,10 +2472,9 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u // Wait indefinitely until one of the upstreams recovers. recovered, err := waitForUpstreamRecoveryFn(p, recoveryCtx, upstreams, &diagnostic) if err != nil { - if p.recoveryOwnsState(gen) { + if p.recoveryGen.Load() == gen { outcome = recoveryOutcomeCanceled } - p.recoveryCanceledCleanup(gen) return } diagnostic.recoveredUpstream = recovered @@ -2482,6 +2554,10 @@ func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string var wg sync.WaitGroup mainLog.Load().Debug().Msgf("Starting upstream recovery check for %d upstreams", len(upstreams)) + defer func() { + cancel() + wg.Wait() + }() for name, uc := range upstreams { wg.Add(1) @@ -2561,7 +2637,6 @@ func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string case <-ctx.Done(): return "", ctx.Err() } - wg.Wait() return recovered, nil } diff --git a/cmd/cli/force_reload_lifetime_test.go b/cmd/cli/force_reload_lifetime_test.go new file mode 100644 index 00000000..b2600d34 --- /dev/null +++ b/cmd/cli/force_reload_lifetime_test.go @@ -0,0 +1,48 @@ +package cli + +import ( + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +func TestForceFetchingAPIDrainsBeforeRestoration(t *testing.T) { + old := cdUID + cdUID = "testuid" + t.Cleanup(func() { cdUID = old }) + p := &prog{cfg: &ctrld.Config{}, stopCh: make(chan struct{}), runAbortCh: make(chan struct{}), apiForceReloadCh: make(chan struct{})} + // Pause the admitted producer after delivery, before it reads cooldown + // configuration. Shutdown must join it even though the consumer is done. + p.mu.Lock() + locked := true + defer func() { + if locked { + p.mu.Unlock() + } + }() + p.forceFetchingAPI("testuid.verify.controld.com") + select { + case <-p.apiForceReloadCh: + case <-time.After(time.Second): + t.Fatal("force-reload producer did not send") + } + closed := make(chan struct{}) + go func() { p.closeNetMonitor(); close(closed) }() + waitNetworkShutdownGate(t, p) + assertNetworkShutdownBlocked(t, closed) + p.mu.Unlock() + locked = false + waitNetworkShutdownSignal(t, closed, "force-reload producer drain") + // Service restoration precedes closing stopCh, so that channel alone + // cannot cancel the producer's send or cooldown while restoration joins it. + if stopRequested(p.stopCh) || stopRequested(p.runAbortCh) { + t.Fatal("network shutdown closed a caller-owned lifetime channel") + } + p.forceFetchingAPI("testuid.verify.controld.com") + select { + case <-p.apiForceReloadCh: + t.Fatal("shutdown admitted a late force-reload producer") + default: + } +} diff --git a/cmd/cli/internal_log_shutdown_test.go b/cmd/cli/internal_log_shutdown_test.go new file mode 100644 index 00000000..56161058 --- /dev/null +++ b/cmd/cli/internal_log_shutdown_test.go @@ -0,0 +1,50 @@ +package cli + +import ( + "errors" + "os" + "path/filepath" + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +func TestFinishRunClosesInternalFilesWithoutServiceStop(t *testing.T) { + p := &prog{ + cfg: &ctrld.Config{}, runDone: make(chan struct{}), runAbortCh: make(chan struct{}), + dnsWatcherStopCh: make(chan struct{}), + internalLogWriter: newLogWriter(), internalJournalWriter: newSmallLogWriter(), + } + var files []*os.File + var paths []string + for i, writer := range []*logWriter{p.internalLogWriter, p.internalJournalWriter} { + path := filepath.Join(t.TempDir(), []string{"debug.log", "journal.log"}[i]) + if err := writer.setLogFile(path, logBudget{maxSize: 4096, backups: 1}); err != nil { + t.Fatal(err) + } + t.Cleanup(writer.closeLogFile) + files = append(files, writer.rotating().f) + paths = append(paths, path) + } + p.onStopped = []func(){func() { + for _, writer := range []*logWriter{p.internalLogWriter, p.internalJournalWriter} { + if _, err := writer.Write([]byte("final cleanup\n")); err != nil { + t.Error(err) + } + } + }} + close(p.runDone) + p.finishRun() + for i, f := range files { + if _, err := f.Stat(); !errors.Is(err, os.ErrClosed) { + t.Errorf("file %d is not closed: %v", i, err) + } + b, err := os.ReadFile(paths[i]) + if err != nil { + t.Fatal(err) + } + if string(b) != "final cleanup\n" { + t.Errorf("file %d lost final cleanup: %q", i, b) + } + } +} diff --git a/cmd/cli/listener_shutdown_test.go b/cmd/cli/listener_shutdown_test.go new file mode 100644 index 00000000..b4f0bd79 --- /dev/null +++ b/cmd/cli/listener_shutdown_test.go @@ -0,0 +1,293 @@ +package cli + +import ( + "context" + "errors" + "io" + "net" + "os" + "os/exec" + "runtime" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/stretchr/testify/require" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +func TestStopRestorationFailureStillStopsWorkers(t *testing.T) { + old := deAllocateIPFn + t.Cleanup(func() { deAllocateIPFn = old }) + want := errors.New("deallocation failed") + calls := 0 + deAllocateIPFn = func(string) error { calls++; return want } + p := &prog{ + cfg: &ctrld.Config{Service: ctrld.ServiceConfig{AllocateIP: true}, Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1"}}}, + waitCh: make(chan struct{}), stopCh: make(chan struct{}), runDone: make(chan struct{}), + runAbortCh: make(chan struct{}), dnsWatcherStopCh: make(chan struct{}), + pinCodeValidCh: make(chan struct{}, 1), + } + // Accept the stop even if another test configured a deactivation pin. + p.pinCodeValidCh <- struct{}{} + require.NoError(t, p.Start(nil)) + require.ErrorIs(t, p.Stop(nil), want) + require.True(t, stopRequested(p.stopCh)) + awaitShutdown(t, p.runDone) + p.finishRun() + require.ErrorIs(t, p.restoreOSState(), want, "the first error must survive sync.Once") + require.Equal(t, 1, calls, "never invoke the real IP deallocator") +} + +func awaitShutdown(t *testing.T, done <-chan struct{}) { + t.Helper() + select { + case <-done: + case <-time.After(3 * time.Second): + t.Fatal("shutdown did not complete") + } +} + +func TestFinishRunWaitsBeyondTimeout(t *testing.T) { + c, peer := net.Pipe() + defer peer.Close() + p := &prog{cfg: &ctrld.Config{}, runAbortCh: make(chan struct{}), runDone: make(chan struct{}), dnsWatcherStopCh: make(chan struct{}), logConn: c} + done := make(chan struct{}) + go func() { p.finishRun(); close(done) }() + select { + case <-done: + t.Fatal("released resources while the run was alive") + case <-time.After(shutdownTimeout + 50*time.Millisecond): + } + close(p.runDone) + awaitShutdown(t, done) + _, err := peer.Write([]byte("x")) + require.ErrorIs(t, err, io.ErrClosedPipe) +} + +// Each case runs in a subprocess because run initializes package-global config +// and logging. No service, interface DNS or network-monitor setup is permitted. +func TestRunListenerShutdown(t *testing.T) { + const env = "CTRLD_TEST_LISTENER_SHUTDOWN" + mode := os.Getenv(env) + if mode == "" { + for _, mode := range []string{"stop-before-ready", "abort-before-ready", "drain", "reload", "reload-before-ready", "stop-during-hook", "stop-during-post", "wire-udp", "wire-tcp", "api-drain"} { + t.Run(mode, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 12*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunListenerShutdown$", "-test.v") + cmd.Env = append(os.Environ(), env+"="+mode) + out, err := cmd.CombinedOutput() + require.NoError(t, err, "%s", out) + }) + } + return + } + defer func() { + if t.Failed() { + t.Log(logOutput.String()) + buf := make([]byte, 1<<20) + n := runtime.Stack(buf, true) + t.Log(string(buf[:n])) + } + }() + isMobile = func() bool { return true } + cdUID = "" + iface = "" + apiEntered, apiCanceled, apiRelease := make(chan struct{}), make(chan struct{}), make(chan struct{}) + if mode == "api-drain" { + cdUID = "testuid" + fetchResolverConfigForReload = func(ctx context.Context, _ *controld.ResolverConfigRequest, _ bool) (*controld.ResolverConfig, error) { + close(apiEntered) + <-ctx.Done() + close(apiCanceled) + <-apiRelease + return nil, ctx.Err() + } + } + var monitorCalls, postCalls atomic.Int32 + monitorNetworkChangesFn = func(*prog) error { monitorCalls.Add(1); return nil } + postRunFn = func(*prog) { postCalls.Add(1) } + dir := t.TempDir() + cfg = ctrld.Config{Service: ctrld.ServiceConfig{LogPath: dir + "/ctrld.log"}, Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1"}}} + p := &prog{ + cfg: &cfg, waitCh: make(chan struct{}), stopCh: make(chan struct{}), runAbortCh: make(chan struct{}), + runDone: make(chan struct{}), dnsWatcherStopCh: make(chan struct{}), apiReloadCh: make(chan *ctrld.Config), + reloadDoneCh: make(chan struct{}, 1), + apiForceReloadCh: make(chan struct{}), + } + logConnection := &closeCountingLogConn{} + p.logConn = logConnection + entered, drain, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + defer unblock() + var hooks atomic.Int32 + p.onStarted = []func(){func() { hooks.Add(1) }} + mutationEntered, mutationRelease, restored := make(chan struct{}), make(chan struct{}), make(chan struct{}) + stopDuringMutation := mode == "stop-during-hook" || mode == "stop-during-post" + if stopDuringMutation { + mutation := func() { + close(mutationEntered) + <-mutationRelease + select { + case <-restored: + t.Error("startup mutation ran after restoration") + default: + } + } + p.onStopped = []func(){func() { close(restored) }} + if mode == "stop-during-hook" { + p.onStarted = []func(){mutation} + } else { + postRunFn = func(*prog) { mutation(); postCalls.Add(1) } + } + } + wire := mode == "wire-udp" || mode == "wire-tcp" + if wire { + ln, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + cfg.Listener["0"].Port = ln.Addr().(*net.TCPAddr).Port + require.NoError(t, ln.Close()) + // The real handler blocks in its semaphore while both server Shutdown + // calls drain. Use an internal query so no external upstream is needed. + serveDNSFn = func(p *prog, n string) error { + p.sema = &shutdownSemaphore{entered: entered, resume: release} + return p.serveDNS(n) + } + } else { + serveDNSFn = func(p *prog, _ string) error { + close(entered) + if mode == "drain" || mode == "reload" || mode == "api-drain" || stopDuringMutation { + p.started <- struct{}{} + } + select { + case <-p.stopCh: + case <-p.runAbortCh: + } + close(drain) + <-release + return nil + } + } + close(p.waitCh) + require.NoError(t, p.Start(nil)) + var queryDone chan error + if wire { + require.Eventually(t, func() bool { return postCalls.Load() == 1 }, time.Second, time.Millisecond) + proto := mode[len("wire-"):] + queryDone = make(chan error, 1) + go func() { + m := new(dns.Msg) + m.SetQuestion(selfCheckInternalTestDomain+".", dns.TypeA) + answer, _, err := (&dns.Client{Net: proto, Timeout: 5 * time.Second}).Exchange(m, net.JoinHostPort("127.0.0.1", strconv.Itoa(cfg.Listener["0"].Port))) + if err == nil && answer.Rcode != dns.RcodeSuccess { + err = errors.New("internal query failed") + } + queryDone <- err + }() + } + awaitShutdown(t, entered) + if mode == "api-drain" { + select { + case p.apiForceReloadCh <- struct{}{}: + case <-time.After(time.Second): + t.Fatal("production run did not start the API reload worker") + } + awaitShutdown(t, apiEntered) + } + if mode == "reload" || mode == "reload-before-ready" { + if mode == "reload" { + require.Eventually(t, func() bool { return postCalls.Load() == 1 }, time.Second, time.Millisecond) + } + configPath = dir + "/reload.toml" + p.apiReloadCh <- &ctrld.Config{ + Service: cfg.Service, + Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 5354}}, + Network: map[string]*ctrld.NetworkConfig{"0": {Cidrs: []string{"127.0.0.0/8"}}}, + Upstream: map[string]*ctrld.UpstreamConfig{"0": {Type: ctrld.ResolverTypeDOT, Endpoint: "example.test:853", BootstrapIP: "127.0.0.1"}}, + } + if mode == "reload-before-ready" { + // Give the reload a chance to reach the old run's cancellation path. + select { + case <-p.reloadDoneCh: + t.Fatal("reload skipped one-time startup") + case <-time.After(30 * time.Millisecond): + } + p.started <- struct{}{} + } + select { + case <-p.reloadDoneCh: + case <-time.After(3 * time.Second): + t.Fatal("reload waited for a persistent listener to stop") + } + select { + case <-drain: + t.Fatal("reload stopped the persistent listener") + default: + } + require.EqualValues(t, 1, hooks.Load()) + require.EqualValues(t, 1, postCalls.Load()) + } + if stopDuringMutation { + awaitShutdown(t, mutationEntered) + stopDone := make(chan struct{}) + go func() { _ = p.Stop(nil); close(stopDone) }() + select { + case <-restored: + t.Fatal("restoration raced a startup OS mutation") + case <-time.After(30 * time.Millisecond): + } + close(mutationRelease) + awaitShutdown(t, stopDone) + awaitShutdown(t, restored) + require.False(t, p.startOSState(func() { t.Error("startup permitted after restoration") })) + } else if mode != "abort-before-ready" { + close(p.stopCh) + } + finished := make(chan struct{}) + go func() { p.finishRun(); close(finished) }() + awaitShutdown(t, p.runDone) + if !wire { + awaitShutdown(t, drain) + } + select { + case <-finished: + t.Fatal("finishRun returned before the listener/handler drained") + case <-time.After(30 * time.Millisecond): + } + unblock() + if mode == "api-drain" { + awaitShutdown(t, apiCanceled) + select { + case <-finished: + t.Fatal("finishRun returned before the API reload worker") + case <-time.After(30 * time.Millisecond): + } + close(apiRelease) + } + awaitShutdown(t, finished) + require.EqualValues(t, 1, logConnection.closes.Load(), "startup and shutdown must share log-connection ownership") + if wire { + require.NoError(t, <-queryDone, "active DNS query must finish before resource retirement") + } + if mode == "stop-before-ready" || mode == "abort-before-ready" { + require.Zero(t, hooks.Load(), "startup hooks must not run after cancellation") + require.Zero(t, postCalls.Load()) + } + require.EqualValues(t, 1, monitorCalls.Load(), "real network monitor must not be reached") +} + +type shutdownSemaphore struct { + entered chan struct{} + resume <-chan struct{} + once sync.Once +} + +func (s *shutdownSemaphore) acquire() { s.once.Do(func() { close(s.entered) }); <-s.resume } +func (*shutdownSemaphore) release() {} diff --git a/cmd/cli/network_monitor_test.go b/cmd/cli/network_monitor_test.go index 33dba8dd..d23a9c1a 100644 --- a/cmd/cli/network_monitor_test.go +++ b/cmd/cli/network_monitor_test.go @@ -13,6 +13,8 @@ type sourceTestMonitor struct { started bool } +func (*sourceTestMonitor) Close() error { return nil } + func (m *sourceTestMonitor) RegisterChangeCallback(fn netmon.ChangeFunc) func() { m.callback = fn return func() {} diff --git a/cmd/cli/network_shutdown_test.go b/cmd/cli/network_shutdown_test.go new file mode 100644 index 00000000..714247a3 --- /dev/null +++ b/cmd/cli/network_shutdown_test.go @@ -0,0 +1,322 @@ +package cli + +import ( + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +func guardNetworkShutdownHostMutations(t *testing.T) { + t.Helper() + oldReset, oldRemove := recoveryResetDNSFn, removeInterceptDNSTargetFn + recoveryResetDNSFn = func(*prog, bool, bool) { panic("unexpected recovery DNS reset") } + // Also fence the first platform boundary inside real resetDNS, so mutation + // testing a bypassed recoveryResetDNSFn can never touch the test runner's DNS/PF. + removeInterceptDNSTargetFn = func(*prog, string) { panic("real resetDNS crossed host safety boundary") } + t.Cleanup(func() { recoveryResetDNSFn, removeInterceptDNSTargetFn = oldReset, oldRemove }) +} + +func waitNetworkShutdownSignal(t *testing.T, ch <-chan struct{}, what string) { + t.Helper() + select { + case <-ch: + case <-time.After(2 * time.Second): + t.Fatalf("timed out waiting for %s", what) + } +} + +func waitNetworkShutdownGate(t *testing.T, p *prog) { + t.Helper() + deadline := time.Now().Add(2 * time.Second) + for !p.networkActivityClosed() { + if time.Now().After(deadline) { + t.Fatal("shutdown did not close admission") + } + time.Sleep(time.Millisecond) + } +} + +func assertNetworkShutdownBlocked(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + t.Fatal("shutdown returned before admitted work finished") + case <-time.After(20 * time.Millisecond): + } +} + +func TestNetworkShutdownRejectsLateMonitorPublication(t *testing.T) { + p := &prog{} + p.closeNetMonitor() + // A zero monitor is never started or closed: on rejected publication the + // initializer retains ownership. No OS monitor or DNS policy is installed. + if p.setNetMonitor(new(netmon.Monitor)) { + t.Fatal("monitor published after shutdown") + } + if p.netMonitor != nil { + t.Fatal("shutdown retained a monitor") + } + if p.beginNetworkActivity() { + p.netMonitorWG.Done() + t.Fatal("work admitted after shutdown") + } +} + +func TestNetworkShutdownDrainsCallbacksAndConcurrentClosers(t *testing.T) { + guardNetworkShutdownHostMutations(t) + p := &prog{} + entered, release := make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + defer unblock() + var calls atomic.Int32 + // Stub the entire callback body before any platform or DNS boundary. This + // is the same wrapper used by RegisterChangeCallback in production. + callback := p.networkChangeCallback(func(*netmon.ChangeDelta) { + calls.Add(1) + close(entered) + <-release + // An admitted callback may try to schedule recovery after Close fenced + // admission. Neither synchronous nor deferred work may start then. + p.debounceRecovery(p.networkAcceptedGen.Load()) + p.handleRecovery(RecoveryReasonNetworkChange) + }) + go callback(nil) + waitNetworkShutdownSignal(t, entered, "callback admission") + + closed1, closed2 := make(chan struct{}), make(chan struct{}) + go func() { p.closeNetMonitor(); close(closed1) }() + waitNetworkShutdownGate(t, p) + go func() { p.closeNetMonitor(); close(closed2) }() + // Model a callback dispatched by netmon before Close but scheduled later. + callback(nil) + assertNetworkShutdownBlocked(t, closed1) + assertNetworkShutdownBlocked(t, closed2) + unblock() + waitNetworkShutdownSignal(t, closed1, "first close") + waitNetworkShutdownSignal(t, closed2, "second close") + callback(nil) + if calls.Load() != 1 || p.recoveryGen.Load() != 0 || p.recoveryDebounceTimer != nil { + t.Fatalf("late work ran: callbacks=%d recovery=%d timer=%v", calls.Load(), p.recoveryGen.Load(), p.recoveryDebounceTimer) + } +} + +func TestNetworkShutdownCancelsAndDrainsRecovery(t *testing.T) { + testNetworkShutdownCancelsAndDrainsRecovery(t, func(p *prog) { + p.handleRecovery(RecoveryReasonRegularFailure) + }) +} + +func TestNetworkShutdownDrainsDeferredRecovery(t *testing.T) { + testNetworkShutdownCancelsAndDrainsRecovery(t, func(p *prog) { + p.networkChangeCallback(func(*netmon.ChangeDelta) { p.debounceRecovery(p.networkAcceptedGen.Load()) })(nil) + }) +} + +func testNetworkShutdownCancelsAndDrainsRecovery(t *testing.T, start func(*prog)) { + logs := captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + guardNetworkShutdownHostMutations(t) + oldReset, oldIntercept := recoveryResetDNSFn, dnsIntercept + dnsIntercept = false + defer func() { recoveryResetDNSFn, dnsIntercept = oldReset, oldIntercept }() + p := &prog{cfg: &ctrld.Config{}} + entered, canceled, release := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var releaseOnce sync.Once + unblock := func() { releaseOnce.Do(func() { close(release) }) } + defer unblock() + var resets atomic.Int32 + // resetDNS is the FIRST host-mutating boundary in traditional recovery. + // Block there; never invoke real resetDNS, setDNS, PF, or NRPT operations. + recoveryResetDNSFn = func(p *prog, isStart, restoreStatic bool) { + resets.Add(1) + p.recoveryCancelMu.Lock() + cancel := p.recoveryCancel + p.recoveryCancel = func() { cancel(); close(canceled) } + p.recoveryCancelMu.Unlock() + close(entered) + <-release + } + recovered := make(chan struct{}) + go func() { start(p); close(recovered) }() + waitNetworkShutdownSignal(t, entered, "recovery reset seam") + closed := make(chan struct{}) + go func() { p.closeNetMonitor(); close(closed) }() + waitNetworkShutdownSignal(t, canceled, "recovery cancellation") + assertNetworkShutdownBlocked(t, closed) + if p.recoveryOwnsState(p.recoveryGen.Load()) { + t.Fatal("shutdown recovery still owns permission to reapply DNS") + } + unblock() + waitNetworkShutdownSignal(t, closed, "recovery drain") + waitNetworkShutdownSignal(t, recovered, "recovery return") + p.handleRecovery(RecoveryReasonRegularFailure) + p.handleRecovery(RecoveryReasonNetworkChange) + if resets.Load() != 1 || p.recoveryRunning.Load() || p.recoveryBypass.Load() || p.recoveryCancel != nil { + t.Fatalf("recovery survived close: resets=%d running=%v bypass=%v", resets.Load(), p.recoveryRunning.Load(), p.recoveryBypass.Load()) + } + ends := jsonLogEvents(t, logs, "Recovery end") + if len(ends) != 1 { + t.Fatalf("recovery end events: got %d, want 1", len(ends)) + } + wantField(t, ends[0], "outcome", recoveryOutcomeCanceled) + wantField(t, ends[0], "bypass_active", false) +} + +func TestNetworkShutdownCancelsIndefiniteRecoveryWait(t *testing.T) { + guardNetworkShutdownHostMutations(t) + oldReset, oldIntercept := recoveryResetDNSFn, dnsIntercept + dnsIntercept = false + defer func() { recoveryResetDNSFn, dnsIntercept = oldReset, oldIntercept }() + p := &prog{cfg: &ctrld.Config{}} + entered := make(chan struct{}) + recoveryResetDNSFn = func(*prog, bool, bool) { close(entered) } + recovered := make(chan struct{}) + // An empty upstream map can never recover. Shutdown must cancel that wait + // without waiting on a mutex held for the recovery's entire lifetime. + go func() { p.handleRecovery(RecoveryReasonRegularFailure); close(recovered) }() + waitNetworkShutdownSignal(t, entered, "recovery reset seam") + closed := make(chan struct{}) + go func() { p.closeNetMonitor(); close(closed) }() + waitNetworkShutdownSignal(t, closed, "canceled indefinite recovery") + waitNetworkShutdownSignal(t, recovered, "recovery return") +} + +func TestNetworkShutdownStopsDeferredRecovery(t *testing.T) { + guardNetworkShutdownHostMutations(t) + p := &prog{} + p.debounceRecovery(p.networkAcceptedGen.Load()) + p.recoveryDebounceMu.Lock() + timer := p.recoveryDebounceTimer + p.recoveryDebounceMu.Unlock() + if timer == nil { + t.Fatal("debounce did not schedule recovery") + } + p.closeNetMonitor() + if timer.Stop() { + t.Fatal("shutdown did not stop pending debounce timer") + } + // Model the already-fired timer's call to handleRecovery, which Stop + // cannot retract, then a callback trying to create another timer. + p.handleRecovery(RecoveryReasonNetworkChange) + p.debounceRecovery(p.networkAcceptedGen.Load()) + if p.recoveryDebounceTimer != nil || p.recoveryGen.Load() != 0 { + t.Fatal("deferred recovery restarted after shutdown") + } +} + +func TestNetworkShutdownRejectsRecoveryAdmittedBeforeClose(t *testing.T) { + p := &prog{} + // Pause between handleRecovery's activity admission and beginRecovery. + if !p.beginNetworkActivity() { + t.Fatal("initial admission failed") + } + closed := make(chan struct{}) + go func() { p.closeNetMonitor(); close(closed) }() + waitNetworkShutdownGate(t, p) + _, _, _, ok := p.beginRecovery(RecoveryReasonNetworkChange) + p.netMonitorWG.Done() + waitNetworkShutdownSignal(t, closed, "admitted recovery drain") + if ok || p.recoveryCancel != nil || p.recoveryRunning.Load() { + t.Fatal("recovery published a new owner after shutdown cancellation") + } +} + +type shutdownTestMonitor struct { + callback netmon.ChangeFunc + starts, closes int +} + +func (m *shutdownTestMonitor) Start() { m.starts++ } +func (m *shutdownTestMonitor) Close() error { m.closes++; return nil } +func (m *shutdownTestMonitor) RegisterChangeCallback(fn netmon.ChangeFunc) func() { + m.callback = fn + return func() {} +} +func (*shutdownTestMonitor) IsMajorChangeFrom(*netmon.State, *netmon.State) bool { + panic("closed callback reached network discovery") +} + +func TestNetworkMonitorProductionPublicationAndCallbackFence(t *testing.T) { + old := newNetworkChangeMonitorFn + t.Cleanup(func() { newNetworkChangeMonitorFn = old }) + for _, late := range []bool{false, true} { + p := &prog{} + m := &shutdownTestMonitor{} + newNetworkChangeMonitorFn = func(func(string, ...any)) (networkChangeMonitor, error) { + if late { + p.closeNetMonitor() + } + return m, nil + } + if err := p.monitorNetworkChanges(); err != nil { + t.Fatal(err) + } + p.closeNetMonitor() + if m.closes != 1 { + t.Fatalf("monitor closed %d times", m.closes) + } + wantStarts := 1 + if late { + wantStarts = 0 + } + if m.starts != wantStarts { + t.Fatalf("monitor starts=%d want=%d", m.starts, wantStarts) + } + // Exercise the callback captured from the actual production registration. + // Removing its wrapper reaches the panic above, before any OS mutation. + m.callback(&netmon.ChangeDelta{}) + } +} + +func TestFinishRunDrainsNetworkJournalOnAbort(t *testing.T) { + captureDebugMainLog(t) + stubHeaderSnapshotSources(t) + p := &prog{ + cfg: &ctrld.Config{}, stopCh: make(chan struct{}), + runAbortCh: make(chan struct{}), runDone: make(chan struct{}), + dnsWatcherStopCh: make(chan struct{}), + } + p.newNetworkJournal() + entered, release := make(chan struct{}), make(chan struct{}) + var once sync.Once + unblock := func() { once.Do(func() { close(release) }) } + defer unblock() + // The poll is a synthetic first platform boundary, even on Darwin. + p.dnsConfig = newDNSConfigPoller(func() ([]dnsResolverEntry, error) { + close(entered) + <-release + return nil, nil + }) + p.waitNetworkJournal = p.startNetworkJournal() + waitNetworkShutdownSignal(t, entered, "journal poll") + close(p.runDone) + done := make(chan struct{}) + go func() { p.finishRun(); close(done) }() + waitNetworkShutdownSignal(t, p.runAbortCh, "run abort") + assertNetworkShutdownBlocked(t, done) + unblock() + waitNetworkShutdownSignal(t, done, "journal drain on abort") + if stopRequested(p.stopCh) { + t.Fatal("finishRun closed the caller-owned stop channel") + } +} + +func TestNetworkShutdownDrainsBeforeOSRestoration(t *testing.T) { + p := &prog{cfg: &ctrld.Config{}, dnsWatcherStopCh: make(chan struct{})} + entered, release, restored := make(chan struct{}), make(chan struct{}), make(chan struct{}) + p.onStopped = []func(){func() { close(restored) }} + go p.networkChangeCallback(func(*netmon.ChangeDelta) { close(entered); <-release })(nil) + waitNetworkShutdownSignal(t, entered, "callback entry") + done := make(chan struct{}) + go func() { _ = p.restoreOSState(); close(done) }() + waitNetworkShutdownGate(t, p) + assertNetworkShutdownBlocked(t, restored) + close(release) + waitNetworkShutdownSignal(t, done, "restoration") + waitNetworkShutdownSignal(t, restored, "restore hook") +} diff --git a/cmd/cli/network_transition.go b/cmd/cli/network_transition.go index 711e592c..fb7825d8 100644 --- a/cmd/cli/network_transition.go +++ b/cmd/cli/network_transition.go @@ -17,6 +17,7 @@ import ( ) type networkChangeMonitor interface { + Close() error RegisterChangeCallback(netmon.ChangeFunc) func() IsMajorChangeFrom(*netmon.State, *netmon.State) bool Start() diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 8cf4a2d8..b77d6c65 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -113,6 +113,7 @@ type prog struct { apiForceReloadCh chan struct{} apiForceReloadGroup singleflight.Group logConn net.Conn + logConnMu sync.Mutex cs *controlServer csSetDnsDone chan struct{} csSetDnsOk bool @@ -321,6 +322,111 @@ type prog struct { onStartedDone chan struct{} onStarted []func() onStopped []func() + + netMonitorMu sync.Mutex + netMonitor networkChangeMonitor + netMonitorClosed bool + netMonitorStopCh chan struct{} + restoreOnce sync.Once + restoreErr error + releaseOnce sync.Once + runDone chan struct{} + runAbortCh chan struct{} + listenerWg sync.WaitGroup + apiReloadWg sync.WaitGroup + osStateMu sync.Mutex + osStateRestored bool + + // netMonitorWG tracks admitted callbacks and their recovery work. Admission + // is serialized with shutdown under netMonitorMu; no Add races with Wait. + netMonitorWG sync.WaitGroup + netMonitorCloseOnce sync.Once + // Published by the joined initializer; read only after runDone. + waitNetworkJournal func() +} + +// setNetMonitor publishes mon as the active network monitor, closing any +// predecessor. It reports false if shutdown already ran, in which case the +// caller owns mon and must not start it. +func (p *prog) setNetMonitor(mon networkChangeMonitor) bool { + p.netMonitorMu.Lock() + defer p.netMonitorMu.Unlock() + if p.netMonitorClosed { + return false + } + old := p.netMonitor + p.netMonitor = mon + if old != nil { + _ = old.Close() + } + return true +} + +// beginNetworkActivity admits a callback or recovery before shutdown. Every +// successful admission must be paired with netMonitorWG.Done. +func (p *prog) beginNetworkActivity() bool { + p.netMonitorMu.Lock() + defer p.netMonitorMu.Unlock() + if p.netMonitorClosed { + return false + } + p.netMonitorWG.Add(1) + return true +} + +func (p *prog) networkActivityClosed() bool { + p.netMonitorMu.Lock() + defer p.netMonitorMu.Unlock() + return p.netMonitorClosed +} + +// networkActivityDone cancels admitted work before service Stop closes stopCh. +func (p *prog) networkActivityDone() <-chan struct{} { + p.netMonitorMu.Lock() + defer p.netMonitorMu.Unlock() + if p.netMonitorStopCh == nil { + p.netMonitorStopCh = make(chan struct{}) + if p.netMonitorClosed { + close(p.netMonitorStopCh) + } + } + return p.netMonitorStopCh +} + +// closeNetMonitor fences publications and callbacks, cancels recovery, and +// drains admitted work before OS state restoration. netmon.Close alone does +// not join change callbacks. Never hold an admission/state mutex while waiting: +// callbacks and canceled recoveries need those mutexes to finish. +func (p *prog) closeNetMonitor() { + p.netMonitorCloseOnce.Do(func() { + p.netMonitorMu.Lock() + mon := p.netMonitor + p.netMonitor = nil + p.netMonitorClosed = true + if p.netMonitorStopCh != nil { + close(p.netMonitorStopCh) + } + p.netMonitorMu.Unlock() + + p.recoveryDebounceMu.Lock() + if p.recoveryDebounceTimer != nil { + p.recoveryDebounceTimer.Stop() + p.recoveryDebounceTimer = nil + } + p.recoveryDebounceMu.Unlock() + + p.recoveryCancelMu.Lock() + if p.recoveryCancel != nil { + p.recoveryCancel() + } + p.recoveryCancelMu.Unlock() + + if mon != nil { + _ = mon.Close() + mainLog.Load().Debug().Msg("network monitor stopped") + } + p.netMonitorWG.Wait() + }) } func (p *prog) Start(s service.Service) error { @@ -330,10 +436,11 @@ func (p *prog) Start(s service.Service) error { // runWait runs ctrld components, waiting for signal to reload. func (p *prog) runWait() { - p.mu.Lock() - p.cfg = &cfg - p.mu.Unlock() + if p.runDone != nil { + defer close(p.runDone) + } reloadSigCh := make(chan os.Signal, 1) + defer stopNotifyReloadSigCh(reloadSigCh) notifyReloadSigCh(reloadSigCh) reload := false @@ -357,6 +464,11 @@ func (p *prog) runWait() { newCfg = apiCfg case <-p.stopCh: close(reloadCh) + <-done + return + case <-p.runAbortCh: + close(reloadCh) + <-done return } @@ -436,6 +548,7 @@ func (p *prog) runWait() { p.setupUpstream(newCfg) p.mu.Lock() + oldUpstreams := p.cfg.Upstream *p.cfg = *newCfg // In DNS-intercept mode on macOS, the DNS listener is bound once at startup and is // NOT re-bound on reload (see prog.run: serveDNS is started only when !reload). When @@ -451,6 +564,7 @@ func (p *prog) runWait() { } p.mu.Unlock() + closeReplacedUpstreams(oldUpstreams, newCfg.Upstream) // The header names the mode, the listeners, and the upstreams, so the // open files take the values of the config that now runs. p.refreshLogHeader() @@ -526,13 +640,42 @@ func (p *prog) postRun() { } } +var fetchResolverConfigForReload = controld.FetchResolverConfig + +// startAPIConfigReload starts one program-lifetime worker, not one per reload. +func (p *prog) startAPIConfigReload() { + p.apiReloadWg.Add(1) + go func() { + defer p.apiReloadWg.Done() + p.apiConfigReload() + }() +} + // apiConfigReload calls API to check for latest config update then reload ctrld if necessary. func (p *prog) apiConfigReload() { if cdUID == "" { return } + ctx, cancel := context.WithCancel(context.Background()) + watcherDone := make(chan struct{}) + defer func() { + cancel() + <-watcherDone + }() + go func() { + defer close(watcherDone) + select { + case <-p.stopCh: + case <-p.runAbortCh: + case <-ctx.Done(): + } + cancel() + }() - ticker := time.NewTicker(timeDurationOrDefault(p.cfg.Service.RefetchTime, 3600) * time.Second) + p.mu.Lock() + refetchInterval := timeDurationOrDefault(p.cfg.Service.RefetchTime, 3600) * time.Second + p.mu.Unlock() + ticker := time.NewTicker(refetchInterval) defer ticker.Stop() logger := mainLog.Load().With().Str("mode", "api-reload").Logger() @@ -550,12 +693,18 @@ func (p *prog) apiConfigReload() { } doReloadApiConfig := func(forced bool, logger zerolog.Logger) { + if ctx.Err() != nil { + return + } req := &controld.ResolverConfigRequest{ RawUID: cdUID, Version: rootCmd.Version, - Metadata: ctrld.SystemMetadataRuntime(context.Background()), + Metadata: ctrld.SystemMetadataRuntime(ctx), + } + resolverConfig, err := fetchResolverConfigForReload(ctx, req, cdDev) + if ctx.Err() != nil { + return } - resolverConfig, err := controld.FetchResolverConfig(context.Background(), req, cdDev) selfUninstallCheck(err, p, logger) if err != nil { logger.Warn().Err(err).Msg("could not fetch resolver config") @@ -608,7 +757,10 @@ func (p *prog) apiConfigReload() { if !noInternalDomainsChanged { logger.Info().Msg("internal domain changes detected, reloading...") } - p.apiReloadCh <- nil + select { + case p.apiReloadCh <- nil: + case <-ctx.Done(): + } return } @@ -623,13 +775,16 @@ func (p *prog) apiConfigReload() { } if cfgErr != nil { logger.Warn().Err(err).Msg("skipping invalid custom config") - if _, err := controld.UpdateCustomLastFailed(context.Background(), cdUID, rootCmd.Version, cdDev, true); err != nil { + if _, err := controld.UpdateCustomLastFailed(ctx, cdUID, rootCmd.Version, cdDev, true); err != nil { logger.Error().Err(err).Msg("could not mark custom last update failed") } return } logger.Debug().Msg("custom config changes detected, reloading...") - p.apiReloadCh <- cfg + select { + case p.apiReloadCh <- cfg: + case <-ctx.Done(): + } } else { logger.Debug().Msg("custom config does not change") } @@ -640,7 +795,7 @@ func (p *prog) apiConfigReload() { doReloadApiConfig(true, logger.With().Bool("forced", true).Logger()) case <-ticker.C: doReloadApiConfig(false, logger) - case <-p.stopCh: + case <-ctx.Done(): return } } @@ -710,8 +865,21 @@ func (p *prog) setupUpstream(cfg *ctrld.Config) { // out the full timeout. A terminal provisioning path calls this right // before failing. func (p *prog) notifyExitToLogServer() { - if p.logConn != nil { - _, _ = p.logConn.Write([]byte(msgExit)) + p.logConnMu.Lock() + conn := p.logConn + p.logConnMu.Unlock() + if conn != nil { + _, _ = conn.Write([]byte(msgExit)) + } +} + +func (p *prog) closeLogConn() { + p.logConnMu.Lock() + conn := p.logConn + p.logConn = nil + p.logConnMu.Unlock() + if conn != nil { + _ = conn.Close() } } @@ -734,7 +902,16 @@ func (p *prog) reportServeDNSFailure(listenerNum string, err error) { // so all listeners could be terminated and re-spawned again. func (p *prog) run(reload bool, reloadCh chan struct{}) { // Wait the caller to signal that we can do our logic. - <-p.waitCh + select { + case <-p.waitCh: + case <-p.stopCh: + return + case <-p.runAbortCh: + return + } + if stopRequested(p.stopCh) || stopRequested(p.runAbortCh) { + return + } if !reload { p.preRun() } @@ -778,7 +955,6 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { } var wg sync.WaitGroup - wg.Add(len(p.cfg.Listener)) for _, nc := range p.cfg.Network { for _, cidr := range nc.Cidrs { @@ -809,7 +985,21 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { // context for managing spawn goroutines. ctx, cancelFunc := context.WithCancel(context.Background()) - defer cancelFunc() + defer func() { + cancelFunc() + wg.Wait() + }() + wg.Add(1) + go func() { + defer wg.Done() + select { + case <-p.stopCh: + case <-p.runAbortCh: + case <-reloadCh: + case <-ctx.Done(): + } + cancelFunc() + }() // Newer versions of android and iOS denies permission which breaks connectivity. if !isMobile() && !reload { @@ -822,13 +1012,19 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { if !reload { p.newNetworkJournal() - go p.startNetworkMonitorAndJournal() + wg.Add(1) + go func() { + defer wg.Done() + p.waitNetworkJournal = p.startNetworkMonitorAndJournal() + }() } for listenerNum := range p.cfg.Listener { p.cfg.Listener[listenerNum].Init() if !reload { + p.listenerWg.Add(1) go func(listenerNum string) { + defer p.listenerWg.Done() listenerConfig := p.cfg.Listener[listenerNum] upstreamConfig := p.cfg.Upstream[listenerNum] if upstreamConfig == nil { @@ -836,31 +1032,30 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { } addr := net.JoinHostPort(listenerConfig.IP, strconv.Itoa(listenerConfig.Port)) mainLog.Load().Info().Msgf("starting DNS server on listener.%s: %s", listenerNum, addr) - if err := p.serveDNS(listenerNum); err != nil { + if err := serveDNSFn(p, listenerNum); err != nil { p.reportServeDNSFailure(listenerNum, err) } mainLog.Load().Debug().Msgf("end of serveDNS listener.%s: %s", listenerNum, addr) }(listenerNum) } - go func() { - defer func() { - cancelFunc() - wg.Done() - }() - select { - case <-p.stopCh: - case <-ctx.Done(): - case <-reloadCh: - } - }() } if !reload { for i := 0; i < numListeners; i++ { - <-p.started + select { + case <-p.started: + case <-p.stopCh: + return + case <-p.runAbortCh: + return + } } - for _, f := range p.onStarted { - f() + if !p.startOSState(func() { + for _, f := range p.onStarted { + f() + } + }) { + return } } @@ -886,11 +1081,9 @@ func (p *prog) run(reload bool, reloadCh chan struct{}) { // Stop writing log to unix socket. consoleWriter.Out = os.Stdout p.initLogging(false) - if p.logConn != nil { - _ = p.logConn.Close() - } - go p.apiConfigReload() - p.postRun() + p.closeLogConn() + p.startAPIConfigReload() + p.startOSState(func() { postRunFn(p) }) } wg.Wait() } @@ -938,7 +1131,7 @@ func (p *prog) newNetworkJournal() { // journal. The trackers do not depend on the monitor, so a monitor that cannot // start leaves them running. It passes on the wait function of the journal. func (p *prog) startNetworkMonitorAndJournal() func() { - if err := p.monitorNetworkChanges(); err != nil { + if err := monitorNetworkChangesFn(p); err != nil { mainLog.Load().Error().Err(err).Msg("Failed to start network monitoring") } return p.startNetworkJournal() @@ -949,12 +1142,21 @@ func (p *prog) startNetworkMonitorAndJournal() func() { // table of each moment after an incident. It returns a function that waits for // both loops, so a caller that closed the stop channel knows when they ended. func (p *prog) startNetworkJournal() func() { - healthDone := p.health.startLoop(p.stopCh, func() (int, bool) { + // Journal trackers persist across reloads, but not a stop or startup abort. + stop := make(chan struct{}) + go func() { + select { + case <-p.stopCh: + case <-p.runAbortCh: + } + close(stop) + }() + healthDone := p.health.startLoop(stop, func() (int, bool) { return p.upstreamMonitorNow().countDownExcept(isInternalDomainUpstream), p.recoveryBypass.Load() }) var configDone <-chan struct{} if p.dnsConfig != nil { - configDone = p.dnsConfig.startLoop(p.stopCh, p.logDNSConfigChanges) + configDone = p.dnsConfig.startLoop(stop, p.logDNSConfigChanges) } p.logNetworkSnapshot("start") return func() { @@ -1012,22 +1214,124 @@ func (p *prog) metricsEnabled() bool { return p.cfg.Service.MetricsQueryStats || p.cfg.Service.MetricsListener != "" } -func (p *prog) Stop(s service.Service) error { - p.stopDnsWatchers() - mainLog.Load().Debug().Msg("dns watchers stopped") - for _, f := range p.onStopped { - f() +// finishRun also cancels workers waiting for startup when preflight returns early. +// The caller's stop channel belongs to the mobile controller, not to this cleanup. +func (p *prog) finishRun() { + // Keep the files open through final diagnostics, including mobile exits + // that never invoke the service Stop callback. + defer p.closeInternalLogs() + close(p.runAbortCh) + select { + case <-p.runDone: + case <-time.After(shutdownTimeout): + mainLog.Load().Warn().Msg("timeout waiting for ctrld components to stop") + // A timeout is not proof that workers stopped. Never release resources + // still owned by the run, or allow a mobile restart to overlap it. + <-p.runDone + } + // Listeners live across config reloads, so they belong to prog rather + // than the per-run wait group. runDone closes after their final Add. + p.apiReloadWg.Wait() + p.listenerWg.Wait() + if p.waitNetworkJournal != nil { + p.waitNetworkJournal() + } + if err := p.shutdown(); err != nil { + mainLog.Load().Warn().Err(err).Msg("error during shutdown") } - mainLog.Load().Debug().Msg("finish running onStopped functions") +} + +// startOSState serializes startup's OS mutations with restoration. A stop +// either restores an already-completed action or prevents it from starting. +func (p *prog) startOSState(f func()) bool { + p.osStateMu.Lock() + defer p.osStateMu.Unlock() + if p.osStateRestored || stopRequested(p.stopCh) || stopRequested(p.runAbortCh) { + return false + } + f() + return true +} + +// restoreOSState puts back what ctrld changed outside the process: the DNS +// settings, the router configuration and the allocated listener IPs. It runs +// while the listeners are still serving, so the OS is never left pointing at a +// resolver that is already gone. Safe to call multiple times. +func (p *prog) restoreOSState() error { + p.restoreOnce.Do(func() { + p.osStateMu.Lock() + p.osStateRestored = true + p.osStateMu.Unlock() + p.closeNetMonitor() + p.stopDnsWatchers() + mainLog.Load().Debug().Msg("dns watchers stopped") + for _, f := range p.onStopped { + f() + } + mainLog.Load().Debug().Msg("finish running onStopped functions") + if derr := p.deAllocateIP(); derr != nil { + mainLog.Load().Error().Err(derr).Msg("de-allocate ip failed") + p.restoreErr = derr + } + }) + return p.restoreErr +} + +// releaseResources releases the control server and upstream transports. It must run after +// the listeners stopped, since retiring an upstream a listener still answers +// queries on would fail those queries. Safe to call multiple times. +func (p *prog) releaseResources() { + p.releaseOnce.Do(func() { + if p.cs != nil { + if cerr := p.cs.stop(); cerr != nil { + mainLog.Load().Warn().Err(cerr).Msg("could not stop control server") + } + } + p.closeLogConn() + p.mu.Lock() + upstreams := p.cfg.Upstream + p.mu.Unlock() + closeReplacedUpstreams(upstreams, nil) + }) +} + +// shutdown runs the full teardown, in the order both halves require. It is +// called once the listeners stopped, on every stop path including the mobile +// one where the OS never terminates the process. +func (p *prog) shutdown() error { + err := p.restoreOSState() + p.releaseResources() + return err +} + +// closeReplacedUpstreams releases the transports of upstreams that cur no longer +// refers to, or all of them when cur is nil. Requests in flight on them fail +// fast and are retried, the same way a re-bootstrap treats connections it +// replaces. +func closeReplacedUpstreams(old, cur map[string]*ctrld.UpstreamConfig) { + inUse := make(map[*ctrld.UpstreamConfig]struct{}, len(cur)) + for _, uc := range cur { + inUse[uc] = struct{}{} + } + for _, uc := range old { + if uc == nil { + continue + } + if _, ok := inUse[uc]; !ok { + uc.CloseTransports() + } + } +} + +func (p *prog) Stop(s service.Service) error { defer func() { mainLog.Load().Info().Msg("Service stopped") - // The files take the last line of this run before they close. p.closeInternalLogs() }() - if err := p.deAllocateIP(); err != nil { - mainLog.Load().Error().Err(err).Msg("de-allocate ip failed") - return err - } + // Only the OS level state is restored here. The listeners are still serving + // at this point, so releasing the upstreams they answer queries on has to + // wait until run observes stopCh and they have stopped. + err := p.restoreOSState() if deactivationPinSet() { select { case <-p.pinCodeValidCh: @@ -1052,7 +1356,7 @@ func (p *prog) Stop(s service.Service) error { } } close(p.stopCh) - return nil + return err } func (p *prog) stopDnsWatchers() { @@ -1080,7 +1384,7 @@ func (p *prog) deAllocateIP() error { return nil } for _, lc := range p.cfg.Listener { - if err := deAllocateIP(lc.IP); err != nil { + if err := deAllocateIPFn(lc.IP); err != nil { return err } } @@ -1095,6 +1399,10 @@ func (p *prog) deAllocateIP() error { // NRPT rule, so a test of what happens *after* it fails must not be the thing that // runs it. var ( + deAllocateIPFn = deAllocateIP + serveDNSFn = (*prog).serveDNS + monitorNetworkChangesFn = (*prog).monitorNetworkChanges + postRunFn = (*prog).postRun localResolverIPFn = router.LocalResolverIP startDNSInterceptFn = (*prog).startDNSIntercept ensureInterceptDNSTargetFn = (*prog).ensureInterceptDNSTarget diff --git a/cmd/cli/recovery_state.go b/cmd/cli/recovery_state.go index 0eae9cdc..090e7e76 100644 --- a/cmd/cli/recovery_state.go +++ b/cmd/cli/recovery_state.go @@ -9,6 +9,12 @@ func (p *prog) beginRecovery(reason RecoveryReason) (ctx context.Context, gen ui p.recoveryCancelMu.Lock() defer p.recoveryCancelMu.Unlock() + // Close publishes the gate before taking recoveryCancelMu to cancel the + // current owner. Thus an admitted worker either publishes its cancel first + // or observes shutdown here; it cannot create an uncancellable late owner. + if p.networkActivityClosed() { + return nil, 0, false, false + } if reason != RecoveryReasonNetworkChange && p.recoveryCancel != nil { return nil, 0, false, false } @@ -28,7 +34,7 @@ func (p *prog) beginRecovery(reason RecoveryReason) (ctx context.Context, gen ui func (p *prog) recoveryOwnsState(gen uint64) bool { p.recoveryCancelMu.Lock() defer p.recoveryCancelMu.Unlock() - return p.recoveryGen.Load() == gen && p.recoveryCancel != nil + return !p.networkActivityClosed() && p.recoveryGen.Load() == gen && p.recoveryCancel != nil } func systemNameserversForInterceptRetry() []string { diff --git a/cmd/cli/reload_others.go b/cmd/cli/reload_others.go index 0977af90..d72eab46 100644 --- a/cmd/cli/reload_others.go +++ b/cmd/cli/reload_others.go @@ -12,6 +12,10 @@ func notifyReloadSigCh(ch chan os.Signal) { signal.Notify(ch, syscall.SIGUSR1) } +func stopNotifyReloadSigCh(ch chan os.Signal) { + signal.Stop(ch) +} + func (p *prog) sendReloadSignal() error { return syscall.Kill(syscall.Getpid(), syscall.SIGUSR1) } diff --git a/cmd/cli/reload_windows.go b/cmd/cli/reload_windows.go index 0e817e46..8d0054ed 100644 --- a/cmd/cli/reload_windows.go +++ b/cmd/cli/reload_windows.go @@ -8,6 +8,8 @@ import ( func notifyReloadSigCh(ch chan os.Signal) {} +func stopNotifyReloadSigCh(ch chan os.Signal) {} + func (p *prog) sendReloadSignal() error { select { case p.reloadCh <- struct{}{}: diff --git a/cmd/cli/restore_lock_test.go b/cmd/cli/restore_lock_test.go new file mode 100644 index 00000000..e80672d8 --- /dev/null +++ b/cmd/cli/restore_lock_test.go @@ -0,0 +1,51 @@ +package cli + +import ( + "sync/atomic" + "testing" + + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +func TestRestoreOSStateReleasesMutexBeforeCallbackDrain(t *testing.T) { + p := &prog{cfg: &ctrld.Config{}, dnsWatcherStopCh: make(chan struct{})} + entered, resume, callbackDone := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var startupCalls, restoreCalls atomic.Int32 + p.onStopped = []func(){func() { + select { + case <-callbackDone: + default: + t.Error("OS restoration ran before the callback drained") + } + restoreCalls.Add(1) + }} + go p.networkChangeCallback(func(*netmon.ChangeDelta) { + close(entered) + <-resume + // This callback was admitted before shutdown. It must be able to take + // osStateMu and observe the closed startup gate while shutdown joins it. + if p.startOSState(func() { startupCalls.Add(1) }) { + t.Error("startup was permitted after restoration began") + } + close(callbackDone) + })(nil) + waitNetworkShutdownSignal(t, entered, "callback entry") + restored := make(chan struct{}) + go func() { + if err := p.restoreOSState(); err != nil { + t.Error(err) + } + close(restored) + }() + waitNetworkShutdownGate(t, p) + close(resume) + waitNetworkShutdownSignal(t, callbackDone, "callback acquiring osStateMu") + waitNetworkShutdownSignal(t, restored, "OS restoration") + if err := p.restoreOSState(); err != nil { + t.Fatal(err) + } + if startupCalls.Load() != 0 || restoreCalls.Load() != 1 { + t.Fatalf("startup calls=%d, restore calls=%d", startupCalls.Load(), restoreCalls.Load()) + } +} diff --git a/cmd/cli/shutdown_test.go b/cmd/cli/shutdown_test.go new file mode 100644 index 00000000..eae8ae9c --- /dev/null +++ b/cmd/cli/shutdown_test.go @@ -0,0 +1,164 @@ +package cli + +import ( + "bufio" + "context" + "errors" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" + "github.com/Control-D-Inc/ctrld/internal/router" +) + +func TestRunWaitStopsBeforeStartup(t *testing.T) { + p := &prog{ + waitCh: make(chan struct{}), + stopCh: make(chan struct{}), + runDone: make(chan struct{}), + } + require.NoError(t, p.Start(nil)) + close(p.stopCh) + select { + case <-p.runDone: + case <-time.After(time.Second): + t.Fatal("stop did not terminate workers waiting for startup") + } +} + +func TestRunDoesNotStartAfterStop(t *testing.T) { + for i := 0; i < 100; i++ { + p := &prog{waitCh: make(chan struct{}), stopCh: make(chan struct{})} + close(p.waitCh) + close(p.stopCh) + // cfg is deliberately nil: reaching component startup is a failure. + p.run(false, make(chan struct{})) + } +} + +type startupTestRouter struct { + router.Router + preRunCalls int +} + +func (r *startupTestRouter) PreRun() error { + r.preRunCalls++ + return nil +} + +func TestRunMobilePreflightCleanup(t *testing.T) { + const childEnv = "CTRLD_TEST_MOBILE_PREFLIGHT" + mode := os.Getenv(childEnv) + if mode == "" { + for _, mode := range []string{"stop", "failure"} { + t.Run(mode, func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 8*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestRunMobilePreflightCleanup$", "-test.v") + cmd.Env = append(os.Environ(), childEnv+"="+mode) + out, err := cmd.CombinedOutput() + require.NoError(t, err, "%s", out) + }) + } + return + } + + // Keep the real RunMobile -> run -> Start -> runWait path, but replace the + // platform/router and API boundaries before any host setup can happen. + isMobile = func() bool { return true } + networkUp = func() bool { return true } + r := &startupTestRouter{} + newRouter = func(*ctrld.Config, bool) router.Router { return r } + dir := t.TempDir() + configPath = filepath.Join(dir, "ctrld.toml") + require.NoError(t, os.WriteFile(configPath, []byte("[service]\n"), 0600)) + want := errors.New("preflight rejected") + for i := 0; i < 3; i++ { + stopCh := make(chan struct{}) + fetchCalls, exits := 0, 0 + processCDFlagsFn = func(ctx context.Context, _ *ctrld.Config) (*controld.ResolverConfig, error) { + fetchCalls++ + require.Eventually(t, startupWorkersRunning, time.Second, time.Millisecond) + if mode == "stop" { + close(stopCh) + <-ctx.Done() + return nil, ctx.Err() + } + return nil, want + } + callback := AppCallback{Exit: func(err string) { + exits++ + require.Equal(t, want.Error(), err) + }} + RunMobile(&AppConfig{CdUID: "testuid", HomeDir: dir, UpstreamProto: "doh", LogPath: filepath.Join(dir, "ctrld.log")}, &callback, stopCh) + require.Equal(t, 1, fetchCalls) + if mode == "failure" { + require.Equal(t, 1, exits) + require.False(t, stopRequested(stopCh), "cleanup must not close the controller-owned channel") + + } else { + require.Zero(t, exits) + } + require.False(t, startupWorkersRunning(), "RunMobile returned before its workers stopped") + if mode == "failure" { + close(stopCh) // A later Controller.Stop must remain safe. + } + } + require.Equal(t, 3, r.preRunCalls, "the real router must never be reached") +} + +func startupWorkersRunning() bool { + stack := make([]byte, 1<<20) + n := runtime.Stack(stack, true) + return strings.Contains(string(stack[:n]), "(*prog).runWait(") || strings.Contains(string(stack[:n]), "(*prog).run(") +} + +func TestControlServerStopOnlyUnlinksOwnedListener(t *testing.T) { + // Keep the Unix socket path below macOS's limit, including its long TMPDIR. + dir, err := os.MkdirTemp("", "ctrld-") + require.NoError(t, err) + t.Cleanup(func() { _ = os.RemoveAll(dir) }) + path := filepath.Join(dir, "control.sock") + owner, err := newControlServer(path) + require.NoError(t, err) + require.NoError(t, owner.start()) + t.Cleanup(func() { _ = owner.stop() }) + unstarted, err := newControlServer(path) + require.NoError(t, err) + require.NoError(t, unstarted.stop()) + c, err := net.Dial("unix", path) + require.NoError(t, err, "stopping before startup must preserve someone else's socket") + require.NoError(t, c.SetDeadline(time.Now().Add(time.Second))) + _, err = io.WriteString(c, "GET / HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n") + require.NoError(t, err) + resp, err := http.ReadResponse(bufio.NewReader(c), nil) + require.NoError(t, err) + require.NoError(t, resp.Body.Close()) + require.NoError(t, c.Close()) + require.NoError(t, owner.stop()) + _, err = os.Stat(path) + require.True(t, os.IsNotExist(err), "owned socket must be unlinked on shutdown: %v", err) +} + +func TestShutdownClosesStartupLogConnection(t *testing.T) { + c, peer := net.Pipe() + defer peer.Close() + p := &prog{cfg: &ctrld.Config{}, dnsWatcherStopCh: make(chan struct{}), logConn: c} + require.NoError(t, p.shutdown()) + require.NoError(t, p.shutdown()) + _ = peer.SetWriteDeadline(time.Now().Add(time.Second)) + _, err := peer.Write([]byte("x")) + require.ErrorIs(t, err, io.ErrClosedPipe) +} diff --git a/cmd/cli/upstream_retirement_test.go b/cmd/cli/upstream_retirement_test.go new file mode 100644 index 00000000..354fc8b5 --- /dev/null +++ b/cmd/cli/upstream_retirement_test.go @@ -0,0 +1,101 @@ +package cli + +import ( + "context" + "crypto/tls" + "crypto/x509" + "net" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/miekg/dns" + "github.com/stretchr/testify/require" + + "github.com/Control-D-Inc/ctrld" +) + +func TestCloseReplacedUpstreamsPreservesRetainedAndNew(t *testing.T) { + certServer := httptest.NewTLSServer(nil) + certificate := certServer.TLS.Certificates[0] + roots := x509.NewCertPool() + roots.AddCert(certServer.Certificate()) + certServer.Close() + + type upstream struct { + config *ctrld.UpstreamConfig + entered <-chan struct{} + release func() + } + newUpstream := func() upstream { + ln, err := tls.Listen("tcp", "127.0.0.1:0", &tls.Config{Certificates: []tls.Certificate{certificate}}) + require.NoError(t, err) + entered, release := make(chan struct{}), make(chan struct{}) + var enteredOnce, releaseOnce sync.Once + s := &dns.Server{Listener: ln, Net: "tcp-tls", Handler: dns.HandlerFunc(func(w dns.ResponseWriter, m *dns.Msg) { + enteredOnce.Do(func() { close(entered) }) + <-release + answer := new(dns.Msg) + answer.SetReply(m) + _ = w.WriteMsg(answer) + })} + done := make(chan error, 1) + go func() { done <- s.ActivateAndServe() }() + uc := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeDOT, Endpoint: ln.Addr().String(), BootstrapIP: "127.0.0.1"} + uc.Init() + uc.SetCertPool(roots) + unblock := func() { releaseOnce.Do(func() { close(release) }) } + t.Cleanup(func() { + unblock() + uc.CloseTransports() + _ = s.Shutdown() + _ = ln.Close() + select { + case <-done: + case <-time.After(time.Second): + t.Error("test DNS server did not stop") + } + }) + return upstream{uc, entered, unblock} + } + removed, retained, replacement := newUpstream(), newUpstream(), newUpstream() + query := func(u upstream) <-chan error { + result := make(chan error, 1) + go func() { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + r, err := ctrld.NewResolver(u.config) + if err == nil { + q := new(dns.Msg) + q.SetQuestion("retirement.example.", dns.TypeA) + _, err = r.Resolve(ctx, q) + } + result <- err + }() + return result + } + removedResult, retainedResult := query(removed), query(retained) + for _, ch := range []<-chan struct{}{removed.entered, retained.entered} { + select { + case <-ch: + case <-time.After(3 * time.Second): + t.Fatal("query did not reach local upstream") + } + } + closeReplacedUpstreams(map[string]*ctrld.UpstreamConfig{ + "0": removed.config, "1": retained.config, "nil": nil, + }, map[string]*ctrld.UpstreamConfig{ + "0": replacement.config, "renamed": retained.config, + }) + select { + case err := <-removedResult: + require.ErrorIs(t, err, net.ErrClosed) + case <-time.After(time.Second): + t.Fatal("removed upstream's active query was not closed") + } + retained.release() + require.NoError(t, <-retainedResult, "retained pointer must remain usable under a different key") + replacement.release() + require.NoError(t, <-query(replacement), "new upstream must not be retired with the old key") +} From 9db6de060f13e76b904552da4316381ecf64305d Mon Sep 17 00:00:00 2001 From: Dev Scribe Date: Fri, 18 Sep 2026 12:04:24 -0400 Subject: [PATCH 13/29] fix(mobile): serialize controller session teardown Co-authored-by: Ginder Singh --- cmd/ctrld_library/main.go | 100 +++++++++--- cmd/ctrld_library/main_test.go | 275 +++++++++++++++++++++++++++++++++ 2 files changed, 353 insertions(+), 22 deletions(-) create mode 100644 cmd/ctrld_library/main_test.go diff --git a/cmd/ctrld_library/main.go b/cmd/ctrld_library/main.go index b2e643db..8b8979c0 100644 --- a/cmd/ctrld_library/main.go +++ b/cmd/ctrld_library/main.go @@ -1,18 +1,32 @@ package ctrld_library import ( + "sync" + "time" + "github.com/Control-D-Inc/ctrld/cmd/cli" ) -// Controller holds global state +// stopTimeout bounds how long Stop waits for ctrld to release its resources. +const stopTimeout = 15 * time.Second + +var ( + runMobile = cli.RunMobile + checkDeactivationPin = cli.CheckDeactivationPin +) + +// Controller holds global state. Start, Stop and IsRunning are safe for +// concurrent use. AppCallback and Config must not be accessed concurrently. type Controller struct { + mu sync.Mutex + pinMu sync.Mutex // Serializes PIN checks, which use CLI global state. stopCh chan struct{} + doneCh chan struct{} AppCallback AppCallback Config cli.AppConfig } // NewController provides reference to global state to be managed by android vpn service and iOS network extension. -// reference is not safe for concurrent use. func NewController(appCallback AppCallback) *Controller { return &Controller{AppCallback: appCallback} } @@ -26,23 +40,36 @@ type AppCallback interface { } // Start configures utility with config.toml from provided directory. -// This function will block until Stop is called +// This function blocks until the run exits, including teardown after Stop. +// A call made while a run is active or stopping returns immediately. // Check port availability prior to calling it. func (c *Controller) Start(CdUID string, ProvisionID string, CustomHostname string, HomeDir string, UpstreamProto string, logLevel int, logPath string) { - if c.stopCh == nil { - c.stopCh = make(chan struct{}) - c.Config = cli.AppConfig{ - CdUID: CdUID, - ProvisionID: ProvisionID, - CustomHostname: CustomHostname, - HomeDir: HomeDir, - UpstreamProto: UpstreamProto, - Verbose: logLevel, - LogPath: logPath, - } - appCallback := mapCallback(c.AppCallback) - cli.RunMobile(&c.Config, &appCallback, c.stopCh) + c.mu.Lock() + if c.stopCh != nil { + c.mu.Unlock() + return + } + stopCh, doneCh := make(chan struct{}), make(chan struct{}) + c.stopCh, c.doneCh = stopCh, doneCh + c.Config = cli.AppConfig{ + CdUID: CdUID, + ProvisionID: ProvisionID, + CustomHostname: CustomHostname, + HomeDir: HomeDir, + UpstreamProto: UpstreamProto, + Verbose: logLevel, + LogPath: logPath, } + appCallback := mapCallback(c.AppCallback) + c.mu.Unlock() + defer func() { + c.mu.Lock() + defer c.mu.Unlock() + // Only the completed run releases the guard, even if Stop timed out. + close(doneCh) + c.stopCh, c.doneCh = nil, nil + }() + runMobile(&c.Config, &appCallback, stopCh) } // As workaround to avoid circular dependency between cli and ctrld_library module @@ -63,20 +90,49 @@ func mapCallback(callback AppCallback) cli.AppCallback { } } +// Stop requests shutdown after validating pin, unless restart bypasses the check. +// A zero result means the stop request was accepted, not that teardown finished. +// After cancellation it waits up to 15 seconds. Callers must check IsRunning +// after Stop: while it remains true, teardown still owns the session and Start +// will be rejected. PIN failure codes are returned unchanged. func (c *Controller) Stop(restart bool, pin int64) int { - var errorCode = 0 + // Keep this request bound to its original run while PIN validation blocks. + c.mu.Lock() + stopCh, doneCh := c.stopCh, c.doneCh + c.mu.Unlock() + // Force disconnect without checking pin. // In iOS restart is required if vpn detects no connectivity after network change. if !restart { - errorCode = cli.CheckDeactivationPin(pin, c.stopCh) + c.pinMu.Lock() + errorCode := checkDeactivationPin(pin, stopCh) + c.pinMu.Unlock() + if errorCode != 0 { + return errorCode + } } - if errorCode == 0 && c.stopCh != nil { - close(c.stopCh) - c.stopCh = nil + if stopCh != nil { + c.mu.Lock() + select { + case <-stopCh: + default: + close(stopCh) + } + c.mu.Unlock() + + timer := time.NewTimer(stopTimeout) + defer timer.Stop() + select { + case <-doneCh: + case <-timer.C: + } } - return errorCode + return 0 } +// IsRunning remains true until the run has finished releasing its resources. func (c *Controller) IsRunning() bool { + c.mu.Lock() + defer c.mu.Unlock() return c.stopCh != nil } diff --git a/cmd/ctrld_library/main_test.go b/cmd/ctrld_library/main_test.go new file mode 100644 index 00000000..3ccb5331 --- /dev/null +++ b/cmd/ctrld_library/main_test.go @@ -0,0 +1,275 @@ +package ctrld_library + +import ( + "runtime" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + "time" + + "github.com/Control-D-Inc/ctrld/cmd/cli" +) + +// Tests replace the first host-touching calls, not the Controller methods. +// They must not run in parallel because these seams are package globals. +func stubMobile(t *testing.T, run func(*cli.AppConfig, *cli.AppCallback, chan struct{}), check func(int64, chan struct{}) int) { + t.Helper() + oldRun, oldCheck := runMobile, checkDeactivationPin + runMobile, checkDeactivationPin = run, check + t.Cleanup(func() { runMobile, checkDeactivationPin = oldRun, oldCheck }) +} + +func startController(c *Controller) <-chan struct{} { + done := make(chan struct{}) + go func() { + c.Start("uid", "provision", "hostname", "home", "doh", 2, "log") + close(done) + }() + return done +} + +func TestControllerConcurrentLifecycle(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := NewController(nil) + var runs, active atomic.Int32 + var pinCalls int // Deliberately non-atomic: PIN checks must be serialized. + stubMobile(t, func(config *cli.AppConfig, _ *cli.AppCallback, stop chan struct{}) { + if active.Add(1) != 1 { + t.Error("overlapping mobile runs") + } + defer active.Add(-1) + runs.Add(1) + want := cli.AppConfig{CdUID: "uid", ProvisionID: "provision", CustomHostname: "hostname", HomeDir: "home", UpstreamProto: "doh", Verbose: 2, LogPath: "log"} + if *config != want { + t.Errorf("config = %+v, want %+v", *config, want) + } + <-stop + }, func(_ int64, _ chan struct{}) int { + pinCalls++ + return 0 + }) + + if c.IsRunning() || c.Stop(true, 0) != 0 { + t.Fatal("new controller must be stopped") + } + for cycle := 0; cycle < 25; cycle++ { + var starters sync.WaitGroup + for i := 0; i < 16; i++ { + starters.Add(1) + go func() { defer starters.Done(); <-startController(c) }() + } + synctest.Wait() + if !c.IsRunning() || runs.Load() != int32(cycle+1) { + t.Fatal("concurrent Starts did not claim exactly one run") + } + var stoppers sync.WaitGroup + for i := 0; i < 16; i++ { + stoppers.Add(1) + go func() { + defer stoppers.Done() + c.IsRunning() + if code := c.Stop(false, 1234); code != 0 { + t.Errorf("Stop = %d", code) + } + c.IsRunning() + }() + } + stoppers.Wait() + starters.Wait() + if c.IsRunning() || active.Load() != 0 { + t.Fatal("Stop returned before run completion") + } + } + if pinCalls != 25*16 { + t.Fatalf("PIN checks = %d, want %d", pinCalls, 25*16) + } + }) +} + +func TestControllerStopWaitsAndTimeoutKeepsGuard(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := NewController(nil) + release := make(chan struct{}) + var runs int + stubMobile(t, func(_ *cli.AppConfig, _ *cli.AppCallback, stop chan struct{}) { + runs++ + <-stop + <-release + }, func(int64, chan struct{}) int { t.Error("restart checked PIN"); return 1 }) + started := startController(c) + synctest.Wait() + stopped := make(chan struct{}) + before := time.Now() + go func() { + var stoppers sync.WaitGroup + for i := 0; i < 8; i++ { + stoppers.Add(1) + go func() { defer stoppers.Done(); c.Stop(true, 0) }() + } + stoppers.Wait() + close(stopped) + }() + synctest.Wait() + select { + case <-stopped: + t.Fatal("Stop did not wait for teardown") + default: + } + c.Start("", "", "", "", "", 0, "") + if !c.IsRunning() || runs != 1 { + t.Fatal("guard released during teardown") + } + <-stopped // synctest advances virtual time, not wall time. + if elapsed := time.Since(before); elapsed != stopTimeout { + t.Fatalf("Stop timeout = %v, want %v", elapsed, stopTimeout) + } + c.Start("", "", "", "", "", 0, "") + if !c.IsRunning() || runs != 1 { + t.Fatal("timeout released a live run's guard") + } + close(release) + <-started + if c.IsRunning() { + t.Fatal("completed teardown retained guard") + } + started = startController(c) + synctest.Wait() + c.Stop(true, 0) + <-started + if runs != 2 || c.IsRunning() { + t.Fatal("could not restart after timed-out teardown completed") + } + }) +} + +func TestControllerPinBehavior(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := NewController(nil) + var runStop chan struct{} + var pins []int64 + stubMobile(t, func(_ *cli.AppConfig, _ *cli.AppCallback, stop chan struct{}) { + runStop = stop + <-stop + }, func(pin int64, stop chan struct{}) int { + pins = append(pins, pin) + if stop != runStop { + t.Error("PIN check received wrong run") + } + if pin == 1234 { + return 0 + } + return 42 + }) + if code := c.Stop(false, 9); code != 42 { + t.Fatalf("idle Stop PIN result = %d, want 42", code) + } + started := startController(c) + synctest.Wait() + if code := c.Stop(false, 8); code != 42 || !c.IsRunning() { + t.Fatalf("invalid PIN Stop = %d, running = %v", code, c.IsRunning()) + } + select { + case <-runStop: + t.Fatal("invalid PIN stopped run") + default: + } + if code := c.Stop(false, 1234); code != 0 { + t.Fatalf("valid PIN Stop = %d", code) + } + <-started + started = startController(c) + synctest.Wait() + c.Stop(true, 9999) + <-started + if len(pins) != 3 || pins[0] != 9 || pins[1] != 8 || pins[2] != 1234 { + t.Fatalf("PIN calls = %v", pins) + } + }) +} + +func TestControllerSlowPinCannotStopReplacement(t *testing.T) { + synctest.Test(t, func(t *testing.T) { + c := NewController(nil) + pinStarted, releasePin := make(chan struct{}), make(chan struct{}) + var currentStop chan struct{} + stubMobile(t, func(_ *cli.AppConfig, _ *cli.AppCallback, stop chan struct{}) { + currentStop = stop + <-stop + }, func(_ int64, stop chan struct{}) int { + if stop != currentStop { + t.Error("wrong PIN session") + } + close(pinStarted) + <-releasePin + return 0 + }) + first := startController(c) + synctest.Wait() + oldStop := make(chan struct{}) + go func() { c.Stop(false, 1234); close(oldStop) }() + <-pinStarted + c.Stop(true, 0) + <-first + second := startController(c) + synctest.Wait() + close(releasePin) + <-oldStop + if !c.IsRunning() { + t.Fatal("old Stop cleared replacement guard") + } + select { + case <-currentStop: + t.Fatal("old Stop cancelled replacement") + default: + } + c.Stop(true, 0) + <-second + }) +} + +func TestControllerMixedConcurrentCalls(t *testing.T) { + c := NewController(nil) + var active atomic.Int32 + stubMobile(t, func(config *cli.AppConfig, _ *cli.AppCallback, _ chan struct{}) { + if active.Add(1) != 1 { + t.Error("overlapping mobile runs") + } + defer active.Add(-1) + runtime.Gosched() + if config.CdUID != "uid" { + t.Error("active config changed") + } + }, func(int64, chan struct{}) int { return 0 }) + var workers sync.WaitGroup + for i := 0; i < 12; i++ { + workers.Add(1) + go func() { + defer workers.Done() + for j := 0; j < 100; j++ { + c.Start("uid", "", "", "", "", 0, "") + c.IsRunning() + c.Stop(j%2 == 0, 1234) + } + }() + } + workers.Wait() + if c.IsRunning() || active.Load() != 0 { + t.Fatal("concurrent calls left a run active") + } +} + +func TestControllerRunReturnReleasesGuard(t *testing.T) { + c := NewController(nil) + var runs int + stubMobile(t, func(*cli.AppConfig, *cli.AppCallback, chan struct{}) { runs++ }, func(int64, chan struct{}) int { return 0 }) + for i := 0; i < 2; i++ { + c.Start("", "", "", "", "", 0, "") + if c.IsRunning() { + t.Fatal("returned run retained guard") + } + } + if runs != 2 { + t.Fatalf("runs = %d, want 2", runs) + } +} From 6eed09338dccb3b76501fa5ae55d67e94df15b7e Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 21 Sep 2026 19:16:36 +0700 Subject: [PATCH 14/29] fix(cli): skip DNS restoration quietly when the interface is gone A successful macOS upgrade printed "could not get interface error=interface not found iface=en5" before reporting success, because the DNS cleanup logged every interface lookup failure at error level. An interface that is gone, an unplugged adapter or a torn down tether, has no DNS settings left to restore, so the skip is now a debug diagnostic naming the interface and the work it skipped. Only that condition is quiet. netInterface returns a sentinel error for a missing interface and no longer reports a failed enumeration as one, so any other lookup failure and any restoration failure on an interface that does exist stay at error level. --- cmd/cli/cli.go | 15 ++- cmd/cli/dns_reset_missing_iface_test.go | 141 ++++++++++++++++++++++++ cmd/cli/prog.go | 18 ++- 3 files changed, 171 insertions(+), 3 deletions(-) create mode 100644 cmd/cli/dns_reset_missing_iface_test.go diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 8679f406..0070dc68 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -1142,6 +1142,12 @@ func processLogAndCacheFlags(v *viper.Viper, cfg *ctrld.Config) { v.Set("service", cfg.Service) } +// errInterfaceNotFound reports that the named interface does not exist on the +// host. Callers use errors.Is to tell this apart from a lookup that failed for +// another reason, because an interface that is simply gone (an unplugged +// adapter, a torn down tether) is an expected outcome rather than a failure. +var errInterfaceNotFound = errors.New("interface not found") + func netInterface(ifaceName string) (*net.Interface, error) { if ifaceName == autoIface { ifaceName = defaultIfaceName() @@ -1152,13 +1158,18 @@ func netInterface(ifaceName string) (*net.Interface, error) { iface = i.Interface } }) + if err != nil { + // Enumeration itself failed, so the interface cannot be reported as + // missing: nothing was positively established about it. + return nil, err + } if iface == nil { - return nil, errors.New("interface not found") + return nil, errInterfaceNotFound } if _, err := patchNetIfaceName(iface); err != nil { return nil, err } - return iface, err + return iface, nil } func defaultIfaceName() string { diff --git a/cmd/cli/dns_reset_missing_iface_test.go b/cmd/cli/dns_reset_missing_iface_test.go new file mode 100644 index 00000000..ce234be7 --- /dev/null +++ b/cmd/cli/dns_reset_missing_iface_test.go @@ -0,0 +1,141 @@ +package cli + +import ( + "encoding/json" + "fmt" + "os" + "strings" + "testing" + + "github.com/rs/zerolog" +) + +// Test_logIfaceLookupFailure is a regression test for issue-608: a macOS +// upgrade that succeeded still printed +// +// ERR could not get interface error="interface not found" iface=en5 +// +// because the DNS cleanup after the upgrade logged every interface-lookup +// failure at error level, including the interface it had been bound to simply +// being gone. Only that positively identified condition may be downgraded. +func Test_logIfaceLookupFailure(t *testing.T) { + for _, tc := range []struct { + name string + skipping string + err error + wantLevel string + wantMsg string + }{ + { + name: "the previous interface is gone", + skipping: "DNS restoration", + err: errInterfaceNotFound, + wantLevel: "debug", + wantMsg: "Skipping DNS restoration: previous interface is no longer present", + }, + { + name: "the missing interface is reported through a wrapped error", + skipping: "DNS restoration", + err: fmt.Errorf("looking up %q: %w", "en5", errInterfaceNotFound), + wantLevel: "debug", + wantMsg: "Skipping DNS restoration: previous interface is no longer present", + }, + { + // The caller names the work it skipped, so the helper stays usable + // outside the DNS reset path. + name: "another caller names its own work", + skipping: "interface snapshot", + err: errInterfaceNotFound, + wantLevel: "debug", + wantMsg: "Skipping interface snapshot: previous interface is no longer present", + }, + { + name: "the lookup failed for another reason", + skipping: "DNS restoration", + err: fmt.Errorf("patching interface name: %w", os.ErrPermission), + wantLevel: "error", + wantMsg: "could not get interface", + }, + { + // A restoration failure reaches the same logger, and must stay + // visible: the interface exists and its DNS was left wrong. + name: "restoring DNS on an existing interface failed", + skipping: "DNS restoration", + err: fmt.Errorf("failed to restore static DNS config on interface %q: %w", "en0", os.ErrPermission), + wantLevel: "error", + wantMsg: "could not get interface", + }, + } { + t.Run(tc.name, func(t *testing.T) { + var buf syncBuffer + logger := zerolog.New(&buf).With().Str("iface", "en5").Logger() + + logIfaceLookupFailure(&logger, tc.skipping, tc.err) + + entry := soleLogEntry(t, buf.String()) + if got := entry["level"]; got != tc.wantLevel { + t.Errorf("level = %v, want %q (line: %s)", got, tc.wantLevel, strings.TrimSpace(buf.String())) + } + if got := entry["message"]; got != tc.wantMsg { + t.Errorf("message = %v, want %q", got, tc.wantMsg) + } + if got := entry["iface"]; got != "en5" { + t.Errorf("iface = %v, want %q: the diagnostic must name the interface", got, "en5") + } + if tc.wantLevel == "error" { + if got := entry["error"]; got == nil { + t.Error("error field missing: a genuine failure must keep its cause") + } + } else if got, ok := entry["error"]; ok { + t.Errorf("error = %v, want no error field on an expected skip", got) + } + }) + } +} + +// Test_resetDNSForRunningIfaceMissingIface drives the real cleanup path with an +// interface that does not exist, which is what an upgrade sees after the +// adapter ctrld was bound to went away. It changes no DNS settings: the lookup +// fails before any restoration runs. +func Test_resetDNSForRunningIfaceMissingIface(t *testing.T) { + level := zerolog.GlobalLevel() + zerolog.SetGlobalLevel(zerolog.DebugLevel) + t.Cleanup(func() { zerolog.SetGlobalLevel(level) }) + + old := mainLog.Load() + var buf syncBuffer + logger := zerolog.New(&buf) + mainLog.Store(&logger) + t.Cleanup(func() { mainLog.Store(old) }) + + // A name no host assigns, so netInterface reports it as missing rather + // than restoring DNS on a real adapter. + p := &prog{runningIface: "ctrld-absent-iface0"} + + if got := p.resetDNSForRunningIface(false, true); got != nil { + t.Errorf("resetDNSForRunningIface returned %v, want nil for a missing interface", got) + } + + entry := soleLogEntry(t, buf.String()) + if got := entry["level"]; got != "debug" { + t.Errorf("level = %v, want %q: a missing interface must not make a successful upgrade look broken (line: %s)", + got, "debug", strings.TrimSpace(buf.String())) + } + if got := entry["iface"]; got != p.runningIface { + t.Errorf("iface = %v, want %q", got, p.runningIface) + } +} + +// soleLogEntry decodes the single JSON log line in out. +func soleLogEntry(t *testing.T, out string) map[string]any { + t.Helper() + lines := strings.Split(strings.TrimSpace(out), "\n") + if len(lines) != 1 || lines[0] == "" { + t.Fatalf("want exactly one log line, got %d:\n%s", len(lines), out) + } + var entry map[string]any + if err := json.Unmarshal([]byte(lines[0]), &entry); err != nil { + t.Fatalf("decoding log line %q: %v", lines[0], err) + } + return entry +} diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index b77d6c65..c0983f26 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -1810,6 +1810,22 @@ func (p *prog) resetDNS(isStart bool, restoreStatic bool) { } } +// logIfaceLookupFailure reports a failed lookup of the interface the caller was +// about to work on, naming that work in skipping, e.g. "DNS restoration". An +// interface that no longer exists has nothing left to act on — an unplugged +// adapter or a torn down tether is gone along with the settings ctrld changed — +// so the skip is a debug diagnostic rather than a user-facing error: it +// otherwise makes a successful upgrade look broken. Every other lookup failure +// still says something went wrong and stays at error level, as does a failure +// on an interface that does exist. +func logIfaceLookupFailure(logger *zerolog.Logger, skipping string, err error) { + if errors.Is(err, errInterfaceNotFound) { + logger.Debug().Msgf("Skipping %s: previous interface is no longer present", skipping) + return + } + logger.Error().Err(err).Msg("could not get interface") +} + // resetDNSForRunningIface performs a DNS reset on the running interface. // The parameter isStart indicates whether this is being called as part of a start (or restart) // command. When true, we check if the current static DNS configuration already differs from the @@ -1824,7 +1840,7 @@ func (p *prog) resetDNSForRunningIface(isStart bool, restoreStatic bool) (runnin logger := mainLog.Load().With().Str("iface", p.runningIface).Logger() netIface, err := netInterface(p.runningIface) if err != nil { - logger.Error().Err(err).Msg("could not get interface") + logIfaceLookupFailure(&logger, "DNS restoration", err) return } runningIface = netIface From 5410cc5f2b7abd51475119b39b087de1e1c31ea7 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 21 Sep 2026 19:36:51 +0700 Subject: [PATCH 15/29] test(cli): cover the DNS reset failure paths through the real code The genuine-failure cases fed a fabricated error straight to logIfaceLookupFailure, so they exercised only its own branching. A restoration failure never reaches that helper, and no test reached the enumeration classification in netInterface at all: discarding the enumeration error, or downgrading an actual restoration failure to debug, both kept the suite green. The host boundaries of the reset path are now variables, in the style the package already uses: the interface enumeration, the interface lookup, the DNS setters and the NetworkManager restore. The tests stub them, so a run changes no host DNS or NetworkManager state, and drive the real netInterface and resetDNSForRunningIface to assert that a failed enumeration keeps its cause and is not reported as a missing interface, and that a failure to restore the saved static config or to reset to DHCP on an interface that does exist stays at error level. Both surviving mutations, and three more, now fail. --- cmd/cli/cli.go | 6 +- cmd/cli/dns_reset_missing_iface_test.go | 284 ++++++++++++++++++++---- cmd/cli/prog.go | 17 +- 3 files changed, 261 insertions(+), 46 deletions(-) diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 0070dc68..bb9988e4 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -1148,12 +1148,16 @@ func processLogAndCacheFlags(v *viper.Viper, cfg *ctrld.Config) { // adapter, a torn down tether) is an expected outcome rather than a failure. var errInterfaceNotFound = errors.New("interface not found") +// foreachInterface enumerates the host network interfaces. It is a variable so +// tests can make enumeration fail without depending on the host. +var foreachInterface = netmon.ForeachInterface + func netInterface(ifaceName string) (*net.Interface, error) { if ifaceName == autoIface { ifaceName = defaultIfaceName() } var iface *net.Interface - err := netmon.ForeachInterface(func(i netmon.Interface, prefixes []netip.Prefix) { + err := foreachInterface(func(i netmon.Interface, prefixes []netip.Prefix) { if i.Name == ifaceName { iface = i.Interface } diff --git a/cmd/cli/dns_reset_missing_iface_test.go b/cmd/cli/dns_reset_missing_iface_test.go index ce234be7..8bf5bab2 100644 --- a/cmd/cli/dns_reset_missing_iface_test.go +++ b/cmd/cli/dns_reset_missing_iface_test.go @@ -2,12 +2,17 @@ package cli import ( "encoding/json" + "errors" "fmt" + "net" + "net/netip" "os" + "path/filepath" "strings" "testing" "github.com/rs/zerolog" + "tailscale.com/net/netmon" ) // Test_logIfaceLookupFailure is a regression test for issue-608: a macOS @@ -17,7 +22,11 @@ import ( // // because the DNS cleanup after the upgrade logged every interface-lookup // failure at error level, including the interface it had been bound to simply -// being gone. Only that positively identified condition may be downgraded. +// being gone. +// +// This covers the helper's own branching. The conditions it must tell apart +// are exercised through the real code in Test_netInterfaceLookupOutcomes and +// Test_resetDNSForRunningIfaceFailures. func Test_logIfaceLookupFailure(t *testing.T) { for _, tc := range []struct { name string @@ -50,21 +59,13 @@ func Test_logIfaceLookupFailure(t *testing.T) { wantMsg: "Skipping interface snapshot: previous interface is no longer present", }, { + // What patchNetIfaceName reports for an interface that does exist. name: "the lookup failed for another reason", skipping: "DNS restoration", err: fmt.Errorf("patching interface name: %w", os.ErrPermission), wantLevel: "error", wantMsg: "could not get interface", }, - { - // A restoration failure reaches the same logger, and must stay - // visible: the interface exists and its DNS was left wrong. - name: "restoring DNS on an existing interface failed", - skipping: "DNS restoration", - err: fmt.Errorf("failed to restore static DNS config on interface %q: %w", "en0", os.ErrPermission), - wantLevel: "error", - wantMsg: "could not get interface", - }, } { t.Run(tc.name, func(t *testing.T) { var buf syncBuffer @@ -93,49 +94,250 @@ func Test_logIfaceLookupFailure(t *testing.T) { } } -// Test_resetDNSForRunningIfaceMissingIface drives the real cleanup path with an -// interface that does not exist, which is what an upgrade sees after the -// adapter ctrld was bound to went away. It changes no DNS settings: the lookup -// fails before any restoration runs. -func Test_resetDNSForRunningIfaceMissingIface(t *testing.T) { +// Test_netInterfaceLookupOutcomes pins which lookup failures netInterface may +// report as a missing interface. Only that error is quiet downstream, so an +// enumeration that never ran must not be reported as one: the host may well +// still have the interface. +func Test_netInterfaceLookupOutcomes(t *testing.T) { + errEnumerate := errors.New("route socket closed") + + t.Run("enumeration failed", func(t *testing.T) { + stubForeachInterface(t, func(fn func(netmon.Interface, []netip.Prefix)) error { + return errEnumerate + }) + + iface, err := netInterface("en5") + + if iface != nil { + t.Errorf("iface = %v, want nil", iface) + } + if !errors.Is(err, errEnumerate) { + t.Errorf("err = %v, want it to keep the enumeration cause %v", err, errEnumerate) + } + if errors.Is(err, errInterfaceNotFound) { + t.Error("a failed enumeration must not report the interface as missing: " + + "the caller would then skip DNS restoration on an interface that still exists") + } + }) + + t.Run("enumeration ran and the interface is absent", func(t *testing.T) { + stubForeachInterface(t, func(fn func(netmon.Interface, []netip.Prefix)) error { + fn(netmon.Interface{Interface: &net.Interface{Index: 1, Name: "en0"}}, nil) + return nil + }) + + iface, err := netInterface("en5") + + if iface != nil { + t.Errorf("iface = %v, want nil", iface) + } + if !errors.Is(err, errInterfaceNotFound) { + t.Errorf("err = %v, want errInterfaceNotFound", err) + } + }) +} + +// Test_resetDNSForRunningIfaceFailures drives the real cleanup path and asserts +// how each outcome is reported: the interface being gone is a debug skip, while +// a failure to restore DNS on an interface that does exist stays an error. The +// OS boundaries are stubbed, so no host DNS or NetworkManager state is touched. +func Test_resetDNSForRunningIfaceFailures(t *testing.T) { + errRestore := errors.New("permission denied") + + t.Run("the previous interface is gone", func(t *testing.T) { + buf := captureDNSResetLogs(t) + // The real lookup over an enumeration that does not list the + // interface, which is what an upgrade sees after the adapter ctrld + // was bound to went away. + stubForeachInterface(t, func(fn func(netmon.Interface, []netip.Prefix)) error { + fn(netmon.Interface{Interface: &net.Interface{Index: 1, Name: "en0"}}, nil) + return nil + }) + stubDNSResetBoundaries(t, func(iface *net.Interface, nameservers []string) error { + t.Fatalf("setDNS called for a missing interface: %v", nameservers) + return nil + }, func(iface *net.Interface) error { + t.Fatal("resetDNS called for a missing interface") + return nil + }) + netInterfaceFn = netInterface + + p := &prog{runningIface: "en5"} + if got := p.resetDNSForRunningIface(false, true); got != nil { + t.Errorf("resetDNSForRunningIface returned %v, want nil for a missing interface", got) + } + + entry := findLogEntry(t, buf.String(), "previous interface is no longer present") + if got := entry["level"]; got != "debug" { + t.Errorf("level = %v, want %q: a missing interface must not make a successful upgrade look broken", + got, "debug") + } + if got := entry["iface"]; got != "en5" { + t.Errorf("iface = %v, want %q", got, "en5") + } + }) + + t.Run("restoring the saved static config failed", func(t *testing.T) { + buf := captureDNSResetLogs(t) + iface := stubExistingIface(t) + saveStaticNameservers(t, iface, "192.0.2.1") + called := false + stubDNSResetBoundaries(t, func(i *net.Interface, nameservers []string) error { + called = true + return errRestore + }, func(i *net.Interface) error { + t.Fatal("resetDNS called although a saved static config exists") + return nil + }) + + p := &prog{runningIface: iface.Name} + p.resetDNSForRunningIface(false, true) + + if !called { + t.Fatal("setDNS was never called: the saved static config was not restored") + } + entry := findLogEntry(t, buf.String(), "failed to restore static DNS config") + if got := entry["level"]; got != "error" { + t.Errorf("level = %v, want %q: the interface exists and its DNS was left wrong", + got, "error") + } + if got := entry["error"]; got != errRestore.Error() { + t.Errorf("error = %v, want %q", got, errRestore) + } + }) + + t.Run("resetting to DHCP failed", func(t *testing.T) { + buf := captureDNSResetLogs(t) + iface := stubExistingIface(t) + // No saved static config, so the cleanup falls back to DHCP. + called := false + stubDNSResetBoundaries(t, func(i *net.Interface, nameservers []string) error { + t.Fatalf("setDNS called without a saved static config: %v", nameservers) + return nil + }, func(i *net.Interface) error { + called = true + return errRestore + }) + + p := &prog{runningIface: iface.Name} + p.resetDNSForRunningIface(false, true) + + if !called { + t.Fatal("resetDNS was never called: the interface was not reset to DHCP") + } + entry := findLogEntry(t, buf.String(), "failed to reset DNS to DHCP") + if got := entry["level"]; got != "error" { + t.Errorf("level = %v, want %q: the interface exists and its DNS was left wrong", + got, "error") + } + if got := entry["error"]; got != errRestore.Error() { + t.Errorf("error = %v, want %q", got, errRestore) + } + }) +} + +// stubForeachInterface replaces the host interface enumeration for one test. +func stubForeachInterface(t *testing.T, fn func(func(netmon.Interface, []netip.Prefix)) error) { + t.Helper() + old := foreachInterface + foreachInterface = fn + t.Cleanup(func() { foreachInterface = old }) +} + +// stubDNSResetBoundaries replaces the parts of the DNS reset path that change +// host state: the DNS setters and the NetworkManager restore. +func stubDNSResetBoundaries(t *testing.T, set func(*net.Interface, []string) error, reset func(*net.Interface) error) { + t.Helper() + oldSet, oldReset, oldNM := setIfaceDNSFn, resetIfaceDNSFn, restoreNetworkManagerFn + setIfaceDNSFn, resetIfaceDNSFn, restoreNetworkManagerFn = set, reset, func() error { return nil } + t.Cleanup(func() { + setIfaceDNSFn, resetIfaceDNSFn, restoreNetworkManagerFn = oldSet, oldReset, oldNM + }) +} + +// stubExistingIface makes the lookup report an interface that is present. +func stubExistingIface(t *testing.T) *net.Interface { + t.Helper() + iface := &net.Interface{Index: 1, Name: "ctrld-test0"} + old := netInterfaceFn + netInterfaceFn = func(name string) (*net.Interface, error) { + if name != iface.Name { + return nil, errInterfaceNotFound + } + return iface, nil + } + t.Cleanup(func() { netInterfaceFn = old }) + return iface +} + +// saveStaticNameservers writes the saved static DNS config that the cleanup +// restores, into a home directory private to this test. +func saveStaticNameservers(t *testing.T, iface *net.Interface, nameservers ...string) { + t.Helper() + old := homedir + homedir = t.TempDir() + t.Cleanup(func() { homedir = old }) + file := filepath.Join(homedir, ".dns_"+iface.Name) + if err := os.WriteFile(file, []byte(strings.Join(nameservers, ",")), 0600); err != nil { + t.Fatalf("writing %s: %v", file, err) + } +} + +// captureDNSResetLogs redirects mainLog and enables debug output for one test. +func captureDNSResetLogs(t *testing.T) *syncBuffer { + t.Helper() level := zerolog.GlobalLevel() zerolog.SetGlobalLevel(zerolog.DebugLevel) - t.Cleanup(func() { zerolog.SetGlobalLevel(level) }) - old := mainLog.Load() - var buf syncBuffer - logger := zerolog.New(&buf) + buf := &syncBuffer{} + logger := zerolog.New(buf) mainLog.Store(&logger) - t.Cleanup(func() { mainLog.Store(old) }) - - // A name no host assigns, so netInterface reports it as missing rather - // than restoring DNS on a real adapter. - p := &prog{runningIface: "ctrld-absent-iface0"} + t.Cleanup(func() { + mainLog.Store(old) + zerolog.SetGlobalLevel(level) + }) + return buf +} - if got := p.resetDNSForRunningIface(false, true); got != nil { - t.Errorf("resetDNSForRunningIface returned %v, want nil for a missing interface", got) +// soleLogEntry decodes the single JSON log line in out. +func soleLogEntry(t *testing.T, out string) map[string]any { + t.Helper() + entries := logEntries(t, out) + if len(entries) != 1 { + t.Fatalf("want exactly one log line, got %d:\n%s", len(entries), out) } + return entries[0] +} - entry := soleLogEntry(t, buf.String()) - if got := entry["level"]; got != "debug" { - t.Errorf("level = %v, want %q: a missing interface must not make a successful upgrade look broken (line: %s)", - got, "debug", strings.TrimSpace(buf.String())) +// findLogEntry returns the one log line whose message contains want. +func findLogEntry(t *testing.T, out, want string) map[string]any { + t.Helper() + var found []map[string]any + for _, entry := range logEntries(t, out) { + if msg, _ := entry["message"].(string); strings.Contains(msg, want) { + found = append(found, entry) + } } - if got := entry["iface"]; got != p.runningIface { - t.Errorf("iface = %v, want %q", got, p.runningIface) + if len(found) != 1 { + t.Fatalf("want exactly one log line containing %q, got %d:\n%s", want, len(found), out) } + return found[0] } -// soleLogEntry decodes the single JSON log line in out. -func soleLogEntry(t *testing.T, out string) map[string]any { +// logEntries decodes the JSON log lines in out. +func logEntries(t *testing.T, out string) []map[string]any { t.Helper() - lines := strings.Split(strings.TrimSpace(out), "\n") - if len(lines) != 1 || lines[0] == "" { - t.Fatalf("want exactly one log line, got %d:\n%s", len(lines), out) + out = strings.TrimSpace(out) + if out == "" { + t.Fatal("no log output") } - var entry map[string]any - if err := json.Unmarshal([]byte(lines[0]), &entry); err != nil { - t.Fatalf("decoding log line %q: %v", lines[0], err) + var entries []map[string]any + for _, line := range strings.Split(out, "\n") { + var entry map[string]any + if err := json.Unmarshal([]byte(line), &entry); err != nil { + t.Fatalf("decoding log line %q: %v", line, err) + } + entries = append(entries, entry) } - return entry + return entries } diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index c0983f26..f07f127e 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -1810,6 +1810,15 @@ func (p *prog) resetDNS(isStart bool, restoreStatic bool) { } } +// The OS boundaries of the DNS reset path, as variables so tests can exercise +// its failure handling without changing the host's DNS or NetworkManager state. +var ( + netInterfaceFn = netInterface + restoreNetworkManagerFn = restoreNetworkManager + setIfaceDNSFn = setDNS + resetIfaceDNSFn = resetDNS +) + // logIfaceLookupFailure reports a failed lookup of the interface the caller was // about to work on, naming that work in skipping, e.g. "DNS restoration". An // interface that no longer exists has nothing left to act on — an unplugged @@ -1838,13 +1847,13 @@ func (p *prog) resetDNSForRunningIface(isStart bool, restoreStatic bool) (runnin return } logger := mainLog.Load().With().Str("iface", p.runningIface).Logger() - netIface, err := netInterface(p.runningIface) + netIface, err := netInterfaceFn(p.runningIface) if err != nil { logIfaceLookupFailure(&logger, "DNS restoration", err) return } runningIface = netIface - if err := restoreNetworkManager(); err != nil { + if err := restoreNetworkManagerFn(); err != nil { logger.Error().Err(err).Msg("could not restore NetworkManager") return } @@ -1874,13 +1883,13 @@ func (p *prog) resetDNSForRunningIface(isStart bool, restoreStatic bool) (runnin saved := savedStaticNameservers(netIface) if len(saved) > 0 && restoreStatic { logger.Debug().Msgf("Restoring interface %q from saved static config: %v", netIface.Name, saved) - if err := setDNS(netIface, saved); err != nil { + if err := setIfaceDNSFn(netIface, saved); err != nil { logger.Error().Err(err).Msgf("failed to restore static DNS config on interface %q", netIface.Name) return } } else { logger.Debug().Msgf("No saved static DNS config for interface %q; resetting to DHCP", netIface.Name) - if err := resetDNS(netIface); err != nil { + if err := resetIfaceDNSFn(netIface); err != nil { logger.Error().Err(err).Msgf("failed to reset DNS to DHCP on interface %q", netIface.Name) return } From 12ab2a0e718d355b885f067f96f9a113b7506636 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 21 Sep 2026 19:50:56 +0700 Subject: [PATCH 16/29] feat: recognize every NextDNS endpoint under nextdns.io NextDNS serves alternative DoH endpoints beside dns.nextdns.io: the ultralow and anycast variants of dns, dns1 and dns2. They are the same service and take the same client info headers, but isNextDNS matched the one host, so an upstream pointed at any of them was treated as a third party and sent no client info. Recognition now follows the parent domain through dns.IsSubDomain, as IsControlD already does for the ControlD domains. The label by label comparison is case insensitive and keeps lookalikes such as notnextdns.io and nextdns.io.example.com out. Based on the change proposed by Mike (Github username @mike406). See: https://github.com/Control-D-Inc/ctrld/pull/335. --- config.go | 9 +++++- config_internal_test.go | 63 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/config.go b/config.go index befa7b59..8bcc93e3 100644 --- a/config.go +++ b/config.go @@ -78,6 +78,12 @@ const ( controlDNetDomain = "controld.net" controlDDevDomain = "controld.dev" + // nextDNSDomain is the parent domain of the NextDNS DoH endpoints. Beside + // dns.nextdns.io, NextDNS serves alternative endpoints under it, such as + // ultralow.dns.nextdns.io and anycast.dns2.nextdns.io, which are the same + // service and take the same client info headers. + nextDNSDomain = "nextdns.io" + endpointPrefixHTTPS = "https://" endpointPrefixQUIC = "quic://" endpointPrefixH3 = "h3://" @@ -790,6 +796,7 @@ func (uc *UpstreamConfig) IsControlD() bool { return false } +// isNextDNS reports whether this is a NextDNS upstream. func (uc *UpstreamConfig) isNextDNS() bool { domain := uc.Domain if domain == "" { @@ -797,7 +804,7 @@ func (uc *UpstreamConfig) isNextDNS() bool { domain = u.Hostname() } } - return domain == "dns.nextdns.io" + return dns.IsSubDomain(nextDNSDomain, domain) } func (uc *UpstreamConfig) dohTransport(dnsType uint16) http.RoundTripper { diff --git a/config_internal_test.go b/config_internal_test.go index d470a142..02509682 100644 --- a/config_internal_test.go +++ b/config_internal_test.go @@ -397,6 +397,59 @@ func TestUpstreamConfig_VerifyDomain(t *testing.T) { } } +// TestUpstreamConfig_isNextDNS is a regression test for issue-610: NextDNS +// serves its alternative DoH endpoints, the "ultralow" and "anycast" variants, +// under nextdns.io, so recognition follows the parent domain instead of the +// single dns.nextdns.io host. Each case runs through the Domain field and +// through the endpoint-hostname fallback, the two ways an upstream names its +// host. +func TestUpstreamConfig_isNextDNS(t *testing.T) { + tests := []struct { + name string + domain string + want bool + }{ + {"the default endpoint", "dns.nextdns.io", true}, + {"the numbered endpoints", "dns1.nextdns.io", true}, + {"the second numbered endpoint", "dns2.nextdns.io", true}, + {"an ultralow endpoint", "ultralow.dns.nextdns.io", true}, + {"a numbered ultralow endpoint", "ultralow.dns1.nextdns.io", true}, + {"the second numbered ultralow endpoint", "ultralow.dns2.nextdns.io", true}, + {"an anycast endpoint", "anycast.dns.nextdns.io", true}, + {"a numbered anycast endpoint", "anycast.dns1.nextdns.io", true}, + {"the second numbered anycast endpoint", "anycast.dns2.nextdns.io", true}, + {"a mixed case endpoint", "UltraLow.DNS.NextDNS.IO", true}, + {"the parent domain itself", "nextdns.io", true}, + + {"another TLD", "dns.nextdns.com", false}, + {"a domain ending in the same labels", "notnextdns.io", false}, + {"the parent domain as a prefix", "xnextdns.io", false}, + {"the parent domain under another one", "nextdns.io.example.com", false}, + {"an unrelated resolver", "dns.google", false}, + {"a ControlD upstream", "freedns.controld.com", false}, + {"no host at all", "", false}, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name+" as a domain", func(t *testing.T) { + t.Parallel() + uc := &UpstreamConfig{Domain: tc.domain} + if got := uc.isNextDNS(); got != tc.want { + t.Errorf("isNextDNS() = %v for domain %q, want %v", got, tc.domain, tc.want) + } + }) + t.Run(tc.name+" as an endpoint", func(t *testing.T) { + t.Parallel() + // Domain is empty, so the lookup falls back to the endpoint host. + uc := &UpstreamConfig{Endpoint: "https://" + tc.domain + "/abc123"} + if got := uc.isNextDNS(); got != tc.want { + t.Errorf("isNextDNS() = %v for endpoint %q, want %v", got, uc.Endpoint, tc.want) + } + }) + } +} + func TestUpstreamConfig_UpstreamSendClientInfo(t *testing.T) { tests := []struct { name string @@ -413,6 +466,16 @@ func TestUpstreamConfig_UpstreamSendClientInfo(t *testing.T) { &UpstreamConfig{Endpoint: "https://freedns.controld.com/p2", Type: ResolverTypeDOH3}, true, }, + { + "default with nextdns upstream DoH", + &UpstreamConfig{Endpoint: "https://dns.nextdns.io/abc123", Type: ResolverTypeDOH}, + true, + }, + { + "default with nextdns ultralow upstream DoH", + &UpstreamConfig{Endpoint: "https://ultralow.dns2.nextdns.io/abc123", Type: ResolverTypeDOH}, + true, + }, { "default with non-ControlD upstream", &UpstreamConfig{Endpoint: "https://dns.google/dns-query", Type: ResolverTypeDOH}, From f52dfd0354615f0c01b2052873a9c1667b1cfda0 Mon Sep 17 00:00:00 2001 From: Anthony Wong Date: Mon, 21 Sep 2026 14:47:45 +0000 Subject: [PATCH 17/29] fix(cli): carry the review fixes of the master port to v1.0 --- cmd/cli/cli.go | 41 +++--- cmd/cli/dns_proxy.go | 8 +- cmd/cli/dns_proxy_sampling_test.go | 48 +++++++ cmd/cli/internal_domains.go | 25 ++-- cmd/cli/log_journal.go | 25 +++- cmd/cli/log_journal_test.go | 49 +++++++ cmd/cli/log_setup_test.go | 176 +++++++++++++++++++++++++ cmd/cli/log_writer.go | 86 ++++++++++++ cmd/cli/loop.go | 13 +- cmd/cli/net_darwin.go | 4 +- cmd/cli/net_darwin_test.go | 23 ++++ cmd/cli/network_snapshot_test.go | 3 + cmd/cli/network_source_state_test.go | 147 +++++++++++++++++++++ cmd/cli/network_transition.go | 67 +++++++--- cmd/cli/provision_result.go | 11 +- cmd/cli/provision_result_test.go | 19 +++ cmd/cli/query_error_sampler.go | 2 +- cmd/cli/query_error_sampler_test.go | 39 ++++++ cmd/cli/upstream_monitor.go | 15 ++- cmd/cli/upstream_monitor_test.go | 16 +++ cmd/cli/upstream_probe_journal_test.go | 141 ++++++++++++++++++++ docs/network-recovery-diagnostics.md | 6 +- internal/controld/config.go | 48 ++++++- internal/controld/log_upload_test.go | 49 +++++++ 24 files changed, 993 insertions(+), 68 deletions(-) create mode 100644 cmd/cli/log_setup_test.go create mode 100644 cmd/cli/network_source_state_test.go create mode 100644 cmd/cli/upstream_probe_journal_test.go create mode 100644 internal/controld/log_upload_test.go diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index bb9988e4..b6cf64f1 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -355,6 +355,7 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { oldLogPath := cfg.Service.LogPath if uid := cdUIDFromProvToken(); uid != "" { cdUID = uid + p.initLoggingAfterProvisioning() } if cdUID != "" { if !validateCdUpstreamProtocol(p.notifyExitToLogServer) { @@ -420,17 +421,7 @@ func run(appCallback *AppCallback, stopCh chan struct{}) { } if newLogPath := cfg.Service.LogPath; newLogPath != "" && oldLogPath != newLogPath { - // After processCDFlags, log config may change, so reset mainLog and re-init logging. - l := zerolog.New(io.Discard) - mainLog.Store(&l) - - // Copy logs written so far to new log file if possible. - if buf, err := os.ReadFile(oldLogPath); err == nil { - if err := os.WriteFile(newLogPath, buf, os.FileMode(0o600)); err != nil { - mainLog.Load().Warn().Err(err).Msg("could not copy old log file") - } - } - initLoggingWithBackup(false) + p.switchLogPath(oldLogPath, newLogPath) } if err := validateConfig(&cfg); err != nil { @@ -839,7 +830,9 @@ func apiRejectionSummary(statusCode int) string { // provisionSecrets lists every secret-bearing value to strip from provisioning // artifacts, including both parts of a composite "/" --cd -// value, which the API may echo back separately. +// value, which the API may echo back separately. redactSecrets drops the +// values that are too short to redact, so a client ID such as "os" leaves the +// words of the message alone. func provisionSecrets() []string { uid, clientID := controld.ParseRawUID(cdUID) return []string{cdUID, cdOrg, uid, clientID} @@ -1309,11 +1302,7 @@ func selfCheckResolveDomain(ctx context.Context, addr, scope string, domain stri } mainLog.Load().Debug().Msgf("self-check against %q failed", domain) // Ping all upstreams to provide better error message to users. - for name, uc := range cfg.Upstream { - if err := uc.ErrorPing(); err != nil { - mainLog.Load().Err(err).Msgf("failed to connect to upstream.%s, endpoint: %s", name, uc.Endpoint) - } - } + logUpstreamPingFailures(cfg.Upstream) marker := strings.Repeat("=", 32) mainLog.Load().Debug().Msg(marker) mainLog.Load().Debug().Msgf("listener address : %s", addr) @@ -1328,6 +1317,24 @@ func selfCheckResolveDomain(ctx context.Context, addr, scope string, domain stri return errSelfCheckNoAnswer } +// upstreamPingFn is a var so tests can drive the self-check report without a +// network probe. +var upstreamPingFn = (*ctrld.UpstreamConfig).ErrorPing + +// logUpstreamPingFailures names the upstreams that do not answer after a +// failed self-check, so the user learns which one is unreachable. The config +// key and the endpoint are operator text that can hold a token, so both stay +// at debug and the retained line names the upstream by its bounded name. +func logUpstreamPingFailures(upstreams map[string]*ctrld.UpstreamConfig) { + for name, uc := range upstreams { + err := upstreamPingFn(uc) + if err == nil { + continue + } + logUpstreamProbeFailure(upstreamPrefix+name, uc, err, mainLog.Load().Error, "failed to connect to the upstream") + } +} + // unixServiceHomeDir is where a ctrld service with root rights keeps its // files on macOS, Linux, and the BSDs. const unixServiceHomeDir = "/etc/controld" diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index 4d3ca6f5..f16ae8ab 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -989,7 +989,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { res.upstream = upstreamConfig.Endpoint return res } - ctrld.Log(ctx, p.querySampler.event(sampleClassAllEndpointsFailed, ""), "all %v endpoints failed", upstreams) + ctrld.Log(ctx, p.querySampler.event(sampleClassAllEndpointsFailed, ""), "all %v endpoints failed", journalUpstreamNames(upstreams)) // An Internal Domain with explicit resolvers is served only by the // configured resolvers. When all of them are unreachable the query fails @@ -2193,8 +2193,7 @@ func (l *upstreamFailureLog) report(upstream string, uc *ctrld.UpstreamConfig, e level = mainLog.Load().Debug } l.reported = true - logUpstreamProbeFailure(isGeneratedInternalDomainUpstreamRef(upstream, uc), uc, err, level, - "Upstream %s check failed after %v", upstream, duration) + logUpstreamProbeFailure(upstream, uc, err, level, "Upstream check failed after %v", duration) } // checkUpstreamOnce sends a test query to the specified upstream. @@ -2204,8 +2203,7 @@ func (p *prog) checkUpstreamOnce(upstream string, uc *ctrld.UpstreamConfig, fail resolver, err := ctrld.NewResolver(uc) if err != nil { - logUpstreamProbeFailure(isGeneratedInternalDomainUpstreamRef(upstream, uc), uc, err, - mainLog.Load().Error, "Failed to create resolver for upstream %s", upstream) + logUpstreamProbeFailure(upstream, uc, err, mainLog.Load().Error, "Failed to create resolver") return err } diff --git a/cmd/cli/dns_proxy_sampling_test.go b/cmd/cli/dns_proxy_sampling_test.go index 00174460..e6d27bbf 100644 --- a/cmd/cli/dns_proxy_sampling_test.go +++ b/cmd/cli/dns_proxy_sampling_test.go @@ -122,3 +122,51 @@ func Test_querySamplingOnTheQueryPath(t *testing.T) { wantField(t, events[1], "upstream", upstreamOS) wantField(t, events[1], "state", "up") } + +// Test_allEndpointsFailedBoundsTheUpstreamName drives the query path with an +// operator-chosen upstream key. The line that reports the loss of every +// endpoint reaches the journal, so it must name the upstream by its bounded +// name. +func Test_allEndpointsFailedBoundsTheUpstreamName(t *testing.T) { + logs := captureDebugMainLog(t) + uc := &ctrld.UpstreamConfig{ + Name: probeSecret, + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 200, + } + uc.Init() + cfg := &ctrld.Config{Upstream: map[string]*ctrld.UpstreamConfig{probeSecret: uc}} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg)} + // The zero value sampler arms a real timer. Close its window after the + // assertions, so no late summary line reaches another test. + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + msg := new(dns.Msg) + msg.SetQuestion("private.test.", dns.TypeA) + res := p.proxy(context.Background(), &proxyRequest{ + msg: msg, + ufr: &upstreamForResult{ + srcAddr: "192.168.0.1:1234", + upstreams: []string{upstreamPrefix + probeSecret}, + }, + }) + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeServerFailure { + t.Fatal("the proxy answered although every endpoint failed") + } + + var reported string + for _, event := range jsonLogEvents(t, logs, "") { + text, _ := event["message"].(string) + if strings.Contains(text, "endpoints failed") { + reported = text + } + } + if !strings.Contains(reported, upstreamPrefix+"custom") { + t.Fatalf("the all-endpoints line does not name the bounded upstream: %q", reported) + } + if strings.Contains(reported, probeSecret) { + t.Fatalf("the all-endpoints line holds the operator key: %q", reported) + } +} diff --git a/cmd/cli/internal_domains.go b/cmd/cli/internal_domains.go index 3461c063..836a7dcf 100644 --- a/cmd/cli/internal_domains.go +++ b/cmd/cli/internal_domains.go @@ -435,22 +435,25 @@ func isGeneratedInternalDomainUpstreamRef(upstream string, uc *ctrld.UpstreamCon return isGeneratedInternalDomainUpstream(strings.TrimPrefix(upstream, upstreamPrefix), uc) } -// logUpstreamProbeFailure reports a background probe failure against an -// upstream. +// logUpstreamProbeFailure reports a background probe failure against the +// upstream that the reference upstream names. // // Loop checking and recovery checking run on timers, so for a generated // Internal Domain resolver they would publish the organization's private -// address without any user ever querying the domain. They get the same -// treatment as a query failure: endpoint and the address-bearing error at -// debug, a classification above it, so the outcome stays visible while the -// address does not. -func logUpstreamProbeFailure(generated bool, uc *ctrld.UpstreamConfig, err error, level func() *zerolog.Event, format string, args ...any) { +// address without any user ever querying the domain. The name and the +// endpoint of any upstream are operator text that can hold a token, so both +// stay on the debug line. The leveled line names the upstream by its bounded +// name, so the outcome stays visible while the operator text does not. A +// generated Internal Domain resolver also trades the address-bearing error +// for a classification. +func logUpstreamProbeFailure(upstream string, uc *ctrld.UpstreamConfig, err error, level func() *zerolog.Event, format string, args ...any) { detail := append([]any{}, args...) - if !generated { - level().Err(err).Msgf(format+" for upstream: %q, endpoint: %q", append(detail, uc.Name, uc.Endpoint)...) + mainLog.Load().Debug().Err(err).Msgf(format+" for upstream: %q, endpoint: %q", append(detail, uc.Name, uc.Endpoint)...) + event := level().Str("upstream", journalUpstreamName(upstream)) + if !isGeneratedInternalDomainUpstreamRef(upstream, uc) { + event.Err(err).Msgf(format, detail...) return } - mainLog.Load().Debug().Err(err).Msgf(format+" for upstream: %q, endpoint: %q", append(detail, uc.Name, uc.Endpoint)...) - level().Str("failure", internalDomainFailureReason(err)). + event.Str("failure", internalDomainFailureReason(err)). Msgf(format+" for an Internal Domain resolver", detail...) } diff --git a/cmd/cli/log_journal.go b/cmd/cli/log_journal.go index 995de485..77011373 100644 --- a/cmd/cli/log_journal.go +++ b/cmd/cli/log_journal.go @@ -6,7 +6,9 @@ import ( "regexp" "github.com/rs/zerolog" + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" ) // journalField marks an event for the retained journal stream. Both packages @@ -17,16 +19,31 @@ const journalField = ctrld.JournalField // writer keeps a line below warn level only when the line contains it. var journalMarker = []byte(`"` + journalField + `":true`) -// retainedURLPath matches the part of a URL that follows the host. The class -// stops the match at the characters that end a value inside a JSON line. -var retainedURLPath = regexp.MustCompile(`(https?://[^/?"\\\s]+)[/?][^"\\\s]*`) +// retainedURLPath matches the part of a URL that follows the host, for every +// scheme: a DoH, DoH3, or DoQ endpoint carries its token in the path. The +// class stops the match at the characters that end a value inside a JSON line. +var retainedURLPath = regexp.MustCompile(`([a-z][a-z0-9+.-]*://[^/?"\\\s]+)[/?][^"\\\s]*`) + +// retainedControlDHost matches a Control D resolver host. The first label of +// the host is the resolver ID, which a DoT or DoQ endpoint carries in place of +// a path. +var retainedControlDHost = regexp.MustCompile(`([A-Za-z0-9-]+)(\.dns\.controld\.(?:com|dev))`) // redactRetainedLine strips the secrets of one line that the journal keeps. // Support downloads the journal whole, and any error text can carry a DoH URL // with the resolver path and the packed query of the user. func redactRetainedLine(p []byte) []byte { stripped := retainedURLPath.ReplaceAll(p, []byte(`${1}/[redacted]`)) - return []byte(redactSecrets(string(stripped), provisionSecrets()...)) + stripped = retainedControlDHost.ReplaceAll(stripped, []byte(`[redacted]${2}`)) + return []byte(redactSecrets(string(stripped), journalSecrets()...)) +} + +// journalSecrets names the values that no retained line may hold. The client +// ID stays out: it is a device label, not a secret. redactSecrets drops the +// values that are too short to redact. +func journalSecrets() []string { + uid, _ := controld.ParseRawUID(cdUID) + return []string{cdUID, cdOrg, uid} } // journal marks an event for the retained journal stream. The event keeps diff --git a/cmd/cli/log_journal_test.go b/cmd/cli/log_journal_test.go index abd8f7e3..58aa425e 100644 --- a/cmd/cli/log_journal_test.go +++ b/cmd/cli/log_journal_test.go @@ -254,3 +254,52 @@ func Test_stopClosesTheInternalLogFiles(t *testing.T) { } } } + +// Test_redactRetainedLineKeepsAShortClientID puts a two-letter client ID in +// the resolver UID. The keys of a JSON line hold the same letters, and a +// redaction of the client ID would break the line. +func Test_redactRetainedLineKeepsAShortClientID(t *testing.T) { + origCdUID := cdUID + t.Cleanup(func() { cdUID = origCdUID }) + cdUID = "uid12345678/os" + + line := string(redactRetainedLine([]byte(`{"level":"info","os":"darwin","resolver":"uid12345678/os"}`))) + + if !strings.Contains(line, `"os":"darwin"`) { + t.Fatalf("the redaction changed the os key: %s", line) + } + if strings.Contains(line, "uid12345678") { + t.Fatalf("the redaction kept the resolver uid: %s", line) + } +} + +// Test_redactRetainedLineStripsEverySchemePath covers the endpoints that carry +// a token in their path with a scheme other than https. +func Test_redactRetainedLineStripsEverySchemePath(t *testing.T) { + for _, tc := range []struct{ in, want string }{ + {`Get "quic://dns.example/org-v1-SECRET0123?dns=pOgBAAAB": timeout`, `Get "quic://dns.example/[redacted]": timeout`}, + {`h3://dns.example/SECRET0123 failed`, `h3://dns.example/[redacted] failed`}, + {`tls://dns.example:853 failed`, `tls://dns.example:853 failed`}, + {`tls://abcdef12.dns.controld.com:853 failed`, `tls://[redacted].dns.controld.com:853 failed`}, + {`quic://abcdef12.dns.controld.dev failed`, `quic://[redacted].dns.controld.dev failed`}, + {`https://freedns.controld.com/p1 failed`, `https://freedns.controld.com/[redacted] failed`}, + } { + if got := string(redactRetainedLine([]byte(tc.in))); got != tc.want { + t.Errorf("redactRetainedLine(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +// Test_redactRetainedLineKeepsAShortResolverUID puts a two-letter resolver UID +// in place. The keys of a JSON line hold the same letters, and a redaction of +// the UID would break every retained line and the header. +func Test_redactRetainedLineKeepsAShortResolverUID(t *testing.T) { + origCdUID := cdUID + t.Cleanup(func() { cdUID = origCdUID }) + cdUID = "os" + + const in = `{"level":"info","os":"darwin","message":"loss of service"}` + if got := string(redactRetainedLine([]byte(in))); got != in { + t.Fatalf("redactRetainedLine(%q) = %q, want the line unchanged", in, got) + } +} diff --git a/cmd/cli/log_setup_test.go b/cmd/cli/log_setup_test.go new file mode 100644 index 00000000..6f2b3ac9 --- /dev/null +++ b/cmd/cli/log_setup_test.go @@ -0,0 +1,176 @@ +package cli + +import ( + "io" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/rs/zerolog" + + "github.com/Control-D-Inc/ctrld" +) + +// setupLogSetupTest puts ctrld in cd mode with no log_path and a temporary +// home directory, and puts back every global that the logging setup touches. +// The globals are the reason why no test of this package calls t.Parallel. +func setupLogSetupTest(t *testing.T) string { + t.Helper() + dir := t.TempDir() + origLogPath, origLogLevel := cfg.Service.LogPath, cfg.Service.LogLevel + origSilent, origVerbose, origCdUID, origHomedir := silent, verbose, cdUID, homedir + origConsoleWriter := consoleWriter + origMainLog, origProxyLogger := mainLog.Load(), ctrld.ProxyLogger.Load() + origLevel, origTimeFormat := zerolog.GlobalLevel(), zerolog.TimeFieldFormat + origLogPathFile, origPendingRotation := logPathFile.Load(), pendingLogPathRotation.Load() + t.Cleanup(func() { + if rf := logPathFile.Load(); rf != nil && rf != origLogPathFile { + _ = rf.close() + } + logPathFile.Store(origLogPathFile) + pendingLogPathRotation.Store(origPendingRotation) + cfg.Service.LogPath, cfg.Service.LogLevel = origLogPath, origLogLevel + silent, verbose, cdUID, homedir = origSilent, origVerbose, origCdUID, origHomedir + consoleWriter = origConsoleWriter + mainLog.Store(origMainLog) + ctrld.ProxyLogger.Store(origProxyLogger) + zerolog.SetGlobalLevel(origLevel) + zerolog.TimeFieldFormat = origTimeFormat + }) + cfg.Service.LogPath, cfg.Service.LogLevel = "", "info" + silent, verbose, cdUID, homedir = false, 0, "", dir + consoleWriter = zerolog.NewConsoleWriter(func(w *zerolog.ConsoleWriter) { w.Out = io.Discard }) + discard := zerolog.New(io.Discard) + mainLog.Store(&discard) + logPathFile.Store(nil) + pendingLogPathRotation.Store(nil) + stubHeaderSnapshotSources(t) + stubLogHeaderNetworkRead(t, "en0") + return dir +} + +// Test_initLoggingAfterProvisioning_opensTheJournal drives a --cd-org start: +// the first logging setup runs before the resolver UID is known, and the +// provisioning step then opens the internal streams. +func Test_initLoggingAfterProvisioning_opensTheJournal(t *testing.T) { + dir := setupLogSetupTest(t) + p := &prog{cfg: &cfg} + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + + journalPath := filepath.Join(dir, journalLogFileName) + if _, err := os.Stat(journalPath); !os.IsNotExist(err) { + t.Fatalf("journal exists before the UID is known (stat err = %v)", err) + } + + cdUID = "provisioned-uid" + p.initLoggingAfterProvisioning() + + firstLine, headers := logHeaderFileLines(t, journalPath) + if firstLine.Message != logHeaderMessage || headers != 1 { + t.Fatalf("journal starts with %q and holds %d headers, want the header once", firstLine.Message, headers) + } + mainLog.Load().Warn().Msg("retained after provisioning") + if content := rotatingFileTestContent(t, journalPath); !strings.Contains(content, "retained after provisioning") { + t.Fatalf("journal misses the line after provisioning: %s", content) + } + + p.initLoggingAfterProvisioning() + if _, headers := logHeaderFileLines(t, journalPath); headers != 1 { + t.Fatalf("a second call added a header: %d headers", headers) + } +} + +// Test_switchLogPath_leadsTheNewFileAndClosesTheInternalFiles drives the +// log_path that the API config sets after the first logging setup. +func Test_switchLogPath_leadsTheNewFileAndClosesTheInternalFiles(t *testing.T) { + dir := setupLogSetupTest(t) + cdUID = "test-uid" + p := &prog{cfg: &cfg} + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + mainLog.Load().Info().Msg("line before the switch") + + newPath := filepath.Join(dir, "logs", "ctrld.log") + cfg.Service.LogPath = newPath + p.switchLogPath("", newPath) + + firstLine, headers := logHeaderFileLines(t, newPath) + if firstLine.Message != logHeaderMessage || headers != 1 { + t.Fatalf("log_path file starts with %q and holds %d headers, want the header once", firstLine.Message, headers) + } + lw, jlw := p.internalWriters() + if lw.rotating() != nil || jlw.rotating() != nil { + t.Fatal("the internal files stayed open after the switch") + } + if files := p.logHeaderFiles(); !slices.Equal(files, []string{newPath}) { + t.Fatalf("log_files = %v, want the log_path only", files) + } + // cd mode has no local log_path, so the lines of the run stay in the + // internal debug stream that this call closes. + if content := rotatingFileTestContent(t, newPath); strings.Contains(content, "line before the switch") { + t.Fatalf("a line moved although the run had no local log_path: %s", content) + } +} + +// Test_switchLogPath_carriesTheOldFileBelowOneHeader drives a run that already +// wrote a local log_path file. The new file must hold one header, and the +// header must lead the lines that the old file holds. +func Test_switchLogPath_carriesTheOldFileBelowOneHeader(t *testing.T) { + dir := setupLogSetupTest(t) + cdUID = "test-uid" + oldPath := filepath.Join(dir, "old", "ctrld.log") + cfg.Service.LogPath = oldPath + p := &prog{cfg: &cfg} + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + mainLog.Load().Info().Msg("line before the switch") + + newPath := filepath.Join(dir, "logs", "ctrld.log") + cfg.Service.LogPath = newPath + p.switchLogPath(oldPath, newPath) + + firstLine, headers := logHeaderFileLines(t, newPath) + if firstLine.Message != logHeaderMessage || headers != 1 { + t.Fatalf("log_path file starts with %q and holds %d headers, want the header once", firstLine.Message, headers) + } + if !slices.Equal(firstLine.LogFiles, []string{newPath}) { + t.Fatalf("header log_files = %v, want the new log_path", firstLine.LogFiles) + } + if content := rotatingFileTestContent(t, newPath); !strings.Contains(content, "line before the switch") { + t.Fatalf("the new file misses the lines of the old file: %s", content) + } +} + +// Test_initLoggingAfterProvisioning_skipsTheGuardedRuns covers the two runs +// that keep no internal files: a run with a local log_path, and a silent run. +// Neither may gain a journal when the provisioning step learns the UID. +func Test_initLoggingAfterProvisioning_skipsTheGuardedRuns(t *testing.T) { + for _, tc := range []struct { + name string + guard func(t *testing.T, dir string) + }{ + {"log_path set", func(t *testing.T, dir string) { + cfg.Service.LogPath = filepath.Join(dir, "operator.log") + }}, + {"silent set", func(t *testing.T, _ string) { silent = true }}, + } { + t.Run(tc.name, func(t *testing.T) { + dir := setupLogSetupTest(t) + tc.guard(t, dir) + p := &prog{cfg: &cfg} + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + + cdUID = "provisioned-uid" + p.initLoggingAfterProvisioning() + + journalPath := filepath.Join(dir, journalLogFileName) + if _, err := os.Stat(journalPath); !os.IsNotExist(err) { + t.Fatalf("the guarded run opened a journal (stat err = %v)", err) + } + }) + } +} diff --git a/cmd/cli/log_writer.go b/cmd/cli/log_writer.go index 73b7f307..0c66fda6 100644 --- a/cmd/cli/log_writer.go +++ b/cmd/cli/log_writer.go @@ -2,7 +2,9 @@ package cli import ( "bytes" + "encoding/json" "io" + "os" "sync" "time" @@ -255,9 +257,93 @@ func (p *prog) initLogging(backup bool) { } } +// initLoggingAfterProvisioning opens the internal streams of a run whose +// resolver UID came from the provisioning token. The first logging setup ran +// before that UID was known, so it opened no debug file and no journal. +func (p *prog) initLoggingAfterProvisioning() { + if !p.needInternalLogging() { + return + } + if lw, _ := p.internalWriters(); lw != nil { + return + } + p.initLogging(false) +} + +// switchLogPath moves the logging of this run to the log_path that the API +// config set. The header of the new file leads it, the lines of an earlier +// local log_path file follow that header, and the internal streams close, +// because a run with a log_path keeps no internal files. A run without a local +// log_path carries no lines: its history stays in the internal debug stream, +// which this call closes. +func (p *prog) switchLogPath(oldLogPath, newLogPath string) { + // After processCDFlags, log config may change, so reset mainLog and re-init logging. + discard := zerolog.New(io.Discard) + mainLog.Store(&discard) + + carried := carriedLogLines(oldLogPath, newLogPath) + p.initLogging(false) + p.carryLogLines(carried) +} + +// carriedLogLines reads the lines of the old log_path file and empties the new +// file, so the header of the new file leads it. It drops the header lines of +// the old file, because each of them names the old file. +func carriedLogLines(oldLogPath, newLogPath string) []byte { + buf, err := os.ReadFile(oldLogPath) + if err != nil { + return nil + } + if err := os.Remove(newLogPath); err != nil && !os.IsNotExist(err) { + mainLog.Load().Warn().Err(err).Msg("could not clear the new log file") + return nil + } + return withoutLogHeaderLines(buf) +} + +// carryLogLines appends the lines of the old log_path file below the header of +// the new one. +func (p *prog) carryLogLines(lines []byte) { + if len(lines) == 0 { + return + } + rf := logPathFile.Load() + if rf == nil { + return + } + if _, err := rf.Write(lines); err != nil { + mainLog.Load().Warn().Err(err).Msg("could not copy old log file") + } +} + +// withoutLogHeaderLines removes every header line of a log file. +func withoutLogHeaderLines(buf []byte) []byte { + var kept bytes.Buffer + for _, line := range bytes.SplitAfter(buf, []byte("\n")) { + if isLogHeaderLine(line) { + continue + } + kept.Write(line) + } + return kept.Bytes() +} + +// isLogHeaderLine reports whether one line of a log file is a header line. +func isLogHeaderLine(line []byte) bool { + var parsed struct { + Message string `json:"message"` + } + if err := json.Unmarshal(bytes.TrimSpace(line), &parsed); err != nil { + return false + } + return parsed.Message == logHeaderMessage +} + // initInternalLogging performs internal logging if there's no log enabled. func (p *prog) initInternalLogging(writers []io.Writer) { if !p.needInternalLogging() { + // A log_path that the API set takes over from the internal files. + p.closeInternalLogs() return } headersWritten := false diff --git a/cmd/cli/loop.go b/cmd/cli/loop.go index e088001b..b4adb8f8 100644 --- a/cmd/cli/loop.go +++ b/cmd/cli/loop.go @@ -86,10 +86,9 @@ func (p *prog) detectLoop(msg *dns.Msg) { func (p *prog) checkDnsLoop() { mainLog.Load().Debug().Msg("start checking DNS loop") upstream := make(map[string]*ctrld.UpstreamConfig) - // Generated Internal Domain resolvers are local addresses, so they pass - // canBeLocalUpstream and are probed every minute. Record which ones they - // are while the config key is still in hand. - generated := make(map[string]bool) + // The probe loop below runs on the upstream UID, and the failure report + // needs the config key. Record the reference while the key is in hand. + reference := make(map[string]string) p.loopMu.Lock() for n, uc := range p.cfg.Upstream { if p.um.isDown("upstream." + n) { @@ -103,7 +102,7 @@ func (p *prog) checkDnsLoop() { uid := uc.UID() p.loop[uid] = false upstream[uid] = uc - generated[uid] = isGeneratedInternalDomainUpstream(n, uc) + reference[uid] = upstreamPrefix + n } p.loopMu.Unlock() @@ -116,11 +115,11 @@ func (p *prog) checkDnsLoop() { } resolver, err := ctrld.NewResolver(uc) if err != nil { - logUpstreamProbeFailure(generated[uid], uc, err, mainLog.Load().Warn, "could not perform loop check") + logUpstreamProbeFailure(reference[uid], uc, err, mainLog.Load().Warn, "could not perform loop check") continue } if _, err := resolver.Resolve(context.Background(), msg); err != nil { - logUpstreamProbeFailure(generated[uid], uc, err, mainLog.Load().Warn, "could not send DNS loop check query") + logUpstreamProbeFailure(reference[uid], uc, err, mainLog.Load().Warn, "could not send DNS loop check query") } } mainLog.Load().Debug().Msg("end checking DNS loop") diff --git a/cmd/cli/net_darwin.go b/cmd/cli/net_darwin.go index 8d9c4abf..4856150c 100644 --- a/cmd/cli/net_darwin.go +++ b/cmd/cli/net_darwin.go @@ -74,7 +74,9 @@ func serviceNamesByDevice(r io.Reader) map[string]string { for scanner.Scan() { line := scanner.Text() if strings.Contains(line, "*") { - // Network services is disabled. + // A disabled service names no device. Without the reset, the device + // line that follows takes the name of the service before it. + prevLine = "" continue } device := serviceOrderDevice(line) diff --git a/cmd/cli/net_darwin_test.go b/cmd/cli/net_darwin_test.go index 280e4d50..1ac1326e 100644 --- a/cmd/cli/net_darwin_test.go +++ b/cmd/cli/net_darwin_test.go @@ -313,3 +313,26 @@ func Test_platformInterfaceMeta(t *testing.T) { t.Error("platformVirtualInterfaces is not nil on darwin") } } + +// Test_serviceNamesByDevice_namesNoDisabledService drives output that holds no +// blank line between the blocks. The device of a disabled service must keep no +// name, and the name of the service before it is the wrong answer. +func Test_serviceNamesByDevice_namesNoDisabledService(t *testing.T) { + const fixture = `An asterisk (*) denotes that a network service is disabled. +(1) Wi-Fi +(Hardware Port: Wi-Fi, Device: en0) +(2) Thunderbolt Bridge +(Hardware Port: Thunderbolt Bridge, Device: bridge0) +(*) Ethernet Adapter (en6) +(Hardware Port: Ethernet Adapter (en6), Device: en6) +` + + names := serviceNamesByDevice(strings.NewReader(fixture)) + + if got := names["en6"]; got != "" { + t.Errorf("service of the disabled en6 = %q, want no name", got) + } + if got := names["bridge0"]; got != "Thunderbolt Bridge" { + t.Errorf("service of bridge0 = %q, want %q", got, "Thunderbolt Bridge") + } +} diff --git a/cmd/cli/network_snapshot_test.go b/cmd/cli/network_snapshot_test.go index e582e913..f1276ddd 100644 --- a/cmd/cli/network_snapshot_test.go +++ b/cmd/cli/network_snapshot_test.go @@ -128,6 +128,9 @@ func Test_linkTypeFor(t *testing.T) { {"thunderbolt_bridge", "Thunderbolt Bridge", "hardware", "ethernet"}, {"iphone_usb_port", "iPhone USB", "hardware", "usb_tether"}, {"tunnel_class", "", "tunnel", "tunnel"}, + // The Windows adapter description reaches linkTypeFor as the port + // name, and a wired word in it wins over the tunnel class. + {"ethernet_word_wins_over_tunnel_class", "SSL VPN Virtual Ethernet Adapter", "tunnel", "ethernet"}, {"no_port_no_class", "", "", "unknown"}, {"unknown_port", "Bluetooth PAN", "hardware", "unknown"}, } { diff --git a/cmd/cli/network_source_state_test.go b/cmd/cli/network_source_state_test.go new file mode 100644 index 00000000..b1f7726d --- /dev/null +++ b/cmd/cli/network_source_state_test.go @@ -0,0 +1,147 @@ +package cli + +import ( + "errors" + "net" + "net/netip" + "slices" + "testing" + + "github.com/rs/zerolog" + "github.com/Control-D-Inc/ctrld" + "tailscale.com/net/netmon" +) + +func Test_interfaceAddressPrefix(t *testing.T) { + for _, tc := range []struct { + name string + address net.Addr + want string + }{ + {"IPv4 with a 4-byte mask", &net.IPNet{IP: net.IPv4(192, 0, 2, 10).To4(), Mask: net.CIDRMask(24, 32)}, "192.0.2.10/24"}, + {"IPv4 in the 16-byte form with a 4-byte mask", &net.IPNet{IP: net.IPv4(192, 0, 2, 10), Mask: net.CIDRMask(24, 32)}, "192.0.2.10/24"}, + {"IPv4 in the 16-byte form with a 16-byte mask", &net.IPNet{IP: net.IPv4(192, 0, 2, 10), Mask: net.CIDRMask(120, 128)}, "192.0.2.10/24"}, + {"IPv6 with a 64-bit mask", &net.IPNet{IP: net.ParseIP("2001:db8::1"), Mask: net.CIDRMask(64, 128)}, "2001:db8::1/64"}, + {"IPv4 without a mask", &net.IPAddr{IP: net.IPv4(192, 0, 2, 10)}, "192.0.2.10/32"}, + {"IPv6 without a mask", &net.IPAddr{IP: net.ParseIP("2001:db8::1")}, "2001:db8::1/128"}, + {"IPv4 with a nil mask", &net.IPNet{IP: net.IPv4(192, 0, 2, 10)}, "192.0.2.10/32"}, + {"IPv4 with an irregular mask", &net.IPNet{IP: net.IPv4(192, 0, 2, 10), Mask: net.IPMask{255, 0, 255, 0}}, "192.0.2.10/32"}, + } { + t.Run(tc.name, func(t *testing.T) { + prefix, err := interfaceAddressPrefix(tc.address) + if err != nil { + t.Fatalf("interfaceAddressPrefix: %v", err) + } + if prefix.String() != tc.want { + t.Fatalf("prefix = %s, want %s", prefix, tc.want) + } + }) + } + if _, err := interfaceAddressPrefix(&net.TCPAddr{IP: net.IPv4(192, 0, 2, 10), Port: 53}); err == nil { + t.Fatal("a TCP address gave a prefix, want an error") + } +} + +// sourceTestInterfaces returns one up interface with a /24 and a /64 address, +// through the reader seam of networkSourceState. +func sourceTestInterfaces() ([]net.Interface, func(*net.Interface) ([]net.Addr, error)) { + interfaces := []net.Interface{{Index: 1, Name: "en0", Flags: net.FlagUp}} + addresses := func(iface *net.Interface) ([]net.Addr, error) { + if iface.Name != "en0" { + return nil, errors.New("unknown interface") + } + return []net.Addr{ + &net.IPNet{IP: net.IPv4(192, 0, 2, 10), Mask: net.CIDRMask(24, 32)}, + &net.IPNet{IP: net.ParseIP("2001:db8::1"), Mask: net.CIDRMask(64, 128)}, + }, nil + } + return interfaces, addresses +} + +func Test_networkSourceState_keepsTheMasks(t *testing.T) { + state, err := networkSourceState(sourceTestInterfaces()) + if err != nil { + t.Fatalf("networkSourceState: %v", err) + } + if !state.Interface["en0"].IsUp() { + t.Fatal("en0 is not up") + } + if got := prefixStrings(state.InterfaceIPs["en0"]); !slices.Equal(got, []string{"192.0.2.10/24", "2001:db8::1/64"}) { + t.Fatalf("prefixes = %v, want the /24 and the /64 with the host address", got) + } +} + +// stubSourceStateRead makes the fresh network read return the test interfaces +// through the real state builder. +func stubSourceStateRead(t *testing.T) { + t.Helper() + origRead := readNetworkSourceStateFn + t.Cleanup(func() { readNetworkSourceStateFn = origRead }) + readNetworkSourceStateFn = func() (*netmon.State, error) { return networkSourceState(sourceTestInterfaces()) } +} + +// Test_freshSnapshotAndHeaderKeepTheMasks proves that the send-time header and +// the fresh snapshot show the /24 and the /64 of the host, as the network +// monitor does. +func Test_freshSnapshotAndHeaderKeepTheMasks(t *testing.T) { + setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + stubHeaderSnapshotSources(t) + stubSourceStateRead(t) + p := &prog{cfg: &ctrld.Config{}} + want := []string{"192.0.2.10/24", "2001:db8::1/64"} + + snapshot := buildNetworkSnapshot(p.freshSnapshotInputs()) + index := slices.IndexFunc(snapshot.Interfaces, func(i snapshotInterface) bool { return i.Name == "en0" }) + if index < 0 { + t.Fatalf("snapshot holds no en0: %+v", snapshot.Interfaces) + } + if got := snapshot.Interfaces[index].IPs; !slices.Equal(got, want) { + t.Fatalf("snapshot ips = %v, want %v", got, want) + } + + parsed, _ := parseLogHeader(t, p.sendLogHeader()) + var headerIPs []string + for _, iface := range parsed.Network.Interfaces { + if iface.Name == "en0" { + headerIPs = iface.IPs + } + } + if !slices.Equal(headerIPs, want) { + t.Fatalf("header ips = %v, want %v", headerIPs, want) + } +} + +// Test_sourceValidationKeepsTheHostAddress proves that a state with masked +// prefixes still names the host address, not the subnet address, as the +// source of the process. +func Test_sourceValidationKeepsTheHostAddress(t *testing.T) { + before4, before6 := ctrld.GetDefaultLocalIPv4(), ctrld.GetDefaultLocalIPv6() + t.Cleanup(func() { + ctrld.SetDefaultLocalIPv4(before4) + ctrld.SetDefaultLocalIPv6(before6) + }) + state, err := networkSourceState(sourceTestInterfaces()) + if err != nil { + t.Fatalf("networkSourceState: %v", err) + } + + ctrld.SetDefaultLocalIPv4(net.ParseIP("192.0.2.10")) + ctrld.SetDefaultLocalIPv6(net.ParseIP("2001:db8::1")) + validateDefaultLocalIPsFromDelta(state, 1) + if got := ctrld.GetDefaultLocalIPv4(); !got.Equal(net.ParseIP("192.0.2.10")) { + t.Fatalf("source IPv4 = %v, want the host address kept", got) + } + if got := ctrld.GetDefaultLocalIPv6(); !got.Equal(net.ParseIP("2001:db8::1")) { + t.Fatalf("source IPv6 = %v, want the host address kept", got) + } + + // The subnet address is not an address of the host. + ctrld.SetDefaultLocalIPv4(net.ParseIP("192.0.2.0")) + validateDefaultLocalIPsFromDelta(state, 2) + if got := ctrld.GetDefaultLocalIPv4(); got != nil { + t.Fatalf("source IPv4 = %v, want the subnet address cleared", got) + } + if reason := sourceInvalidReason(state, netip.MustParseAddr("192.0.2.10").AsSlice()); reason != "" { + t.Fatalf("host address reported %q, want a valid source", reason) + } +} diff --git a/cmd/cli/network_transition.go b/cmd/cli/network_transition.go index fb7825d8..082516f3 100644 --- a/cmd/cli/network_transition.go +++ b/cmd/cli/network_transition.go @@ -50,34 +50,71 @@ func readNetworkSourceState() (*netmon.State, error) { if err != nil { return nil, err } + return networkSourceState(interfaces, (*net.Interface).Addrs) +} + +// networkSourceState builds the state of the given interfaces. The address +// reader is a parameter, so a test feeds addresses without a host network. +func networkSourceState(interfaces []net.Interface, addressesOf func(*net.Interface) ([]net.Addr, error)) (*netmon.State, error) { state := &netmon.State{Interface: map[string]netmon.Interface{}, InterfaceIPs: map[string][]netip.Prefix{}} for _, iface := range interfaces { - addresses, err := iface.Addrs() + addresses, err := addressesOf(&iface) if err != nil { return nil, err } state.Interface[iface.Name] = netmon.Interface{Interface: &iface} for _, address := range addresses { - var ip net.IP - switch address := address.(type) { - case *net.IPNet: - ip = address.IP - case *net.IPAddr: - ip = address.IP - default: - return nil, errors.New("unsupported interface address type") + prefix, err := interfaceAddressPrefix(address) + if err != nil { + return nil, err } - addr, ok := netip.AddrFromSlice(ip) - if !ok { - return nil, errors.New("invalid interface address") - } - addr = addr.Unmap() - state.InterfaceIPs[iface.Name] = append(state.InterfaceIPs[iface.Name], netip.PrefixFrom(addr, addr.BitLen())) + state.InterfaceIPs[iface.Name] = append(state.InterfaceIPs[iface.Name], prefix) } } return state, nil } +// interfaceAddressPrefix turns one interface address into a prefix. An address +// with a subnet mask keeps the mask and its host bits, so a fresh state shows +// the same /24 or /64 as the network monitor. An address without a mask is a +// host prefix. +func interfaceAddressPrefix(address net.Addr) (netip.Prefix, error) { + var ip net.IP + var mask net.IPMask + switch address := address.(type) { + case *net.IPNet: + ip, mask = address.IP, address.Mask + case *net.IPAddr: + ip = address.IP + default: + return netip.Prefix{}, errors.New("unsupported interface address type") + } + addr, ok := netip.AddrFromSlice(ip) + if !ok { + return netip.Prefix{}, errors.New("invalid interface address") + } + addr = addr.Unmap() + return netip.PrefixFrom(addr, prefixBits(addr, mask)), nil +} + +// prefixBits reads the length of a mask for an unmapped address. The mask of +// an IPv4 address is 4 or 16 bytes long, and a 16-byte mask counts the 96 +// bits of the IPv6 mapping too. A missing or irregular mask means a host +// prefix. +func prefixBits(addr netip.Addr, mask net.IPMask) int { + bits, total := mask.Size() + if total == 0 { + return addr.BitLen() + } + if addr.Is4() && total == 128 { + bits -= 96 + } + if bits < 0 || bits > addr.BitLen() { + return addr.BitLen() + } + return bits +} + // netmon v1.74.0 caches only major snapshots. Minor and time-jump events // belong to their Old snapshot, not their distinct New snapshot. func networkSnapshotCurrent(delta *netmon.ChangeDelta, major bool, current *netmon.State) bool { diff --git a/cmd/cli/provision_result.go b/cmd/cli/provision_result.go index 606b8a41..c0b91f61 100644 --- a/cmd/cli/provision_result.go +++ b/cmd/cli/provision_result.go @@ -192,10 +192,17 @@ func newProvisionResult(code provisionFailureCode, message string, attempts []pr return r } -// redactSecrets removes every non-empty secret from s. +// redactSecretMinLen is the length below which a value is too short to +// redact. Two or three letters also match the keys of a JSON line and the +// words of a message, so a redaction of them corrupts the output. Corrupt +// output costs support more than a short value that the API rejects at the +// next step. It is the shortest token that the API parses. +const redactSecretMinLen = provisionTokenMinLen + +// redactSecrets removes every secret from s that is long enough to redact. func redactSecrets(s string, secrets ...string) string { for _, secret := range secrets { - if secret == "" { + if len(secret) < redactSecretMinLen { continue } s = strings.ReplaceAll(s, secret, "[redacted]") diff --git a/cmd/cli/provision_result_test.go b/cmd/cli/provision_result_test.go index f7f95d3c..e4720aba 100644 --- a/cmd/cli/provision_result_test.go +++ b/cmd/cli/provision_result_test.go @@ -292,3 +292,22 @@ func TestFailProvisionWritesLogsNotifiesAndExits(t *testing.T) { t.Errorf("persisted code = %q", out.Code) } } + +// Test_redactSecretsKeepsAShortClientID puts a two-letter client ID in the +// resolver UID. The provisioning messages hold the same letters in ordinary +// words, and a redaction of the client ID would mangle them. +func Test_redactSecretsKeepsAShortClientID(t *testing.T) { + origCdUID, origCdOrg := cdUID, cdOrg + t.Cleanup(func() { cdUID, cdOrg = origCdUID, origCdOrg }) + cdUID, cdOrg = "uid12345678/os", "" + + const in = "failed to reach the host uid12345678/os" + got := redactSecrets(in, provisionSecrets()...) + + if !strings.Contains(got, "failed to reach the host ") { + t.Fatalf("the redaction mangled the words of the message: %s", got) + } + if strings.Contains(got, "uid12345678") { + t.Fatalf("the redaction kept the resolver uid: %s", got) + } +} diff --git a/cmd/cli/query_error_sampler.go b/cmd/cli/query_error_sampler.go index af382f7c..c1c3ea55 100644 --- a/cmd/cli/query_error_sampler.go +++ b/cmd/cli/query_error_sampler.go @@ -145,7 +145,7 @@ func (s *errorSampler) logSummaries(summaries []sampleSummary) { for _, summary := range summaries { mainLog.Load().Error(). Str("class", summary.key.class). - Str("upstream", summary.key.upstream). + Str("upstream", journalUpstreamName(summary.key.upstream)). Int("count", summary.count). Int("suppressed", summary.suppressed). Int("window_s", int(querySampleWindow/time.Second)). diff --git a/cmd/cli/query_error_sampler_test.go b/cmd/cli/query_error_sampler_test.go index 57eaa32d..17a77cb2 100644 --- a/cmd/cli/query_error_sampler_test.go +++ b/cmd/cli/query_error_sampler_test.go @@ -155,6 +155,26 @@ func Test_errorSamplerLimitsWindowAndSummarizes(t *testing.T) { wantField(t, last, "level", "error") } +// Test_errorSamplerKeepsACustomNameOutOfTheSummary drives a window on an +// operator-chosen upstream key. The key can hold a token, so the summary names +// the upstream by a bounded name. +func Test_errorSamplerKeepsACustomNameOutOfTheSummary(t *testing.T) { + const upstream = upstreamPrefix + "dns.controld.com/org-v1-SECRET0123" + h := newSamplerHarness(t) + + for i := 0; i < querySampleLimit+1; i++ { + h.emit(sampleClassResolveFailed, upstream) + } + h.advance(querySampleWindow) + h.sampler.closeExpired(h.now) + + summaries := samplerSummaries(h.events(t)) + if len(summaries) != 1 { + t.Fatalf("summaries after the window closed: got %d, want 1", len(summaries)) + } + wantField(t, summaries[0], "upstream", upstreamPrefix+"custom") +} + func Test_errorSamplerKeepsBudgetPerUpstream(t *testing.T) { h := newSamplerHarness(t) @@ -258,3 +278,22 @@ func Test_errorSamplerZeroValueWorks(t *testing.T) { t.Fatalf("first line of a window is not at error level: %s", buf.String()) } } + +// Test_errorSamplerKeepsAnEmptyUpstreamEmpty drives a window of a class that +// names no upstream. The summary must not report the window as a custom +// upstream. +func Test_errorSamplerKeepsAnEmptyUpstreamEmpty(t *testing.T) { + h := newSamplerHarness(t) + + for i := 0; i < querySampleLimit+1; i++ { + h.emit(sampleClassAllEndpointsFailed, "") + } + h.advance(querySampleWindow) + h.sampler.closeExpired(h.now) + + summaries := samplerSummaries(h.events(t)) + if len(summaries) != 1 { + t.Fatalf("summaries after the window closed: got %d, want 1", len(summaries)) + } + wantField(t, summaries[0], "upstream", "") +} diff --git a/cmd/cli/upstream_monitor.go b/cmd/cli/upstream_monitor.go index 6d1e62f1..029c9c53 100644 --- a/cmd/cli/upstream_monitor.go +++ b/cmd/cli/upstream_monitor.go @@ -182,14 +182,25 @@ func (um *upstreamMonitor) markUp(upstream, reason string) { // journalUpstreamName bounds the upstream name that a journal event carries. // ctrld generates upstream.os, upstream., and upstream.internal_; every // other key is operator text that can hold a token, so it reads as -// upstream.custom. +// upstream.custom. An empty name holds no operator text and stays empty, +// because a line that names no upstream must not read as a custom one. func journalUpstreamName(upstream string) string { - if upstream == upstreamOS || generatedUpstreamKey(strings.TrimPrefix(upstream, upstreamPrefix)) { + if upstream == "" || upstream == upstreamOS || generatedUpstreamKey(strings.TrimPrefix(upstream, upstreamPrefix)) { return upstream } return upstreamPrefix + "custom" } +// journalUpstreamNames bounds every name of a list, for a line that names +// several upstreams. +func journalUpstreamNames(upstreams []string) []string { + names := make([]string, 0, len(upstreams)) + for _, upstream := range upstreams { + names = append(names, journalUpstreamName(upstream)) + } + return names +} + // generatedUpstreamKey reports whether key is a number, or internal_ and a // number, the two shapes that ctrld writes itself. func generatedUpstreamKey(key string) bool { diff --git a/cmd/cli/upstream_monitor_test.go b/cmd/cli/upstream_monitor_test.go index 68780d86..20f6a9b1 100644 --- a/cmd/cli/upstream_monitor_test.go +++ b/cmd/cli/upstream_monitor_test.go @@ -1,6 +1,7 @@ package cli import ( + "slices" "testing" "time" @@ -230,6 +231,21 @@ func Test_upstreamMonitorKeepsACustomNameOutOfTheJournal(t *testing.T) { } } +// Test_journalUpstreamNamesBoundsEveryName covers the line that names several +// upstreams at once. One operator key in the list must not reach the journal. +func Test_journalUpstreamNamesBoundsEveryName(t *testing.T) { + got := journalUpstreamNames([]string{ + upstreamPrefix + "0", + upstreamPrefix + "dns.controld.com/org-v1-SECRET0123", + upstreamOS, + }) + + want := []string{upstreamPrefix + "0", upstreamPrefix + "custom", upstreamOS} + if !slices.Equal(got, want) { + t.Fatalf("journalUpstreamNames = %v, want %v", got, want) + } +} + func Test_upstreamMonitorCountDown(t *testing.T) { captureDebugMainLog(t) um := newUpstreamMonitor(&ctrld.Config{}) diff --git a/cmd/cli/upstream_probe_journal_test.go b/cmd/cli/upstream_probe_journal_test.go new file mode 100644 index 00000000..b36208d9 --- /dev/null +++ b/cmd/cli/upstream_probe_journal_test.go @@ -0,0 +1,141 @@ +package cli + +import ( + "errors" + "strings" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +// probeSecret is the operator text that a custom upstream key, name, and +// endpoint hold in these tests. +const probeSecret = "org-v1-SECRET0123" + +// customProbeUpstream builds an upstream whose key, name, and endpoint all +// hold operator text. +func customProbeUpstream() (string, *ctrld.UpstreamConfig) { + return upstreamPrefix + probeSecret, &ctrld.UpstreamConfig{ + Name: probeSecret, + Endpoint: "https://dns.example.com/" + probeSecret, + } +} + +// retainedProbeLines returns the captured lines that the journal keeps. +func retainedProbeLines(t *testing.T, logs *syncBuffer) []map[string]any { + t.Helper() + var retained []map[string]any + for _, event := range jsonLogEvents(t, logs, "") { + switch event["level"] { + case "error", "warn": + retained = append(retained, event) + } + } + return retained +} + +// wantNoProbeSecret fails when a retained line carries the operator text. +func wantNoProbeSecret(t *testing.T, logs *syncBuffer) { + t.Helper() + for _, event := range retainedProbeLines(t, logs) { + for field, value := range event { + text, ok := value.(string) + if ok && strings.Contains(text, probeSecret) { + t.Fatalf("retained field %q holds the operator text: %s", field, text) + } + } + } +} + +// Test_upstreamFailureLogKeepsACustomKeyOutOfTheJournal drives one recovery +// check failure against an operator-chosen upstream key. The journal keeps +// error lines, so the key, the name, and the endpoint must stay off the line. +func Test_upstreamFailureLogKeepsACustomKeyOutOfTheJournal(t *testing.T) { + logs := captureDebugMainLog(t) + upstream, uc := customProbeUpstream() + + (&upstreamFailureLog{}).report(upstream, uc, errors.New("probe failed"), time.Second) + + retained := retainedProbeLines(t, logs) + if len(retained) != 1 { + t.Fatalf("retained lines: got %d, want 1", len(retained)) + } + wantField(t, retained[0], "upstream", upstreamPrefix+"custom") + wantNoProbeSecret(t, logs) + if !strings.Contains(logs.String(), probeSecret) { + t.Fatal("the operator text must stay at debug level for diagnosis") + } +} + +// Test_checkUpstreamOnceKeepsACustomKeyOutOfTheJournal drives the resolver +// creation failure of a recovery check. It logs at error level, so the journal +// keeps it. +func Test_checkUpstreamOnceKeepsACustomKeyOutOfTheJournal(t *testing.T) { + logs := captureDebugMainLog(t) + upstream, uc := customProbeUpstream() + // An empty type has no resolver, so the check fails before it sends. + uc.Type = "" + p := &prog{} + + if err := p.checkUpstreamOnce(upstream, uc, &upstreamFailureLog{}); err == nil { + t.Fatal("expected the resolver creation to fail") + } + + retained := retainedProbeLines(t, logs) + if len(retained) != 1 { + t.Fatalf("retained lines: got %d, want 1", len(retained)) + } + wantField(t, retained[0], "upstream", upstreamPrefix+"custom") + wantNoProbeSecret(t, logs) +} + +// Test_checkDnsLoopKeepsACustomKeyOutOfTheJournal drives the loop checker +// against an operator-chosen upstream key. The loop checker writes at warn +// level, so the journal keeps its line too. +func Test_checkDnsLoopKeepsACustomKeyOutOfTheJournal(t *testing.T) { + logs := captureDebugMainLog(t) + uc := &ctrld.UpstreamConfig{ + Name: probeSecret, + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 200, + } + uc.Init() + cfg := &ctrld.Config{Upstream: map[string]*ctrld.UpstreamConfig{probeSecret: uc}} + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg), loop: make(map[string]bool)} + + p.checkDnsLoop() + + retained := retainedProbeLines(t, logs) + if len(retained) == 0 { + t.Fatal("the loop check failure must stay visible above debug") + } + for _, event := range retained { + wantField(t, event, "upstream", upstreamPrefix+"custom") + } + wantNoProbeSecret(t, logs) +} + +// Test_logUpstreamPingFailuresKeepsACustomKeyOutOfTheJournal drives the report +// that follows a failed self-check. It writes at error level, so the journal +// keeps it and the operator text must stay off the line. +func Test_logUpstreamPingFailuresKeepsACustomKeyOutOfTheJournal(t *testing.T) { + logs := captureDebugMainLog(t) + _, uc := customProbeUpstream() + origPing := upstreamPingFn + t.Cleanup(func() { upstreamPingFn = origPing }) + upstreamPingFn = func(*ctrld.UpstreamConfig) error { return errors.New("ping failed") } + + logUpstreamPingFailures(map[string]*ctrld.UpstreamConfig{probeSecret: uc}) + + retained := retainedProbeLines(t, logs) + if len(retained) != 1 { + t.Fatalf("retained lines: got %d, want 1", len(retained)) + } + wantField(t, retained[0], "upstream", upstreamPrefix+"custom") + wantNoProbeSecret(t, logs) + if !strings.Contains(logs.String(), probeSecret) { + t.Fatal("the operator text must stay at debug level for diagnosis") + } +} diff --git a/docs/network-recovery-diagnostics.md b/docs/network-recovery-diagnostics.md index d676ae1c..1e019478 100644 --- a/docs/network-recovery-diagnostics.md +++ b/docs/network-recovery-diagnostics.md @@ -68,7 +68,9 @@ The two routes and the two gateways come from the route table of macOS, one rout Windows and Linux read no route table in this release. They fill `default_route_v4` from the network monitor and leave `default_route_v6`, `gateway_v4`, and `gateway_v6` empty. `link_type` is one of `wifi`, `ethernet`, `usb_tether`, `tunnel`, and `unknown`. -macOS reads it from the name of the hardware port. Linux reads `wifi`, `ethernet`, `usb_tether`, and `tunnel` from the kernel. Windows names a tunnel and leaves the other links unknown. +macOS reads it from the name of the hardware port. Linux reads `wifi`, `ethernet`, `usb_tether`, and `tunnel` from the kernel. +Windows reads the description of the adapter in place of a port name. The words Ethernet, LAN, and Thunderbolt win over the tunnel class on every platform, so a VPN adapter with a description such as `SSL VPN Virtual Ethernet Adapter` reads as `ethernet`. +Windows gives `tunnel` to the other VPN adapters. The other links are `unknown`. The field `service` names a macOS network service. It stays empty on Windows and on Linux. `tethered` is true on every platform when the interface of the default route holds an address of the range 172.20.10.0/28. It is also true when the default gateway is in that range, and when the hardware port of the default route is `iPhone USB`. @@ -85,7 +87,7 @@ The trigger `start` always writes. A noise delta changes interfaces of the noise class only, and it does not move the default route. The noise class holds `awdl*` and `llw*` on macOS. -On Linux it holds the names `docker*`, `veth*`, `virbr*`, and `br-*`, plus the devices of the tun and tap types. +On Linux it holds the names `docker*`, `veth*`, `virbr*`, and `br-*`. A device of the tun or tap type is a tunnel, not a noise interface. On Windows it holds the virtual adapters without a connector. A Windows adapter whose description names TAP, Wintun, WireGuard, or VPN is a tunnel, not a noise interface. On Linux and on Windows, an interface that holds a global address on either side of the delta is never noise. A delta is also not noise when it flips `have_v4` or `have_v6`, or when it carries a time jump. diff --git a/internal/controld/config.go b/internal/controld/config.go index aad670ab..adf83a7a 100644 --- a/internal/controld/config.go +++ b/internal/controld/config.go @@ -10,6 +10,7 @@ import ( "io" "net" "net/http" + "os" "runtime" "strings" "time" @@ -289,10 +290,18 @@ func SendLogs(ctx context.Context, lr *LogsRequest, cdDev bool) error { if cdDev { apiUrl = logURLDev } - req, err := http.NewRequestWithContext(ctx, "POST", apiUrl, lr.Data) + body, size, getBody, cleanup, err := spoolLogBody(lr.Data) + if err != nil { + return err + } + defer cleanup() + + req, err := http.NewRequestWithContext(ctx, "POST", apiUrl, body) if err != nil { return fmt.Errorf("http.NewRequest: %w", err) } + req.ContentLength = size + req.GetBody = getBody q := req.URL.Query() q.Set("uid", lr.UID) req.URL.RawQuery = q.Encode() @@ -418,6 +427,15 @@ func doWithFallback(client *http.Client, req *http.Request, apiIp string) (*http ipReq := req.Clone(req.Context()) ipReq.Host = apiIp ipReq.URL.Host = apiIp + // The first attempt consumed the body. A request that can give its body + // again sends the whole body a second time. + if req.GetBody != nil { + body, bodyErr := req.GetBody() + if bodyErr != nil { + return nil, fmt.Errorf("request failed: %w; fallback to direct ip %s failed: %w", err, apiIp, bodyErr) + } + ipReq.Body = body + } resp, fallbackErr := client.Do(ipReq) if fallbackErr != nil { return nil, fmt.Errorf("request failed: %w; fallback to direct ip %s failed: %w", err, apiIp, fallbackErr) @@ -425,6 +443,34 @@ func doWithFallback(client *http.Client, req *http.Request, apiIp string) (*http return resp, nil } +// spoolLogBody copies an upload to a temporary file, so the request can go +// out a second time to the direct IP of the API. A log reader yields its +// bytes once, and the first attempt consumes them. The caller runs cleanup +// after the response arrived. +func spoolLogBody(data io.Reader) (body *os.File, size int64, getBody func() (io.ReadCloser, error), cleanup func(), err error) { + f, err := os.CreateTemp("", "ctrld-log-upload-*") + if err != nil { + return nil, 0, nil, nil, fmt.Errorf("creating the upload spool: %w", err) + } + cleanup = func() { + _ = f.Close() + _ = os.Remove(f.Name()) + } + size, err = io.Copy(f, data) + if err != nil { + cleanup() + return nil, 0, nil, nil, fmt.Errorf("writing the upload spool: %w", err) + } + if _, err := f.Seek(0, io.SeekStart); err != nil { + cleanup() + return nil, 0, nil, nil, fmt.Errorf("rewinding the upload spool: %w", err) + } + // The transport closes the body of a failed attempt, so the second body + // is a new handle on the same file. + getBody = func() (io.ReadCloser, error) { return os.Open(f.Name()) } + return f, size, getBody, cleanup, nil +} + // apiServerIP returns the direct IP to connect to API server. func apiServerIP(cdDev bool) string { if cdDev { diff --git a/internal/controld/log_upload_test.go b/internal/controld/log_upload_test.go new file mode 100644 index 00000000..5c9e5f7d --- /dev/null +++ b/internal/controld/log_upload_test.go @@ -0,0 +1,49 @@ +package controld + +import ( + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" +) + +// TestDoWithFallbackResendsTheSpooledBody drives the two-attempt upload: the +// first attempt fails on a closed port and the fallback to the direct IP must +// carry the whole body, which the first attempt consumed. +func TestDoWithFallbackResendsTheSpooledBody(t *testing.T) { + payload := strings.Repeat("log line\n", 4096) + var got string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + b, _ := io.ReadAll(r.Body) + got = string(b) + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + body, size, getBody, cleanup, err := spoolLogBody(strings.NewReader(payload)) + if err != nil { + t.Fatalf("spoolLogBody: %v", err) + } + defer cleanup() + if size != int64(len(payload)) { + t.Fatalf("size = %d, want %d", size, len(payload)) + } + // Port 1 refuses the connection, so the first attempt fails at once. + req, err := http.NewRequestWithContext(context.Background(), http.MethodPost, "http://127.0.0.1:1/logs", body) + if err != nil { + t.Fatalf("NewRequest: %v", err) + } + req.ContentLength = size + req.GetBody = getBody + + resp, err := doWithFallback(srv.Client(), req, strings.TrimPrefix(srv.URL, "http://")) + if err != nil { + t.Fatalf("doWithFallback: %v", err) + } + resp.Body.Close() + if got != payload { + t.Fatalf("the fallback carried %d bytes, want %d", len(got), len(payload)) + } +} From 72507f3e2c0b145c868c543aaac86f0c1cecf7de Mon Sep 17 00:00:00 2001 From: Dev Scribe Date: Mon, 21 Sep 2026 12:29:10 -0400 Subject: [PATCH 18/29] test(cli): verify closed log files portably on Windows Use Write rather than Stat to verify retained file handles are closed. Windows Stat returns ERROR_INVALID_HANDLE instead of os.ErrClosed. Retain final-cleanup content assertions and strict closed-error matching. --- cmd/cli/internal_log_shutdown_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/cli/internal_log_shutdown_test.go b/cmd/cli/internal_log_shutdown_test.go index 56161058..8f7b0e11 100644 --- a/cmd/cli/internal_log_shutdown_test.go +++ b/cmd/cli/internal_log_shutdown_test.go @@ -36,7 +36,7 @@ func TestFinishRunClosesInternalFilesWithoutServiceStop(t *testing.T) { close(p.runDone) p.finishRun() for i, f := range files { - if _, err := f.Stat(); !errors.Is(err, os.ErrClosed) { + if _, err := f.Write([]byte("unexpected write\n")); !errors.Is(err, os.ErrClosed) { t.Errorf("file %d is not closed: %v", i, err) } b, err := os.ReadFile(paths[i]) From c7048f71b1ad50d5c49f91c76d62692dc94a030e Mon Sep 17 00:00:00 2001 From: Dev Scribe Date: Wed, 23 Sep 2026 07:49:40 +0000 Subject: [PATCH 19/29] Scope OS-only recovery and retain DNS-target diagnostics --- cmd/cli/dns_intercept_darwin.go | 2 + cmd/cli/dns_proxy.go | 79 ++++- cmd/cli/dns_target_darwin.go | 38 ++- cmd/cli/dns_target_darwin_test.go | 2 +- cmd/cli/dns_target_diagnostic.go | 126 +++++++ cmd/cli/dns_target_diagnostic_darwin_test.go | 272 ++++++++++++++++ cmd/cli/dns_target_diagnostic_test.go | 200 ++++++++++++ cmd/cli/prog.go | 6 + cmd/cli/recovery_scope_test.go | 326 +++++++++++++++++++ cmd/cli/recovery_skip_test.go | 147 +++++++++ docs/network-recovery-diagnostics.md | 42 ++- nameservers_darwin.go | 5 +- 12 files changed, 1224 insertions(+), 21 deletions(-) create mode 100644 cmd/cli/dns_target_diagnostic.go create mode 100644 cmd/cli/dns_target_diagnostic_darwin_test.go create mode 100644 cmd/cli/dns_target_diagnostic_test.go create mode 100644 cmd/cli/recovery_scope_test.go create mode 100644 cmd/cli/recovery_skip_test.go diff --git a/cmd/cli/dns_intercept_darwin.go b/cmd/cli/dns_intercept_darwin.go index 2402f3e8..77b911ef 100644 --- a/cmd/cli/dns_intercept_darwin.go +++ b/cmd/cli/dns_intercept_darwin.go @@ -108,6 +108,8 @@ const ( type pfState struct { anchorFile string anchorName string + // Serialized by prog.interceptDNSTargetMu; bounded to one failed decision. + targetDiagnostic dnsTargetDecisionDiagnostic } // ensureCtrldGroup creates the _ctrld system group if it doesn't exist and returns its GID. diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index f16ae8ab..d2099ee0 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -1009,8 +1009,8 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { } else { reason = RecoveryReasonRegularFailure } - mainLog.Load().Debug().Msgf("No healthy upstreams, triggering recovery with reason: %v", reason) - go p.handleRecovery(reason) + mainLog.Load().Debug().Msgf("Selected upstreams unavailable; requesting recovery with reason: %v", reason) + go queryRecoveryFn(p, reason) } else { mainLog.Load().Debug().Msg("Recovery already in progress; skipping duplicate trigger from down detection") } @@ -2361,6 +2361,26 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u mainLog.Load().Debug().Uint64("transition_id", transitionID).Msg("Recovery skipped: transition superseded") return } + // Admission and completion use the same pool. A failed OS-only policy + // is not evidence that configured general DNS is unavailable. Check here, + // rather than only at the query, because a queued trigger can arrive late. + upstreams := p.buildRecoveryUpstreams(reason) + upstreamNames := make([]string, 0, len(upstreams)) + for name := range upstreams { + upstreamNames = append(upstreamNames, name) + } + slices.Sort(upstreamNames) + if reason == RecoveryReasonOSFailure { + if _, osOnly := upstreams[upstreamOS]; !osOnly { + for _, name := range upstreamNames { + if !p.um.isDown(name) { + p.networkSourceMu.Unlock() + p.refreshOSResolverAfterRecoverySkip(name) + return + } + } + } + } recoveryCtx, gen, interceptRecovery, ok := p.beginRecovery(reason) p.networkSourceMu.Unlock() if !ok { @@ -2370,7 +2390,8 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u return } diagnostic := recoveryDiagnostic{transitionID: transitionID, generation: gen, reason: reason, started: time.Now()} - journal(diagnostic.event(mainLog.Load().Info())).Bool("intercept", interceptRecovery).Msg("Recovery begin") + journal(diagnostic.event(mainLog.Load().Info())).Bool("intercept", interceptRecovery). + Strs("probe_upstreams", journalUpstreamNames(upstreamNames)).Msg("Recovery begin") p.logNetworkSnapshot("recovery_begin") targetBefore := p.interceptTargetSnapshot() outcome := recoveryOutcomeSuperseded @@ -2464,10 +2485,7 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u } } - // Build upstream map based on the recovery reason. - upstreams := p.buildRecoveryUpstreams(reason) - - // Wait indefinitely until one of the upstreams recovers. + // Wait for the same upstream pool used to admit this recovery. recovered, err := waitForUpstreamRecoveryFn(p, recoveryCtx, upstreams, &diagnostic) if err != nil { if p.recoveryGen.Load() == gen { @@ -2538,10 +2556,38 @@ func (p *prog) handleRecoveryForTransition(reason RecoveryReason, transitionID u } } +// refreshOSResolverAfterRecoverySkip repairs stale OS resolver discovery without +// enabling bypass, changing host DNS, or clearing the monitor's failure state. +// Only a successful query establishes that the OS resolver recovered. +func (p *prog) refreshOSResolverAfterRecoverySkip(healthy string) { + if !p.osRecoveryRefreshMu.TryLock() { + return + } + defer p.osRecoveryRefreshMu.Unlock() + + now := networkEventsNowFn() + if p.osRecoverySkipLogAt.IsZero() || healthy != p.osRecoverySkipUpstream || now.Sub(p.osRecoverySkipLogAt) >= 5*time.Minute { + journal(mainLog.Load().Info()).Str("recovery_reason", recoveryReasonName(RecoveryReasonOSFailure)). + Str("healthy_upstream", journalUpstreamName(healthy)). + Msg("Recovery skipped: configured upstream is not marked down") + p.osRecoverySkipLogAt = now + p.osRecoverySkipUpstream = healthy + } + // Failed policy queries can arrive concurrently and at high volume. Retry + // discovery on later traffic, but never run overlapping or per-query reads. + if !p.osRecoveryRefreshAt.IsZero() && now.Sub(p.osRecoveryRefreshAt) < upstreamDownDelay { + return + } + p.osRecoveryRefreshAt = now + initializeOsResolverWithSystemNameserversFn(true, recoveryResolverReason) +} + // waitForUpstreamRecoveryFn is the seam of the upstream probe. A test drives // the whole recovery flow without a query to a real upstream. var waitForUpstreamRecoveryFn = (*prog).waitForUpstreamRecovery +var queryRecoveryFn = (*prog).handleRecovery + // waitForUpstreamRecovery checks the provided upstreams concurrently until one recovers. // It returns the name of the recovered upstream or an error if the check times out. func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string]*ctrld.UpstreamConfig, diagnostic *recoveryDiagnostic) (string, error) { @@ -2572,7 +2618,7 @@ func (p *prog) waitForUpstreamRecovery(ctx context.Context, upstreams map[string return default: attempts++ - // checkUpstreamOnce will reset any failure counters on success. + // The owning recovery resets the monitor after this probe succeeds. err := p.checkUpstreamOnce(name, uc, &failures) if err == nil || errors.Is(err, errOsHealthcheckSuppressed) { mainLog.Load().Debug().Msgf("Upstream %s recovered successfully", name) @@ -2650,17 +2696,26 @@ func sleepWithContext(ctx context.Context, d time.Duration) bool { } // buildRecoveryUpstreams constructs the map of upstream configurations to test. -// For OS failures we supply the manual OS resolver upstream configuration. -// For network change or regular failure we use the upstreams defined in p.cfg (ignoring OS). +// OS failures use configured non-OS upstreams for both admission and recovery, +// excluding generated Internal Domain resolvers. OS-only configurations retain +// the OS recovery path. Other recovery reasons retain their configured pool. func (p *prog) buildRecoveryUpstreams(reason RecoveryReason) map[string]*ctrld.UpstreamConfig { upstreams := make(map[string]*ctrld.UpstreamConfig) switch reason { case RecoveryReasonOSFailure: - upstreams[upstreamOS] = osUpstreamConfig + for k, uc := range p.cfg.Upstream { + name := upstreamPrefix + k + if uc != nil && uc.Type != ctrld.ResolverTypeOS && !isInternalDomainUpstream(name) { + upstreams[name] = uc + } + } + if len(upstreams) == 0 { + upstreams[upstreamOS] = osUpstreamConfig + } case RecoveryReasonNetworkChange, RecoveryReasonRegularFailure: // Use all configured upstreams except any OS type. for k, uc := range p.cfg.Upstream { - if uc.Type != ctrld.ResolverTypeOS { + if uc != nil && uc.Type != ctrld.ResolverTypeOS { upstreams[upstreamPrefix+k] = uc } } diff --git a/cmd/cli/dns_target_darwin.go b/cmd/cli/dns_target_darwin.go index ca8b00eb..28d68447 100644 --- a/cmd/cli/dns_target_darwin.go +++ b/cmd/cli/dns_target_darwin.go @@ -101,29 +101,52 @@ func (p *prog) ensureInterceptDNSTarget(systemDiscovery []string) { if !dnsIntercept || p.dnsInterceptState == nil { return } + p.interceptDNSTargetMu.Lock() + defer p.interceptDNSTargetMu.Unlock() + p.loadInterceptDNSTargetStateLocked() + var diagnostic *dnsTargetDecisionDiagnostic + if state, ok := p.dnsInterceptState.(*pfState); ok { + diagnostic = &state.targetDiagnostic + } + decision := dnsTargetDecisionContext{ + ownership: p.interceptDNSTargetOwnershipLocked(), + generation: p.recoveryGen.Load(), + transitionID: p.networkAcceptedGen.Load(), + } + // Only completed discovery resolves an outstanding diagnostic. A resolution + // describes the decision and observed ownership, not successful DNS repair. + resolvedReason := "" + defer func() { + if resolvedReason != "" { + diagnostic.resolved(decision, p.interceptDNSTargetOwnershipLocked(), resolvedReason) + } + }() if systemDiscovery == nil { + diagnostic.failed(decision, "system_discovery", nil) mainLog.Load().Debug().Msg("intercept DNS target: system DNS discovery was not performed; not changing DNS") return } - p.interceptDNSTargetMu.Lock() - defer p.interceptDNSTargetMu.Unlock() - p.loadInterceptDNSTargetStateLocked() drIfaceName, err := interceptDefaultRouteInterfaceFn() + decision.iface = drIfaceName if err != nil || drIfaceName == "" { // Mid-transition with no default route; the next recovery decides. + diagnostic.failed(decision, "default_route", err) return } iface, err := interceptInterfaceByNameFn(drIfaceName) if err != nil || iface == nil { + diagnostic.failed(decision, "interface_lookup", err) return } // Resolve the network service name (e.g. en5 -> "iPhone USB") so // networksetup operates on the right service. if _, err := interceptPatchNetIfaceNameFn(iface); err != nil { + diagnostic.failed(decision, "service_lookup", err) mainLog.Load().Debug().Err(err).Msgf("intercept DNS target: could not resolve network service for %s", drIfaceName) return } + decision.service = iface.Name staticDNS, err := interceptCurrentStaticDNSFn(iface) if err != nil { @@ -131,6 +154,7 @@ func (p *prog) ensureInterceptDNSTarget(systemDiscovery []string) { // networksetup cannot address them, ctrld never writes to them, and // any target set on the underlying physical service stays in place — // still correct while ctrld runs. + diagnostic.failed(decision, "static_dns", err) mainLog.Load().Debug().Err(err).Msgf("intercept DNS target: could not read static DNS for %q", iface.Name) return } @@ -141,22 +165,26 @@ func (p *prog) ensureInterceptDNSTarget(systemDiscovery []string) { staticDNS = filterOwnTarget(staticDNS, p.interceptDNSTargetSetValue) } if hasIPv4DNS(staticDNS) { + resolvedReason = "static_ipv4_dns" p.removeInterceptDNSTargetLocked("network has usable static IPv4 DNS") return } routeDHCPDNS, err := interceptDHCPNameserversForInterfaceFn(drIfaceName) if err != nil { + diagnostic.failed(decision, "dhcp_dns", err) mainLog.Load().Debug().Err(err).Msgf("intercept DNS target: could not read DHCP DNS for default-route service %q", iface.Name) return } if hasIPv4DNS(routeDHCPDNS) { + resolvedReason = "dhcp_ipv4_dns" // The default-route service regained DHCP option 6. Remove a target // previously set on this or another service. p.removeInterceptDNSTargetLocked("network has usable DHCP IPv4 DNS") return } + resolvedReason = "dns_less_network" target := p.interceptDNSTargetValue() if p.interceptDNSTargetService == iface.Name && p.interceptDNSTargetSetValue == target { return // already set on this service @@ -232,6 +260,10 @@ func (p *prog) removeInterceptDNSTargetLocked(reason string) { Msgf("intercept DNS target: removed %s from %q (%s)", val, svc, reason) } +func (p *prog) interceptDNSTargetOwnershipLocked() dnsTargetOwnership { + return dnsTargetOwnership{p.interceptDNSTargetService, p.interceptDNSTargetSetValue} +} + func (p *prog) clearInterceptDNSTargetStateLocked() { p.setInterceptDNSTargetLocked("", "") p.persistInterceptDNSTargetStateLocked() diff --git a/cmd/cli/dns_target_darwin_test.go b/cmd/cli/dns_target_darwin_test.go index a04f785d..0bdb1818 100644 --- a/cmd/cli/dns_target_darwin_test.go +++ b/cmd/cli/dns_target_darwin_test.go @@ -117,7 +117,7 @@ func newInterceptTargetProg() *prog { cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{ "0": {IP: "127.0.0.1", Port: 5354}, }}, - dnsInterceptState: &interceptStateStub{}, + dnsInterceptState: &pfState{}, } } diff --git a/cmd/cli/dns_target_diagnostic.go b/cmd/cli/dns_target_diagnostic.go new file mode 100644 index 00000000..7c96e057 --- /dev/null +++ b/cmd/cli/dns_target_diagnostic.go @@ -0,0 +1,126 @@ +package cli + +import ( + "context" + "errors" + "net" + "os" + "os/exec" + + "github.com/rs/zerolog" +) + +// dnsTargetDecisionDiagnostic keeps one outstanding discovery failure, not a +// history keyed by interface or recovery generation. The target mutex serializes +// access. Correlation IDs are sampled before discovery, never after a slow read. +// A successful read (including empty DNS) closes the failure; it does not claim +// that a subsequent DNS write succeeded. repeat_count counts suppressed reads +// across the whole failure episode, including condition changes; first retains +// its original correlation IDs until discovery succeeds. ID changes alone do +// not reopen the condition or turn periodic reconciliation into a log flood. +type dnsTargetDecisionDiagnostic struct { + pending bool + condition dnsTargetFailureCondition + first dnsTargetDecisionContext + repeats uint64 +} + +type dnsTargetOwnership struct { + service string + value string +} + +type dnsTargetDecisionContext struct { + iface string + service string + ownership dnsTargetOwnership + generation uint64 + transitionID uint64 +} + +type dnsTargetFailureCondition struct { + iface string + service string + ownership dnsTargetOwnership + stage string + class string +} + +// dnsTargetReadErrorClass deliberately never serializes command output or error +// text. Classify wrapped causes by type; unknown errors remain read_failed and +// must never be guessed to mean "no DNS". +func dnsTargetReadErrorClass(err error) string { + var networkError net.Error + var exitError *exec.ExitError + switch { + case err == nil: + return "unavailable" + case errors.Is(err, context.DeadlineExceeded), errors.As(err, &networkError) && networkError.Timeout(): + return "timeout" + case errors.Is(err, context.Canceled): + return "canceled" + case errors.Is(err, os.ErrPermission): + return "permission_denied" + case errors.As(err, &exitError): + return "command_failed" + default: + return "read_failed" + } +} + +func dnsTargetDecisionEvent(e *zerolog.Event, c dnsTargetDecisionContext, after dnsTargetOwnership) *zerolog.Event { + action := "unchanged" + if c.ownership != after { + switch { + case after.service == "": + action = "removed" + case c.ownership.service == "": + action = "set" + default: + action = "changed" + } + } + return journal(e).Str("interface", c.iface).Str("service", c.service). + Uint64("recovery_generation", c.generation).Uint64("transition_id", c.transitionID). + Str("action", action). + Bool("ownership_before", c.ownership.service != "").Bool("ownership_after", after.service != ""). + Str("owned_service_before", c.ownership.service).Str("owned_service_after", after.service). + Str("target_before", c.ownership.value).Str("target_after", after.value) +} + +func (d *dnsTargetDecisionDiagnostic) failed(c dnsTargetDecisionContext, stage string, err error) { + if d == nil { + return + } + condition := dnsTargetFailureCondition{c.iface, c.service, c.ownership, stage, dnsTargetReadErrorClass(err)} + if d.pending && d.condition == condition { + // Saturate rather than wrap during an arbitrarily long failure. + if d.repeats != ^uint64(0) { + d.repeats++ + } + return + } + if !d.pending { + d.first = c + } + d.pending = true + d.condition = condition + dnsTargetDecisionEvent(mainLog.Load().Warn(), c, c.ownership). + Str("outcome", "discovery_failed").Str("stage", stage).Str("error_class", condition.class). + Uint64("repeat_count", d.repeats). + Uint64("failure_recovery_generation", d.first.generation).Uint64("failure_transition_id", d.first.transitionID). + Msg("intercept DNS target: decision unavailable; DNS unchanged") +} + +func (d *dnsTargetDecisionDiagnostic) resolved(c dnsTargetDecisionContext, after dnsTargetOwnership, reason string) { + if d == nil || !d.pending { + return + } + dnsTargetDecisionEvent(mainLog.Load().Info(), c, after). + Str("outcome", "decision_available").Str("reason", reason). + Str("stage", d.condition.stage).Str("error_class", d.condition.class). + Uint64("repeat_count", d.repeats). + Uint64("failure_recovery_generation", d.first.generation).Uint64("failure_transition_id", d.first.transitionID). + Msg("intercept DNS target: decision available again") + *d = dnsTargetDecisionDiagnostic{} +} diff --git a/cmd/cli/dns_target_diagnostic_darwin_test.go b/cmd/cli/dns_target_diagnostic_darwin_test.go new file mode 100644 index 00000000..c83f2984 --- /dev/null +++ b/cmd/cli/dns_target_diagnostic_darwin_test.go @@ -0,0 +1,272 @@ +//go:build darwin + +package cli + +import ( + "encoding/json" + "errors" + "net" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +func TestEnsureInterceptDNSTargetFailedDecision(t *testing.T) { + for _, owned := range []bool{false, true} { + name := "without target" + if owned { + name = "existing target" + } + t.Run(name, func(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + p.recoveryGen.Store(3) + p.networkAcceptedGen.Store(7) + if owned { + persistInterceptTargetForTest(t, p, "Wi-Fi", "127.0.0.53") + h.dns["Wi-Fi"] = []string{"127.0.0.53"} + } + before := p.interceptDNSTargetOwnershipLocked() + fileBefore, _ := os.ReadFile(h.statePath) + // This is the first mutating call in the no-target path. Replacing it + // with a fatal stub makes any failure-to-absence regression safe/red. + save := interceptSaveCurrentStaticDNSFn + interceptSaveCurrentStaticDNSFn = func(*net.Interface) error { t.Fatal("discovery failure attempted a DNS backup"); return nil } + // Exercise the production getoption/getpacket wrapper, not a synthetic + // error. PATH contains only our fixture so no host ipconfig can run. + binDir := t.TempDir() + const script = `#!/bin/sh +case "$*" in + 'getoption en1 domain_name_server') code=17 ;; + 'getpacket en1') code=19 ;; + *) exit 99 ;; +esac +printf 'private.example command output\n' +printf 'private.example command stderr\n' >&2 +exit "$code" +` + if err := os.WriteFile(filepath.Join(binDir, "ipconfig"), []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", binDir) + dhcpStub := interceptDHCPNameserversForInterfaceFn + interceptDHCPNameserversForInterfaceFn = func(iface string) ([]string, error) { + servers, err := ctrld.DHCPNameserversForInterface(iface) + h.dhcpErr = err + return servers, err + } + for range 4 { + p.ensureInterceptDNSTarget([]string{}) + } + wrapped, ok := h.dhcpErr.(interface{ Unwrap() []error }) + if !ok || len(wrapped.Unwrap()) != 2 { + t.Fatalf("DHCP error did not preserve both command causes: %v", h.dhcpErr) + } + for i, cause := range wrapped.Unwrap() { + var exitErr *exec.ExitError + if !errors.As(cause, &exitErr) || exitErr.ExitCode() != []int{17, 19}[i] { + t.Fatalf("command cause %d = %v, want original exit error", i, cause) + } + if !strings.Contains(string(exitErr.Stderr), "private.example") { + t.Fatal("command fixture did not exercise private stderr") + } + } + if got := dnsTargetReadErrorClass(h.dhcpErr); got != "command_failed" { + t.Fatalf("production DHCP error class = %q", got) + } + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 { + t.Fatal("failed discovery mutated DNS") + } + if p.interceptDNSTargetOwnershipLocked() != before { + t.Fatal("failed discovery changed ownership") + } + fileAfter, _ := os.ReadFile(h.statePath) + if !slices.Equal(fileBefore, fileAfter) { + t.Fatal("failed discovery changed persisted ownership") + } + failed := oneRecoveryEvent(t, logs, dnsTargetFailedMessage) + wantField(t, failed, "stage", "dhcp_dns") + wantField(t, failed, "error_class", "command_failed") + wantField(t, failed, "ownership_before", owned) + wantField(t, failed, "ownership_after", owned) + wantField(t, failed, "action", "unchanged") + wantField(t, failed, "service", "Wi-Fi") + wantField(t, failed, "interface", "en1") + wantField(t, failed, "journal", true) + detail := jsonLogEvents(t, logs, `intercept DNS target: could not read DHCP DNS for default-route service "Wi-Fi"`) + if len(detail) != 4 { + t.Fatal("detailed discovery errors disappeared from the debug stream") + } + wantField(t, detail[0], "level", "debug") + wantField(t, detail[0], "error", h.dhcpErr.Error()) + + // Successful empty discovery is different from failure: only now may + // ctrld install the loopback target (or keep an existing one). + interceptSaveCurrentStaticDNSFn = save + interceptDHCPNameserversForInterfaceFn = dhcpStub + h.dhcpErr = nil + p.recoveryGen.Store(4) + p.networkAcceptedGen.Store(8) + p.ensureInterceptDNSTarget([]string{}) + p.ensureInterceptDNSTarget([]string{}) + resolved := oneRecoveryEvent(t, logs, dnsTargetResolvedMessage) + wantField(t, resolved, "error_class", "command_failed") + for _, event := range []map[string]any{failed, resolved} { + encoded, err := json.Marshal(event) + if err != nil { + t.Fatal(err) + } + for _, raw := range []string{"error reading DHCP", "getoption", "getpacket", "exit status", "private.example"} { + if strings.Contains(string(encoded), raw) { + t.Fatalf("raw command error/output leaked into decision diagnostic: %s", encoded) + } + } + if _, ok := event["error"]; ok { + t.Fatal("decision diagnostic included a raw error field") + } + } + wantField(t, resolved, "reason", "dns_less_network") + wantField(t, resolved, "repeat_count", float64(3)) + wantField(t, resolved, "failure_recovery_generation", float64(3)) + wantField(t, resolved, "failure_transition_id", float64(7)) + wantField(t, resolved, "recovery_generation", float64(4)) + wantField(t, resolved, "transition_id", float64(8)) + wantField(t, resolved, "ownership_before", owned) + wantField(t, resolved, "ownership_after", true) + action := "set" + if owned { + action = "unchanged" + } + wantField(t, resolved, "action", action) + wantSets := 1 + if owned { + wantSets = 0 + } + if len(h.setCalls) != wantSets || len(h.resetCalls) != 0 { + t.Fatalf("set/reset = %v/%v", h.setCalls, h.resetCalls) + } + if !slices.Equal(h.dns["Wi-Fi"], []string{"127.0.0.53"}) { + t.Fatal("empty discovery did not preserve a target") + } + + // DHCP IPv4 arriving later must still remove it exactly once. + h.dhcp = []string{"192.0.2.1"} + p.ensureInterceptDNSTarget([]string{}) + p.ensureInterceptDNSTarget([]string{}) + if len(h.resetCalls) != 1 || p.interceptDNSTargetService != "" { + t.Fatal("later IPv4 DNS did not remove target once") + } + }) + } +} + +func TestEnsureInterceptDNSTargetFailureThenIPv4(t *testing.T) { + for _, static := range []bool{false, true} { + name := "dhcp" + if static { + name = "static" + } + t.Run(name, func(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Wi-Fi", "127.0.0.53") + h.dns["Wi-Fi"] = []string{"127.0.0.53"} + h.dhcpErr = errors.New("failed read") + p.ensureInterceptDNSTarget([]string{}) + h.dhcpErr = nil + h.dhcp = []string{"192.0.2.1"} + reason := "dhcp_ipv4_dns" + if static { + h.dns["Wi-Fi"] = []string{"192.0.2.1"} + reason = "static_ipv4_dns" + } + p.ensureInterceptDNSTarget([]string{}) + resolved := oneRecoveryEvent(t, logs, dnsTargetResolvedMessage) + wantField(t, resolved, "reason", reason) + wantField(t, resolved, "action", "removed") + wantField(t, resolved, "ownership_before", true) + wantField(t, resolved, "ownership_after", false) + if static && !slices.Equal(h.dns["Wi-Fi"], []string{"192.0.2.1"}) { + t.Fatal("external static DNS was changed") + } + }) + } +} + +func TestEnsureInterceptDNSTargetDiscoveryFailureStages(t *testing.T) { + for _, stage := range []string{"system_discovery", "default_route", "interface_lookup", "service_lookup", "static_dns"} { + t.Run(stage, func(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + input := []string{} + err := errors.New("synthetic read error") + switch stage { + case "system_discovery": + input = nil + case "default_route": + interceptDefaultRouteInterfaceFn = func() (string, error) { return "", err } + case "interface_lookup": + interceptInterfaceByNameFn = func(string) (*net.Interface, error) { return nil, err } + case "service_lookup": + interceptPatchNetIfaceNameFn = func(*net.Interface) (bool, error) { return false, err } + case "static_dns": + h.readErr = err + } + interceptSaveCurrentStaticDNSFn = func(*net.Interface) error { t.Fatal("failed discovery attempted backup"); return nil } + p.ensureInterceptDNSTarget(input) + p.ensureInterceptDNSTarget(input) + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 { + t.Fatal("failed discovery mutated DNS") + } + wantField(t, oneRecoveryEvent(t, logs, dnsTargetFailedMessage), "stage", stage) + }) + } +} + +func TestEnsureInterceptDNSTargetResolutionDoesNotClaimWriteSuccess(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + h.dhcpErr = errors.New("read failed") + p.ensureInterceptDNSTarget([]string{}) + h.dhcpErr = nil + h.setErr = errors.New("synthetic set failure") + p.ensureInterceptDNSTarget([]string{}) + event := oneRecoveryEvent(t, logs, dnsTargetResolvedMessage) + wantField(t, event, "outcome", "decision_available") + wantField(t, event, "reason", "dns_less_network") + wantField(t, event, "action", "unchanged") + wantField(t, event, "ownership_before", false) + wantField(t, event, "ownership_after", false) + if len(h.setCalls) != 1 || p.interceptDNSTargetService != "" { + t.Fatal("failed target write was not exercised or acquired ownership") + } +} + +func TestEnsureInterceptDNSTargetSamplesCorrelationBeforeRead(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + p.recoveryGen.Store(2) + p.networkAcceptedGen.Store(5) + interceptDHCPNameserversForInterfaceFn = func(string) ([]string, error) { + p.recoveryGen.Store(3) + p.networkAcceptedGen.Store(6) + return nil, errors.New("read failed") + } + p.ensureInterceptDNSTarget([]string{}) + event := oneRecoveryEvent(t, logs, dnsTargetFailedMessage) + wantField(t, event, "recovery_generation", float64(2)) + wantField(t, event, "transition_id", float64(5)) + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 { + t.Fatal("failed discovery mutated DNS") + } +} diff --git a/cmd/cli/dns_target_diagnostic_test.go b/cmd/cli/dns_target_diagnostic_test.go new file mode 100644 index 00000000..9a323dcd --- /dev/null +++ b/cmd/cli/dns_target_diagnostic_test.go @@ -0,0 +1,200 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +const ( + dnsTargetFailedMessage = "intercept DNS target: decision unavailable; DNS unchanged" + dnsTargetResolvedMessage = "intercept DNS target: decision available again" +) + +func TestDNSTargetReadErrorClass(t *testing.T) { + for _, tc := range []struct { + name string + err error + want string + }{ + {"missing", nil, "unavailable"}, + {"deadline", fmt.Errorf("wrapped: %w", context.DeadlineExceeded), "timeout"}, + {"canceled", context.Canceled, "canceled"}, + {"permission", &os.PathError{Op: "read", Path: "private", Err: os.ErrPermission}, "permission_denied"}, + {"command", &exec.ExitError{}, "command_failed"}, + {"wrapped command", fmt.Errorf("wrapped: %w", &exec.ExitError{}), "command_failed"}, + {"untyped error", errors.New("exit status 1"), "read_failed"}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := dnsTargetReadErrorClass(tc.err); got != tc.want { + t.Fatalf("class = %q, want %q", got, tc.want) + } + }) + } +} + +func TestDNSTargetDiagnosticFailureAndResolution(t *testing.T) { + for _, owned := range []bool{false, true} { + t.Run(fmt.Sprintf("owned=%t", owned), func(t *testing.T) { + logs := captureDebugMainLog(t) + d := dnsTargetDecisionDiagnostic{} + c := dnsTargetDecisionContext{iface: "en1", service: "Wi-Fi", generation: 3, transitionID: 7} + if owned { + c.ownership = dnsTargetOwnership{"Wi-Fi", "127.0.0.53"} + } + // A healthy pass must not turn the watchdog into a journal heartbeat. + d.resolved(c, c.ownership, "dns_less_network") + failure := errors.New("private.example must not be retained") + d.failed(c, "dhcp_dns", failure) + for range 100 { + c.generation++ + c.transitionID++ + d.failed(c, "dhcp_dns", failure) + } + failed := oneRecoveryEvent(t, logs, dnsTargetFailedMessage) + wantField(t, failed, "journal", true) + wantField(t, failed, "level", "warn") + wantField(t, failed, "interface", "en1") + wantField(t, failed, "service", "Wi-Fi") + wantField(t, failed, "stage", "dhcp_dns") + wantField(t, failed, "error_class", "read_failed") + wantField(t, failed, "action", "unchanged") + wantField(t, failed, "ownership_before", owned) + wantField(t, failed, "ownership_after", owned) + wantField(t, failed, "recovery_generation", float64(3)) + wantField(t, failed, "transition_id", float64(7)) + d.resolved(c, c.ownership, "dns_less_network") + d.resolved(c, c.ownership, "dns_less_network") + resolved := oneRecoveryEvent(t, logs, dnsTargetResolvedMessage) + wantField(t, resolved, "journal", true) + wantField(t, resolved, "level", "info") + wantField(t, resolved, "repeat_count", float64(100)) + wantField(t, resolved, "failure_recovery_generation", float64(3)) + wantField(t, resolved, "failure_transition_id", float64(7)) + wantField(t, resolved, "recovery_generation", float64(103)) + wantField(t, resolved, "transition_id", float64(107)) + if strings.Contains(logs.String(), "private.example") { + t.Fatal("raw read error leaked into diagnostics") + } + if d != (dnsTargetDecisionDiagnostic{}) { + t.Fatal("resolved diagnostic retained failure state") + } + if len(jsonLogEvents(t, logs, "")) != 2 { + t.Fatal("repeated failures or healthy decisions were logged") + } + }) + } +} + +func TestDNSTargetDiagnosticConditionChanges(t *testing.T) { + for _, change := range []string{"interface", "service", "ownership", "stage", "class"} { + t.Run(change, func(t *testing.T) { + logs := captureDebugMainLog(t) + d := dnsTargetDecisionDiagnostic{} + c := dnsTargetDecisionContext{iface: "en1", service: "Wi-Fi"} + err := errors.New("read failed") + stage := "dhcp_dns" + d.failed(c, stage, err) + d.failed(c, stage, err) + switch change { + case "interface": + c.iface = "en2" + case "service": + c.service = "USB" + case "ownership": + c.ownership = dnsTargetOwnership{"USB", "127.0.0.53"} + case "stage": + stage = "static_dns" + case "class": + err = context.DeadlineExceeded + } + d.failed(c, stage, err) + d.failed(c, stage, err) + if got := len(jsonLogEvents(t, logs, dnsTargetFailedMessage)); got != 2 { + t.Fatalf("condition-change failures = %d, want 2", got) + } + d.resolved(c, dnsTargetOwnership{}, "dhcp_ipv4_dns") + wantField(t, oneRecoveryEvent(t, logs, dnsTargetResolvedMessage), "repeat_count", float64(2)) + // A new episode must log again and reset the suppressed count. + d.failed(c, stage, err) + if d.repeats != 0 { + t.Fatal("new episode retained old repeat count") + } + }) + } +} + +func TestDNSTargetDiagnosticRepeatCountSaturates(t *testing.T) { + captureDebugMainLog(t) + d := dnsTargetDecisionDiagnostic{} + c := dnsTargetDecisionContext{} + d.failed(c, "dhcp_dns", nil) + d.repeats = ^uint64(0) + d.failed(c, "dhcp_dns", nil) + if d.repeats != ^uint64(0) { + t.Fatal("repeat count wrapped") + } +} + +func TestDNSTargetDiagnosticOwnershipActions(t *testing.T) { + owned := dnsTargetOwnership{"Wi-Fi", "127.0.0.53"} + for _, tc := range []struct { + action string + before, after dnsTargetOwnership + }{ + {"set", dnsTargetOwnership{}, owned}, + {"removed", owned, dnsTargetOwnership{}}, + {"changed", owned, dnsTargetOwnership{"USB", "127.0.0.53"}}, + {"unchanged", owned, owned}, + } { + t.Run(tc.action, func(t *testing.T) { + logs := captureDebugMainLog(t) + d := dnsTargetDecisionDiagnostic{} + c := dnsTargetDecisionContext{ownership: tc.before} + d.failed(c, "dhcp_dns", os.ErrPermission) + d.resolved(c, tc.after, "dns_less_network") + event := oneRecoveryEvent(t, logs, dnsTargetResolvedMessage) + wantField(t, event, "action", tc.action) + wantField(t, event, "owned_service_before", tc.before.service) + wantField(t, event, "owned_service_after", tc.after.service) + wantField(t, event, "target_before", tc.before.value) + wantField(t, event, "target_after", tc.after.value) + }) + } +} + +func TestDNSTargetDiagnosticSurvivesDebugTruncation(t *testing.T) { + p, _ := startInternalLogging(t) + useTestSendBudget(t, 1024) + d := dnsTargetDecisionDiagnostic{} + c := dnsTargetDecisionContext{iface: "en1", service: "Wi-Fi"} + d.failed(c, "dhcp_dns", errors.New("private.example")) + for range 5 { + d.failed(c, "dhcp_dns", errors.New("private.example")) + } + d.resolved(c, dnsTargetOwnership{"Wi-Fi", "127.0.0.53"}, "dns_less_network") + for range 40 { + mainLog.Load().Debug().Msg(strings.Repeat("debug filler", 20)) + } + start := time.Now() + debugPart, journalPart := splitUpload(t, readLogReader(t, p, false), start) + for _, message := range []string{dnsTargetFailedMessage, dnsTargetResolvedMessage} { + if strings.Contains(string(debugPart), message) { + t.Fatalf("fixture did not truncate %q from debug", message) + } + if strings.Count(string(journalPart), message) != 1 { + t.Fatalf("journal did not retain exactly one %q", message) + } + } + if !strings.Contains(string(journalPart), `"repeat_count":5`) { + t.Fatal("journal lost repeat count") + } + if strings.Contains(string(journalPart), "private.example") { + t.Fatal("raw error leaked into journal") + } +} diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index f07f127e..0a3cc85c 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -175,6 +175,12 @@ type prog struct { // only touches shared recovery state if it is still the newest (#597). recoveryGen atomic.Uint64 + // OS-only failures can refresh the resolver without owning global recovery. + osRecoveryRefreshMu sync.Mutex + osRecoveryRefreshAt time.Time + osRecoverySkipLogAt time.Time + osRecoverySkipUpstream string + // All deltas get an ID. Only accepted deltas replace recovery ownership. networkTransitionGen atomic.Uint64 networkAcceptedGen atomic.Uint64 diff --git a/cmd/cli/recovery_scope_test.go b/cmd/cli/recovery_scope_test.go new file mode 100644 index 00000000..fc294269 --- /dev/null +++ b/cmd/cli/recovery_scope_test.go @@ -0,0 +1,326 @@ +package cli + +import ( + "context" + "net" + "reflect" + "sync/atomic" + "testing" + "time" + + "github.com/miekg/dns" + ctrld "github.com/Control-D-Inc/ctrld" +) + +// Loopback transports exercise the real proxy and recovery loop without host +// DNS changes. The event harness intercepts the first platform mutation. +func scopeTestResolver(t *testing.T, answering *atomic.Bool, calls *atomic.Int64) *ctrld.UpstreamConfig { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + started := make(chan struct{}) + srv := &dns.Server{PacketConn: pc, Net: "udp", NotifyStartedFunc: func() { close(started) }, Handler: dns.HandlerFunc(func(w dns.ResponseWriter, m *dns.Msg) { + calls.Add(1) + if answering.Load() { + a := new(dns.Msg) + a.SetReply(m) + _ = w.WriteMsg(a) + } + })} + go func() { _ = srv.ActivateAndServe() }() + <-started + t.Cleanup(func() { _ = srv.Shutdown() }) + return &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy, Endpoint: pc.LocalAddr().String(), Timeout: 40} +} + +type scopeTestHarness struct { + *recoveryEventHarness + osAnswers, defaultAnswers atomic.Bool + osCalls, defaultCalls atomic.Int64 + finished chan struct{} +} + +func newScopeTestHarness(t *testing.T) *scopeTestHarness { + t.Helper() + h := &scopeTestHarness{recoveryEventHarness: newRecoveryEventHarness(t), finished: make(chan struct{}, 8)} + h.dhcpServers = nil + initializeOsResolverWithSystemNameserversFn = func(bool, string) ([]string, []string) { return nil, nil } + oldOS, oldQueryRecovery := osUpstreamConfig, queryRecoveryFn + t.Cleanup(func() { osUpstreamConfig, queryRecoveryFn = oldOS, oldQueryRecovery }) + osUpstreamConfig = scopeTestResolver(t, &h.osAnswers, &h.osCalls) + h.defaultAnswers.Store(true) + h.prog.cfg.Upstream = map[string]*ctrld.UpstreamConfig{"0": scopeTestResolver(t, &h.defaultAnswers, &h.defaultCalls)} + leak := true + h.prog.cfg.Service.LeakOnUpstreamFailure = &leak + h.prog.um = newUpstreamMonitor(h.prog.cfg) + h.prog.um.after = func(time.Duration, func()) {} + h.prog.um.clearRecovered(upstreamOS) + h.prog.um.clearRecovered("upstream.0") + queryRecoveryFn = func(p *prog, reason RecoveryReason) { + p.handleRecovery(reason) + h.finished <- struct{}{} + } + return h +} + +func (h *scopeTestHarness) query(t *testing.T, policy bool) *proxyResponse { + t.Helper() + name := "ordinary.example." + ufr := &upstreamForResult{upstreams: []string{"upstream.0"}, matched: true} + if policy { + name = "os-only.example." + lc := listenerWithRules(ctrld.Rule{"os-only.example": []string{}}) + ufr = h.prog.upstreamFor(context.Background(), "0", lc, &net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 53000}, "", "os-only.example") + if !ufr.matched || len(ufr.upstreams) != 0 { + t.Fatalf("wrong policy fixture: %+v", ufr) + } + } + m := new(dns.Msg) + m.SetQuestion(name, dns.TypeTXT) + return h.prog.proxy(context.Background(), &proxyRequest{msg: m, ufr: ufr}) +} + +func (h *scopeTestHarness) markOSDown(t *testing.T) { + t.Helper() + if got := h.query(t, true).answer.Rcode; got != dns.RcodeServerFailure { + t.Fatalf("OS fixture rcode=%d", got) + } + // Fire the real delayed-down callback without waiting ten seconds. + h.prog.um.markDownAfterDelay(upstreamOS) + if !h.prog.um.isDown(upstreamOS) { + t.Fatal("OS down precondition failed") + } +} + +func scopeWait(t *testing.T, ch <-chan struct{}) { + t.Helper() + select { + case <-ch: + case <-time.After(5 * time.Second): + t.Fatal("recovery handler did not return") + } +} + +func TestOSPolicyFailureDoesNotBypassHealthyDefault(t *testing.T) { + h := newScopeTestHarness(t) + if h.query(t, false).answer.Rcode != dns.RcodeSuccess { + t.Fatal("default fixture failed") + } + h.markOSDown(t) + for range 3 { + if h.query(t, true).answer.Rcode != dns.RcodeServerFailure { + t.Fatal("policy changed resolver") + } + scopeWait(t, h.finished) + if h.prog.recoveryBypass.Load() || h.prog.recoveryRunning.Load() { + t.Fatal("policy failure started global recovery") + } + } + before := h.osCalls.Load() + if h.query(t, false).answer.Rcode != dns.RcodeSuccess { + t.Fatal("default request failed") + } + if h.osCalls.Load() != before { + t.Fatal("unrelated request detoured through OS DNS") + } + if len(jsonLogEvents(t, h.logs, recoveryBeginMessage)) != 0 { + t.Fatal("rejected recovery touched host state") + } +} + +func TestQueuedOSFailureRechecksHealthyDefault(t *testing.T) { + h := newScopeTestHarness(t) + h.markOSDown(t) + h.prog.um.mu.Lock() + h.prog.um.markDown("upstream.0", 1, "test") + h.prog.um.mu.Unlock() + entered, proceed := make(chan struct{}), make(chan struct{}) + queryRecoveryFn = func(p *prog, reason RecoveryReason) { + close(entered) + <-proceed + p.handleRecovery(reason) + h.finished <- struct{}{} + } + h.query(t, true) + scopeWait(t, entered) + if h.query(t, false).answer.Rcode != dns.RcodeSuccess { + t.Fatal("default did not recover") + } + close(proceed) + scopeWait(t, h.finished) + if len(jsonLogEvents(t, h.logs, recoveryBeginMessage)) != 0 { + t.Fatal("late OS trigger ignored newer default success") + } +} + +func TestOSFailureRecoversThroughConfiguredPool(t *testing.T) { + h := newScopeTestHarness(t) + h.markOSDown(t) + h.prog.um.mu.Lock() + h.prog.um.markDown("upstream.0", 1, "test") + h.prog.um.mu.Unlock() + candidates := make(chan map[string]*ctrld.UpstreamConfig, 1) + proceed := make(chan struct{}) + waitForUpstreamRecoveryFn = func(p *prog, ctx context.Context, ups map[string]*ctrld.UpstreamConfig, diagnostic *recoveryDiagnostic) (string, error) { + candidates <- ups + <-proceed + return p.waitForUpstreamRecovery(ctx, ups, diagnostic) + } + h.query(t, true) + select { + case ups := <-candidates: + if len(ups) != 1 || ups["upstream.0"] != h.prog.cfg.Upstream["0"] { + t.Fatalf("wrong recovery pool: %v", ups) + } + case <-time.After(time.Second): + t.Fatal("general outage did not enter recovery") + } + if !h.prog.recoveryBypass.Load() { + t.Fatal("general outage did not enable bypass") + } + close(proceed) + scopeWait(t, h.finished) + if h.prog.recoveryBypass.Load() || h.prog.recoveryRunning.Load() { + t.Fatal("healthy default did not end recovery") + } + if !h.prog.um.isDown(upstreamOS) { + t.Fatal("fixture OS resolver unexpectedly recovered") + } +} + +func TestOSOnlyConfigurationRetainsOSRecovery(t *testing.T) { + h := newScopeTestHarness(t) + h.prog.cfg.Upstream = nil + h.markOSDown(t) + waitForUpstreamRecoveryFn = func(p *prog, ctx context.Context, ups map[string]*ctrld.UpstreamConfig, diagnostic *recoveryDiagnostic) (string, error) { + if len(ups) != 1 || ups[upstreamOS] != osUpstreamConfig { + return "", context.Canceled + } + h.osAnswers.Store(true) + return p.waitForUpstreamRecovery(ctx, ups, diagnostic) + } + h.query(t, true) + scopeWait(t, h.finished) + end := oneRecoveryEvent(t, h.logs, recoveryEndMessage) + wantField(t, end, "outcome", "completed") + wantField(t, end, "recovered_upstream", upstreamOS) + if h.prog.recoveryBypass.Load() || h.prog.um.isDown(upstreamOS) { + t.Fatal("OS-only recovery did not finish") + } +} + +func TestOSRecoveryRejectedBeforeHostWork(t *testing.T) { + for _, intercept := range []bool{false, true} { + t.Run(map[bool]string{false: "traditional", true: "intercept"}[intercept], func(t *testing.T) { + h := newScopeTestHarness(t) + dnsIntercept = intercept + oldReset := recoveryResetDNSFn + t.Cleanup(func() { recoveryResetDNSFn = oldReset }) + calls := 0 + recoveryResetDNSFn = func(*prog, bool, bool) { calls++ } + refreshes := 0 + initializeOsResolverWithSystemNameserversFn = func(bool, string) ([]string, []string) { refreshes++; return nil, nil } + ensureInterceptDNSTargetFn = func(*prog, []string) { calls++ } + waitForUpstreamRecoveryFn = func(*prog, context.Context, map[string]*ctrld.UpstreamConfig, *recoveryDiagnostic) (string, error) { + calls++ + return "", context.Canceled + } + before := h.prog.recoveryGen.Load() + h.prog.handleRecovery(RecoveryReasonOSFailure) + if refreshes != 1 || calls != 0 || h.prog.recoveryGen.Load() != before || h.prog.recoveryRunning.Load() || h.prog.recoveryBypass.Load() { + t.Fatal("rejected recovery failed to refresh only OS resolver discovery") + } + }) + } +} + +func TestOSRecoveryKeepsExistingOwner(t *testing.T) { + h := newScopeTestHarness(t) + ctx, gen, _, ok := h.prog.beginRecovery(RecoveryReasonRegularFailure) + if !ok { + t.Fatal("could not establish prior owner") + } + t.Cleanup(func() { h.prog.completeRecovery(gen) }) + h.prog.handleRecovery(RecoveryReasonOSFailure) + if ctx.Err() != nil || h.prog.recoveryGen.Load() != gen || !h.prog.recoveryOwnsState(gen) || !h.prog.recoveryBypass.Load() { + t.Fatal("rejected OS recovery displaced the active owner") + } +} + +func TestOSRecoveryConfiguredPoolDoesNotBlockRegularFailure(t *testing.T) { + h := newScopeTestHarness(t) + h.markOSDown(t) + h.prog.cfg.Upstream["unused"] = &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy, Endpoint: "127.0.0.1:1"} + h.prog.um.mu.Lock() + h.prog.um.markDown("upstream.0", 1, "test") + h.prog.um.mu.Unlock() + h.query(t, true) + scopeWait(t, h.finished) + if len(jsonLogEvents(t, h.logs, recoveryBeginMessage)) != 0 { + t.Fatal("OS admission ignored an unmarked configured candidate") + } + // Ordinary default failure still follows its original admission path. + h.defaultAnswers.Store(false) + h.query(t, false) + scopeWait(t, h.finished) + begin := oneRecoveryEvent(t, h.logs, recoveryBeginMessage) + wantField(t, begin, "recovery_reason", "upstream_failure") +} + +func TestOtherRecoveryPoolsRemainUnchanged(t *testing.T) { + remote := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeDOH} + internal := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy} + p := &prog{cfg: &ctrld.Config{Upstream: map[string]*ctrld.UpstreamConfig{ + "0": remote, "internal_0": internal, "1": {Type: ctrld.ResolverTypeOS}, "2": nil, + }}} + want := map[string]*ctrld.UpstreamConfig{"upstream.0": remote, "upstream.internal_0": internal} + for _, reason := range []RecoveryReason{RecoveryReasonRegularFailure, RecoveryReasonNetworkChange} { + if got := p.buildRecoveryUpstreams(reason); !reflect.DeepEqual(got, want) { + t.Fatalf("reason=%v got=%v want=%v", reason, got, want) + } + } +} + +func TestScopeHarnessRestoresRecoveryWait(t *testing.T) { + original := waitForUpstreamRecoveryFn + t.Cleanup(func() { waitForUpstreamRecoveryFn = original }) + waitForUpstreamRecoveryFn = func(*prog, context.Context, map[string]*ctrld.UpstreamConfig, *recoveryDiagnostic) (string, error) { + return "outer", nil + } + t.Run("overridden wait", func(t *testing.T) { + newScopeTestHarness(t) + waitForUpstreamRecoveryFn = func(*prog, context.Context, map[string]*ctrld.UpstreamConfig, *recoveryDiagnostic) (string, error) { + return "inner", nil + } + }) + got, err := waitForUpstreamRecoveryFn(nil, context.Background(), nil, nil) + if err != nil || got != "outer" { + t.Fatalf("composed harness leaked its wait function: got %q, %v", got, err) + } +} + +func TestOSRecoveryCandidates(t *testing.T) { + remote := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeDOH} + internal := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy} + osConfig := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeOS} + for _, tc := range []struct { + name string + configured map[string]*ctrld.UpstreamConfig + want map[string]*ctrld.UpstreamConfig + }{ + {"configured", map[string]*ctrld.UpstreamConfig{"0": remote, "internal_0": internal, "1": osConfig, "2": nil}, map[string]*ctrld.UpstreamConfig{"upstream.0": remote}}, + {"custom_policy_or_unused", map[string]*ctrld.UpstreamConfig{"0": remote, "unused": internal}, map[string]*ctrld.UpstreamConfig{"upstream.0": remote, "upstream.unused": internal}}, + {"only_os", map[string]*ctrld.UpstreamConfig{"0": osConfig}, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}}, + {"only_internal", map[string]*ctrld.UpstreamConfig{"internal_0": internal}, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}}, + {"empty", nil, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}}, + } { + t.Run(tc.name, func(t *testing.T) { + p := &prog{cfg: &ctrld.Config{Upstream: tc.configured}} + if got := p.buildRecoveryUpstreams(RecoveryReasonOSFailure); !reflect.DeepEqual(got, tc.want) { + t.Fatalf("got %v want %v", got, tc.want) + } + }) + } +} diff --git a/cmd/cli/recovery_skip_test.go b/cmd/cli/recovery_skip_test.go new file mode 100644 index 00000000..69cdc5d2 --- /dev/null +++ b/cmd/cli/recovery_skip_test.go @@ -0,0 +1,147 @@ +package cli + +import ( + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/miekg/dns" + ctrld "github.com/Control-D-Inc/ctrld" +) + +const recoverySkipMessage = "Recovery skipped: configured upstream is not marked down" + +func TestOSRecoverySkipRefreshesWithoutNetworkChange(t *testing.T) { + h := newScopeTestHarness(t) + h.markOSDown(t) + var refreshes atomic.Int64 + initializeOsResolverWithSystemNameserversFn = func(guard bool, reason string) ([]string, []string) { + if !guard || reason != recoveryResolverReason { + t.Error("refresh lost empty-discovery guard or reason") + } + refreshes.Add(1) + h.osAnswers.Store(true) + return nil, nil + } + // This query still fails; the asynchronously triggered refresh repairs + // discovery for the next policy query, without a network-change callback. + h.query(t, true) + scopeWait(t, h.finished) + if refreshes.Load() != 1 || !h.prog.um.isDown(upstreamOS) { + t.Fatal("refresh must run without claiming a successful OS response") + } + if h.query(t, true).answer.Rcode != dns.RcodeSuccess || h.prog.um.isDown(upstreamOS) { + t.Fatal("OS policy did not recover on its own resolver") + } + if h.prog.recoveryBypass.Load() || h.prog.recoveryRunning.Load() || h.prog.recoveryGen.Load() != 0 { + t.Fatal("local refresh acquired global recovery state") + } +} + +func TestOSRecoverySkipRefreshIsBounded(t *testing.T) { + h := newScopeTestHarness(t) + now := time.Unix(1000, 0) + oldNow := networkEventsNowFn + networkEventsNowFn = func() time.Time { return now } + t.Cleanup(func() { networkEventsNowFn = oldNow }) + calls := 0 + initializeOsResolverWithSystemNameserversFn = func(bool, string) ([]string, []string) { + calls++ + return nil, nil + } + for range 100 { + h.prog.handleRecovery(RecoveryReasonOSFailure) + } + if calls != 1 { + t.Fatalf("burst discovery calls=%d, want 1", calls) + } + now = now.Add(upstreamDownDelay - time.Nanosecond) + h.prog.handleRecovery(RecoveryReasonOSFailure) + if calls != 1 { + t.Fatal("discovery retried before the cooldown") + } + now = now.Add(time.Nanosecond) + h.prog.handleRecovery(RecoveryReasonOSFailure) + if calls != 2 { + t.Fatal("later OS failures cannot retry discovery") + } + event := oneRecoveryEvent(t, h.logs, recoverySkipMessage) + wantField(t, event, "journal", true) + wantField(t, event, "healthy_upstream", "upstream.0") + wantField(t, event, "recovery_reason", "os_resolver_failure") + now = now.Add(5 * time.Minute) + h.prog.handleRecovery(RecoveryReasonOSFailure) + if len(jsonLogEvents(t, h.logs, recoverySkipMessage)) != 2 { + t.Fatal("persistent skips lost the bounded journal heartbeat") + } +} + +func TestOSRecoverySkipCoalescesConcurrentRefresh(t *testing.T) { + h := newScopeTestHarness(t) + entered, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + var calls atomic.Int64 + initializeOsResolverWithSystemNameserversFn = func(bool, string) ([]string, []string) { + if calls.Add(1) == 1 { + close(entered) + } + <-release + return nil, nil + } + go func() { h.prog.handleRecovery(RecoveryReasonOSFailure); close(done) }() + scopeWait(t, entered) + other := make(chan struct{}) + go func() { h.prog.handleRecovery(RecoveryReasonOSFailure); close(other) }() + // Release the first worker even if coalescing regresses, to keep cleanup safe. + select { + case <-other: + case <-time.After(time.Second): + close(release) + scopeWait(t, done) + scopeWait(t, other) + t.Fatal("concurrent failure waited on discovery") + } + close(release) + scopeWait(t, done) + if calls.Load() != 1 { + t.Fatalf("overlapping discovery calls=%d", calls.Load()) + } +} + +func TestOSRecoverySkipJournalsChangedCandidatePrivately(t *testing.T) { + h := newScopeTestHarness(t) + h.prog.cfg.Upstream["private-token.example"] = &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy} + h.prog.handleRecovery(RecoveryReasonOSFailure) + h.prog.um.mu.Lock() + h.prog.um.markDown("upstream.0", 1, "test") + h.prog.um.mu.Unlock() + h.prog.handleRecovery(RecoveryReasonOSFailure) + events := jsonLogEvents(t, h.logs, recoverySkipMessage) + if len(events) != 2 { + t.Fatalf("candidate change events=%d", len(events)) + } + wantField(t, events[0], "healthy_upstream", "upstream.0") + wantField(t, events[1], "healthy_upstream", "upstream.custom") + if strings.Contains(h.logs.String(), "private-token.example") { + t.Fatal("operator-provided upstream key leaked") + } +} + +func TestOSRecoverySkipSurvivesDebugTruncation(t *testing.T) { + h := newScopeTestHarness(t) + p, _ := startInternalLogging(t) + useTestSendBudget(t, 1024) + for range 100 { + h.prog.handleRecovery(RecoveryReasonOSFailure) + } + for range 40 { + mainLog.Load().Debug().Msg(strings.Repeat("debug filler", 20)) + } + debugPart, journalPart := splitUpload(t, readLogReader(t, p, false), time.Now()) + if strings.Contains(string(debugPart), recoverySkipMessage) { + t.Fatal("fixture did not truncate the skip from debug") + } + if strings.Count(string(journalPart), recoverySkipMessage) != 1 || !strings.Contains(string(journalPart), `"healthy_upstream":"upstream.0"`) { + t.Fatal("journal did not retain the bounded skip and admission candidate") + } +} diff --git a/docs/network-recovery-diagnostics.md b/docs/network-recovery-diagnostics.md index 1e019478..ed02baaa 100644 --- a/docs/network-recovery-diagnostics.md +++ b/docs/network-recovery-diagnostics.md @@ -24,7 +24,8 @@ The table gives the trigger, the level, and the fields of each event. | `OS resolver set changed` | A nameserver read that differs from the read before | info | `source` (`scutil`, `dhcp`, `resolv.conf`, `unknown`), `before`, `after`, `default_route`, `reason` (`start`, `recovery`, `delayed_recheck`, `wake_probe`, `vpn_settle`, `unspecified`) | | `Reinitialized OS resolver with nameservers: …` | ctrld read the nameservers of the system again and built the OS resolver again | info | none, the nameservers are part of the message | | `DNS configuration changed` | A `scutil --dns` poll that finds a changed resolver, one event per resolver (macOS) | info | `nameservers`, `if_index`, `interface`, `scoped`, `action` (`added`, `changed`, `removed`), `flags`, `search_domain_count`, `order`, `reachable` | -| `Recovery begin` | The start of every recovery | info | `transition_id`, `recovery_generation`, `recovery_reason`, `intercept` | +| `Recovery begin` | The start of every recovery | info | `transition_id`, `recovery_generation`, `recovery_reason`, `intercept`, `probe_upstreams` | +| `Recovery skipped: configured upstream is not marked down` | An OS-only failure does not start global recovery because a configured candidate is not marked down. First/changed candidate, then at most once per 5 minutes for an unchanged candidate | info | `recovery_reason`, `healthy_upstream` | | `Recovery end` | The end of every recovery | info, warn for a canceled recovery and for a recovery with a failure | `transition_id`, `recovery_generation`, `recovery_reason`, `outcome`, `had_failure`, `duration_ms`, `recovered_upstream`, `dhcp_servers`, `dhcp_server_count`, `intercept_target_action`, `bypass_active` | | `Upstream … recovered …` | An upstream that was down answers again | info | `down_for_ms` | | `Recovery canceled with no successor …` | A recovery stops and no other recovery follows it | info | none | @@ -167,6 +168,45 @@ A later recovery cannot receive credit for an earlier probe. Update support runbooks and log searches to use the structured event. The cancellation event alone does not prove that interface DNS remains removed. +### OS-triggered recovery scope + +An OS-only policy failure does not establish that configured DNS is unavailable. +Before starting OS-triggered recovery, ctrld checks its configured non-OS upstreams, excluding generated Internal Domain resolvers. +If any candidate is not marked down, it leaves shared recovery state and DNS settings unchanged. +The skipped trigger still refreshes in-memory OS-resolver discovery, at most once per 10 seconds while OS failures continue. Concurrent triggers do not queue duplicate discovery work. +This read preserves the existing resolver when discovery yields no nameservers. It does not clear the OS failure state; a successful OS query does that. +It does not remove interface DNS, reload interception rules, or enable bypass. A network change is not required for this refresh. +`Recovery skipped: configured upstream is not marked down` enters the journal with `healthy_upstream`, the candidate that admitted the skip. Generated IDs are retained; operator-defined keys are redacted to `upstream.custom`. +An unchanged candidate is journaled at most once per 5 minutes, rather than once per failed query. A changed candidate is journaled immediately when no refresh is in progress. +If all candidates are down, it uses that same pool for recovery probes instead of waiting only for OS DNS. The recovery still refreshes OS discovery initially, but only configured candidates can end it; the OS-only periodic probe/reinitialization loop is not part of that pool. +With no such configured candidates, it retains OS-only recovery. +`Recovery begin` includes `probe_upstreams` so the exit condition is visible in the journal. + +This is a configured pool, not a reconstruction of every listener's effective default route. +Custom configurations can include policy-only or unused upstreams, and an untested upstream is not marked down. +Ordinary upstream-failure and network-change admission and candidate selection are unchanged. +A success racing with admission can still cause brief bypass; probing the configured pool prevents the old OS-only wait from persisting. +The original OS policy remains OS-routed. + +### DNS-target decision failures (macOS) + +`intercept DNS target: decision unavailable; DNS unchanged` retains the first discovery failure and changes to that condition. +Identical consecutive failures are suppressed, even if recovery IDs change. +A later `decision available again` event closes the episode and reports the cumulative `repeat_count`. +Do not add repeat counts from separate events in the same episode. + +These events include the interface/service, discovery stage, bounded error class, target ownership before/after, and correlation IDs captured before discovery. +`failure_recovery_generation` and `failure_transition_id` identify the episode's first failed decision. +Raw error detail remains in the debug stream, not the journal fields. +A stage can be `system_discovery`, `default_route`, `interface_lookup`, `service_lookup`, `static_dns`, or `dhcp_dns`. +An error class can be `unavailable`, `timeout`, `canceled`, `permission_denied`, `command_failed`, or `read_failed`. + +A DHCP error is not an empty DNS result and never authorizes a DNS write. +Resolution means discovery can make a decision again, not that a DNS write or client lookup succeeded. +`action` describes the change in tracked target ownership; it is not independent proof of an OS write. +The target's existing set/remove events still report those operations. +The journal retains these decisions separately from the bounded debug upload. + ## PF probes (macOS) `DNS intercept probe result` includes these fields: diff --git a/nameservers_darwin.go b/nameservers_darwin.go index d8ab7048..0133dc4e 100644 --- a/nameservers_darwin.go +++ b/nameservers_darwin.go @@ -112,10 +112,7 @@ func getDHCPNameservers(iface string) ([]string, error) { // domain_name_server (ip_mult): {192.168.1.1, 8.8.8.8} output, packetErr := exec.Command("ipconfig", "getpacket", iface).Output() if packetErr != nil { - if err != nil { - return nil, fmt.Errorf("error reading DHCP DNS option: getoption: %v; getpacket: %v", err, packetErr) - } - return nil, fmt.Errorf("error reading DHCP packet: %v", packetErr) + return nil, fmt.Errorf("error reading DHCP DNS option: getoption: %w; getpacket: %w", err, packetErr) } return parseDHCPPacketNameservers(output), nil } From e34fef31673744ccf1a3887c97078a1cac554847 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Wed, 23 Sep 2026 15:31:38 +0700 Subject: [PATCH 20/29] fix(cli): bound the DNS loop check probe by upstream timeout checkDnsLoop resolved with context.Background() and no dns.Client timeout, so the DNS client fell back to its own 2s default and the configured upstream timeout was ignored. The probes run serially on a one-minute ticker, so each unreachable local upstream stalled the loop for a fixed 2s no matter what the config asked for. This is most visible on Windows, where UDP to a closed loopback port does not surface an immediate refusal the way it does on Linux, so the probe waits out the full default. Apply the upstream timeout the way checkUpstreamOnce already does, and keep 2s for an upstream that configures none so the unconfigured case is unchanged. The probe context deliberately carries no logger. A resolver logs its own failure at error level with the endpoint in the message, which would place an Internal Domain resolver address in the retained journal; logUpstreamProbeFailure remains the reporter and bounds what the line may hold. TestInternalDomainsLoopCheckHidesResolverAddress covers this. --- cmd/cli/loop.go | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/cmd/cli/loop.go b/cmd/cli/loop.go index b4adb8f8..0580be93 100644 --- a/cmd/cli/loop.go +++ b/cmd/cli/loop.go @@ -14,6 +14,11 @@ import ( const ( loopTestDomain = ".test" loopTestQtype = dns.TypeTXT + + // defaultLoopCheckTimeout bounds one loop probe when the upstream sets no + // timeout. It matches the DNS client's own default, so the unconfigured + // case keeps its previous behavior. + defaultLoopCheckTimeout = 2 * time.Second ) // newLoopGuard returns new loopGuard. @@ -118,13 +123,34 @@ func (p *prog) checkDnsLoop() { logUpstreamProbeFailure(reference[uid], uc, err, mainLog.Load().Warn, "could not perform loop check") continue } - if _, err := resolver.Resolve(context.Background(), msg); err != nil { + // Bound the probe, like checkUpstreamOnce does. Without a deadline the + // DNS client falls back to its own default, so one unreachable local + // upstream stalls this serial loop far past the configured timeout. + timeout := defaultLoopCheckTimeout + if uc.Timeout > 0 { + timeout = time.Millisecond * time.Duration(uc.Timeout) + } + if err := resolveLoopTestMsg(resolver, msg, timeout); err != nil { logUpstreamProbeFailure(reference[uid], uc, err, mainLog.Load().Warn, "could not send DNS loop check query") } } mainLog.Load().Debug().Msg("end checking DNS loop") } +// resolveLoopTestMsg sends one loop test query under its own deadline. The +// cancel runs before the next upstream, so a long loop leaks no contexts. +// +// The context deliberately carries no logger. A resolver logs its own failure +// at error level with the endpoint in the message, which would put an Internal +// Domain resolver address in the retained journal; logUpstreamProbeFailure is +// what reports this failure, and it bounds what the line may hold. +func resolveLoopTestMsg(resolver ctrld.Resolver, msg *dns.Msg, timeout time.Duration) error { + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + _, err := resolver.Resolve(ctx, msg) + return err +} + // checkDnsLoopTicker performs p.checkDnsLoop every minute. func (p *prog) checkDnsLoopTicker(ctx context.Context) { timer := time.NewTicker(time.Minute) From 48fa94eb7f5d1410732d1d4c2e6da38122f7f018 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Wed, 23 Sep 2026 15:32:01 +0700 Subject: [PATCH 21/29] test(cli): scope probe journal assertions to their own log lines captureDebugMainLog swaps the process-wide mainLog, so every mainLog.Load().Warn()/.Error() site in the package writes into the buffer while it is installed, including goroutines left running by other tests. retainedProbeLines returned every retained line and the tests asserted field values over all of them, so one unrelated warn failed the run. Test_checkDnsLoopKeepsACustomKeyOutOfTheJournal hit this on Windows, where the loop probe held the buffer open for two seconds: upstream_probe_journal_test.go: field "upstream": got , want upstream.custom The sibling tests were exposed the same way; they index retained[0] after a strict length check, so a stray line fails them as got 2, want 1. Let retainedProbeLines take the messages a caller owns and return only those lines. wantNoProbeSecret keeps scanning every retained line, which is what that leak check is for. --- cmd/cli/upstream_probe_journal_test.go | 28 ++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/cmd/cli/upstream_probe_journal_test.go b/cmd/cli/upstream_probe_journal_test.go index b36208d9..93473a00 100644 --- a/cmd/cli/upstream_probe_journal_test.go +++ b/cmd/cli/upstream_probe_journal_test.go @@ -22,14 +22,30 @@ func customProbeUpstream() (string, *ctrld.UpstreamConfig) { } } -// retainedProbeLines returns the captured lines that the journal keeps. -func retainedProbeLines(t *testing.T, logs *syncBuffer) []map[string]any { +// retainedProbeLines returns the captured lines that the journal keeps. When +// messages are given, only the lines that start with one of them come back. +// captureDebugMainLog swaps the process-wide mainLog, so any goroutine of +// another test can add a warn or error line to the same buffer; a caller that +// asserts on field values names the lines its own probe wrote. +func retainedProbeLines(t *testing.T, logs *syncBuffer, messages ...string) []map[string]any { t.Helper() var retained []map[string]any for _, event := range jsonLogEvents(t, logs, "") { switch event["level"] { case "error", "warn": + default: + continue + } + if len(messages) == 0 { retained = append(retained, event) + continue + } + message, _ := event["message"].(string) + for _, want := range messages { + if strings.HasPrefix(message, want) { + retained = append(retained, event) + break + } } } return retained @@ -57,7 +73,7 @@ func Test_upstreamFailureLogKeepsACustomKeyOutOfTheJournal(t *testing.T) { (&upstreamFailureLog{}).report(upstream, uc, errors.New("probe failed"), time.Second) - retained := retainedProbeLines(t, logs) + retained := retainedProbeLines(t, logs, "Upstream check failed after ") if len(retained) != 1 { t.Fatalf("retained lines: got %d, want 1", len(retained)) } @@ -82,7 +98,7 @@ func Test_checkUpstreamOnceKeepsACustomKeyOutOfTheJournal(t *testing.T) { t.Fatal("expected the resolver creation to fail") } - retained := retainedProbeLines(t, logs) + retained := retainedProbeLines(t, logs, "Failed to create resolver") if len(retained) != 1 { t.Fatalf("retained lines: got %d, want 1", len(retained)) } @@ -107,7 +123,7 @@ func Test_checkDnsLoopKeepsACustomKeyOutOfTheJournal(t *testing.T) { p.checkDnsLoop() - retained := retainedProbeLines(t, logs) + retained := retainedProbeLines(t, logs, "could not perform loop check", "could not send DNS loop check query") if len(retained) == 0 { t.Fatal("the loop check failure must stay visible above debug") } @@ -129,7 +145,7 @@ func Test_logUpstreamPingFailuresKeepsACustomKeyOutOfTheJournal(t *testing.T) { logUpstreamPingFailures(map[string]*ctrld.UpstreamConfig{probeSecret: uc}) - retained := retainedProbeLines(t, logs) + retained := retainedProbeLines(t, logs, "failed to connect to the upstream") if len(retained) != 1 { t.Fatalf("retained lines: got %d, want 1", len(retained)) } From 7613c050766d18661e6b050ceb139682421c1395 Mon Sep 17 00:00:00 2001 From: Dev Scribe Date: Wed, 23 Sep 2026 22:56:37 +0000 Subject: [PATCH 22/29] macOS: gate DNS-target fallback on native CLAT and diagnose IPv6 blocking --- cmd/cli/dns_intercept_darwin.go | 3 + cmd/cli/dns_target_darwin.go | 101 ++++- cmd/cli/dns_target_darwin_test.go | 31 ++ cmd/cli/dns_target_dhcp_darwin_test.go | 54 +++ cmd/cli/dns_target_integration_darwin_test.go | 97 +++++ cmd/cli/dns_target_native.go | 317 +++++++++++++++ cmd/cli/dns_target_native_capture_test.go | 84 ++++ cmd/cli/dns_target_native_darwin.go | 41 ++ cmd/cli/dns_target_native_darwin_test.go | 380 ++++++++++++++++++ cmd/cli/dns_target_native_reader_test.go | 312 ++++++++++++++ cmd/cli/dns_target_native_test.go | 55 +++ cmd/cli/pf_diagnostic.go | 192 +++++++++ cmd/cli/pf_diagnostic_darwin.go | 50 +++ cmd/cli/pf_diagnostic_darwin_test.go | 62 +++ cmd/cli/pf_diagnostic_log.go | 47 +++ cmd/cli/pf_diagnostic_log_test.go | 54 +++ cmd/cli/pf_diagnostic_test.go | 208 ++++++++++ cmd/cli/pf_probe_darwin_test.go | 3 + docs/ipv6-only-dns-target-qa.md | 29 ++ docs/network-recovery-diagnostics.md | 38 ++ nameservers_darwin.go | 24 +- nameservers_darwin_deadline_test.go | 18 + nameservers_dhcp_commands.go | 41 ++ nameservers_dhcp_commands_test.go | 112 ++++++ 24 files changed, 2317 insertions(+), 36 deletions(-) create mode 100644 cmd/cli/dns_target_dhcp_darwin_test.go create mode 100644 cmd/cli/dns_target_integration_darwin_test.go create mode 100644 cmd/cli/dns_target_native.go create mode 100644 cmd/cli/dns_target_native_capture_test.go create mode 100644 cmd/cli/dns_target_native_darwin.go create mode 100644 cmd/cli/dns_target_native_darwin_test.go create mode 100644 cmd/cli/dns_target_native_reader_test.go create mode 100644 cmd/cli/dns_target_native_test.go create mode 100644 cmd/cli/pf_diagnostic.go create mode 100644 cmd/cli/pf_diagnostic_darwin.go create mode 100644 cmd/cli/pf_diagnostic_darwin_test.go create mode 100644 cmd/cli/pf_diagnostic_log.go create mode 100644 cmd/cli/pf_diagnostic_log_test.go create mode 100644 cmd/cli/pf_diagnostic_test.go create mode 100644 docs/ipv6-only-dns-target-qa.md create mode 100644 nameservers_darwin_deadline_test.go create mode 100644 nameservers_dhcp_commands.go create mode 100644 nameservers_dhcp_commands_test.go diff --git a/cmd/cli/dns_intercept_darwin.go b/cmd/cli/dns_intercept_darwin.go index 77b911ef..68a7dbbe 100644 --- a/cmd/cli/dns_intercept_darwin.go +++ b/cmd/cli/dns_intercept_darwin.go @@ -106,6 +106,8 @@ const ( // pfState holds the state of the pf DNS interception on macOS. type pfState struct { + ipv6Diagnostic pfDiagnosticState + anchorFile string anchorName string // Serialized by prog.interceptDNSTargetMu; bounded to one failed decision. @@ -2426,6 +2428,7 @@ func (p *prog) probePFIntercept() pfProbeObservation { Str("outcome", observed.result.String()).Uint64("repeated_results", repeats). Bool("repair_eligible", observed.result == pfProbeNotIntercepted). Dur("elapsed_ms", time.Since(started)).Msg("DNS intercept probe result") + p.logPFIPv6Diagnostic(observed, family) return observed } diff --git a/cmd/cli/dns_target_darwin.go b/cmd/cli/dns_target_darwin.go index 28d68447..4f2fb323 100644 --- a/cmd/cli/dns_target_darwin.go +++ b/cmd/cli/dns_target_darwin.go @@ -3,13 +3,14 @@ package cli import ( + "context" "encoding/json" + "errors" "net" "os" + "slices" "tailscale.com/net/netmon" - - "github.com/Control-D-Inc/ctrld" ) // interceptDNSTargetStateFile persists which service/value ctrld set, so a @@ -19,16 +20,21 @@ import ( const interceptDNSTargetStateFile = ".intercept_dns_target" var ( - interceptDNSTargetStatePathFn = func() string { return absHomeDir(interceptDNSTargetStateFile) } - interceptDefaultRouteInterfaceFn = netmon.DefaultRouteInterface - interceptInterfaceByNameFn = net.InterfaceByName - interceptPatchNetIfaceNameFn = patchNetIfaceName - interceptCurrentStaticDNSFn = currentStaticDNS + interceptDNSTargetStatePathFn = func() string { return absHomeDir(interceptDNSTargetStateFile) } + interceptDefaultRouteInterfaceFn = netmon.DefaultRouteInterface + interceptInterfaceByNameFn = net.InterfaceByName + interceptPatchNetIfaceNameFn = patchNetIfaceName + interceptCurrentStaticDNSFn = readTargetStaticDNS + interceptNativeCLATDefaultServiceFn = nativeCLATDefaultService + interceptNativeStaticDNSFn = readNativeTargetStaticDNS + interceptSaveStaticDNSSnapshotFn = func(iface *net.Interface, dns []string, owned string) error { + return saveTargetStaticDNSSnapshot(savedStaticDnsSettingsFilePath(iface), dns, owned) + } interceptSaveCurrentStaticDNSFn = saveCurrentStaticDNS interceptSetDNSFn = setDNS interceptSavedStaticNameserversFn = savedStaticNameservers interceptResetDNSIgnoreUnusableIfaceFn = resetDnsIgnoreUnusableInterface - interceptDHCPNameserversForInterfaceFn = ctrld.DHCPNameserversForInterface + interceptDHCPNameserversForInterfaceFn = readTargetDHCPNameservers ) type interceptDNSTargetState struct { @@ -139,25 +145,53 @@ func (p *prog) ensureInterceptDNSTarget(systemDiscovery []string) { diagnostic.failed(decision, "interface_lookup", err) return } - // Resolve the network service name (e.g. en5 -> "iPhone USB") so - // networksetup operates on the right service. - if _, err := interceptPatchNetIfaceNameFn(iface); err != nil { + routeDHCPDNS, dhcpErr := interceptDHCPNameserversForInterfaceFn(drIfaceName) + nativeFallback := false + ctx, cancel := context.WithTimeout(context.Background(), nativeTargetReadBudget) + defer cancel() + var nativeService nativeTargetService + readStatic := interceptCurrentStaticDNSFn + if dhcpErr != nil { + // Resolve the exact primary UUID before any expanded mutation. A + // device can have multiple services; the first listed is not proof. + var nativeErr error + nativeService, nativeErr = interceptNativeCLATDefaultServiceFn(ctx, drIfaceName) + nativeFallback = nativeErr == nil && nativeService.ID != "" && nativeService.Name != "" && nativeService.Device == drIfaceName && !hasIPv4DNS(routeDHCPDNS) + mainLog.Load().Debug().Err(nativeErr).Bool("native_clat_verified", nativeFallback). + Str("interface", drIfaceName).Msg("intercept DNS target: native fallback evidence") + } + if nativeFallback { + iface.Name = nativeService.Name + readStatic = func(iface *net.Interface) ([]string, error) { return interceptNativeStaticDNSFn(ctx, iface) } + } else if _, err := interceptPatchNetIfaceNameFn(iface); err != nil { diagnostic.failed(decision, "service_lookup", err) mainLog.Load().Debug().Err(err).Msgf("intercept DNS target: could not resolve network service for %s", drIfaceName) return } decision.service = iface.Name - staticDNS, err := interceptCurrentStaticDNSFn(iface) + staticDNS, err := readStatic(iface) if err != nil { - // Interfaces without a network service (utun/VPN tunnels) land here: - // networksetup cannot address them, ctrld never writes to them, and - // any target set on the underlying physical service stays in place — - // still correct while ctrld runs. diagnostic.failed(decision, "static_dns", err) mainLog.Load().Debug().Err(err).Msgf("intercept DNS target: could not read static DNS for %q", iface.Name) return } + // Retain the exact validated snapshot for backup and equality checks, + // including an owned non-loopback value that must never be snapshotted. + snapshot := staticDNS + if nativeFallback { + device, routeErr := interceptDefaultRouteInterfaceFn() + check, serviceErr := interceptNativeCLATDefaultServiceFn(ctx, drIfaceName) + if routeErr != nil || serviceErr != nil || device != drIfaceName || check != nativeService { + diagnostic.failed(decision, "native_recheck", errors.Join(routeErr, serviceErr)) + return + } + current, readErr := readStatic(iface) + if readErr != nil || ctx.Err() != nil || !slices.Equal(current, snapshot) { + diagnostic.failed(decision, "static_recheck", errors.Join(readErr, ctx.Err())) + return + } + } // Never count ctrld's own previously-set entry as network-provided DNS, // or the next recovery on the same DNS-less network would remove it and // the one after re-add it. @@ -169,11 +203,19 @@ func (p *prog) ensureInterceptDNSTarget(systemDiscovery []string) { p.removeInterceptDNSTargetLocked("network has usable static IPv4 DNS") return } + if nativeFallback && slices.ContainsFunc(staticDNS, func(s string) bool { + ip := net.ParseIP(s) + return ip != nil && ip.To4() == nil && ip.IsLoopback() + }) { + // lo0 can serve another local resolver. The static backup reader + // discards loopback, so expanded admission cannot safely replace it. + resolvedReason = "static_loopback_dns" + return + } - routeDHCPDNS, err := interceptDHCPNameserversForInterfaceFn(drIfaceName) - if err != nil { - diagnostic.failed(decision, "dhcp_dns", err) - mainLog.Load().Debug().Err(err).Msgf("intercept DNS target: could not read DHCP DNS for default-route service %q", iface.Name) + if dhcpErr != nil && !nativeFallback { + diagnostic.failed(decision, "dhcp_dns", dhcpErr) + mainLog.Load().Debug().Err(dhcpErr).Msgf("intercept DNS target: could not read DHCP DNS for default-route service %q", iface.Name) return } if hasIPv4DNS(routeDHCPDNS) { @@ -189,16 +231,30 @@ func (p *prog) ensureInterceptDNSTarget(systemDiscovery []string) { if p.interceptDNSTargetService == iface.Name && p.interceptDNSTargetSetValue == target { return // already set on this service } + if nativeFallback { + owned := "" + if p.interceptDNSTargetService == iface.Name { + owned = p.interceptDNSTargetSetValue + } + if err := interceptSaveStaticDNSSnapshotFn(iface, snapshot, owned); err != nil { + return // no expanded mutation without a restorable backup + } + } // Default route moved to a different DNS-less service (or the listener // config changed): clear the stale entry first. p.removeInterceptDNSTargetLocked("default route service changed") + if p.interceptDNSTargetService != "" { + return // cleanup failed; never overwrite ownership of the old service + } // Preserve any existing (IPv6-only) static entries for later restore. // saveCurrentStaticDNS filters loopback on write, and // savedStaticNameservers filters loopback on read, so ctrld's own // loopback target can never be recorded or restored as user DNS. - if err := interceptSaveCurrentStaticDNSFn(iface); err != nil { - mainLog.Load().Debug().Err(err).Msgf("intercept DNS target: could not save static DNS for %q", iface.Name) + if !nativeFallback { + if err := interceptSaveCurrentStaticDNSFn(iface); err != nil { + mainLog.Load().Debug().Err(err).Msgf("intercept DNS target: could not save static DNS for %q", iface.Name) + } } if err := interceptSetDNSFn(iface, []string{target}); err != nil { mainLog.Load().Warn().Err(err).Msgf("intercept DNS target: could not set %s on %q", target, iface.Name) @@ -207,6 +263,7 @@ func (p *prog) ensureInterceptDNSTarget(systemDiscovery []string) { p.setInterceptDNSTargetLocked(iface.Name, target) p.persistInterceptDNSTargetStateLocked() journal(mainLog.Load().Warn()).Str("service", iface.Name).Str("target", target). + Bool("native_clat_fallback", nativeFallback). Str("reason", "dns_less_network"). Msgf("intercept DNS target: service %q provides no usable IPv4 DNS; set %s so macOS can emit DNS queries (removed automatically when the network provides IPv4 DNS)", iface.Name, target) } diff --git a/cmd/cli/dns_target_darwin_test.go b/cmd/cli/dns_target_darwin_test.go index 0bdb1818..b22d9dd3 100644 --- a/cmd/cli/dns_target_darwin_test.go +++ b/cmd/cli/dns_target_darwin_test.go @@ -3,6 +3,7 @@ package cli import ( + "context" "errors" "net" "os" @@ -20,6 +21,9 @@ type interceptTargetHarness struct { serviceByDev map[string]string dhcp []string dhcpErr error + nativeCLAT bool + nativeErr error + saveErr error readErr error setErr error resetErr error @@ -47,6 +51,9 @@ func newInterceptTargetHarness(t *testing.T) *interceptTargetHarness { origSaved := interceptSavedStaticNameserversFn origReset := interceptResetDNSIgnoreUnusableIfaceFn origDHCP := interceptDHCPNameserversForInterfaceFn + origNative := interceptNativeCLATDefaultServiceFn + origNativeStatic := interceptNativeStaticDNSFn + origSnapshot := interceptSaveStaticDNSSnapshotFn origIntercept := dnsIntercept t.Cleanup(func() { interceptDNSTargetStatePathFn = origPath @@ -59,9 +66,30 @@ func newInterceptTargetHarness(t *testing.T) *interceptTargetHarness { interceptSavedStaticNameserversFn = origSaved interceptResetDNSIgnoreUnusableIfaceFn = origReset interceptDHCPNameserversForInterfaceFn = origDHCP + interceptNativeCLATDefaultServiceFn = origNative + interceptNativeStaticDNSFn = origNativeStatic + interceptSaveStaticDNSSnapshotFn = origSnapshot dnsIntercept = origIntercept }) + interceptNativeCLATDefaultServiceFn = func(_ context.Context, device string) (nativeTargetService, error) { + if !h.nativeCLAT || h.nativeErr != nil { + return nativeTargetService{}, h.nativeErr + } + return nativeTargetService{ID: nativeTargetTestServiceID, Name: h.serviceByDev[device], Device: device}, nil + } + interceptNativeStaticDNSFn = func(_ context.Context, iface *net.Interface) ([]string, error) { + return interceptCurrentStaticDNSFn(iface) + } + interceptSaveStaticDNSSnapshotFn = func(iface *net.Interface, snapshot []string, owned string) error { + if h.saveErr != nil { + return h.saveErr + } + if !isInterceptDNSTargetOnly(snapshot, owned) { + h.saved[iface.Name] = slices.Clone(filterOwnTarget(snapshot, owned)) + } + return nil + } dnsIntercept = true interceptDNSTargetStatePathFn = func() string { return h.statePath } interceptDefaultRouteInterfaceFn = func() (string, error) { return "en1", nil } @@ -81,6 +109,9 @@ func newInterceptTargetHarness(t *testing.T) *interceptTargetHarness { return slices.Clone(h.dns[iface.Name]), nil } interceptSaveCurrentStaticDNSFn = func(iface *net.Interface) error { + if h.saveErr != nil { + return h.saveErr + } h.saved[iface.Name] = slices.Clone(h.dns[iface.Name]) return nil } diff --git a/cmd/cli/dns_target_dhcp_darwin_test.go b/cmd/cli/dns_target_dhcp_darwin_test.go new file mode 100644 index 00000000..747409a8 --- /dev/null +++ b/cmd/cli/dns_target_dhcp_darwin_test.go @@ -0,0 +1,54 @@ +//go:build darwin + +package cli + +import ( + "context" + "errors" + "reflect" + "slices" + "testing" + "time" +) + +func TestInterceptDNSTargetDHCPDeadline(t *testing.T) { + if reflect.ValueOf(interceptDHCPNameserversForInterfaceFn).Pointer() != reflect.ValueOf(readTargetDHCPNameservers).Pointer() { + t.Fatal("target discovery bypasses bounded DHCP reader") + } + h := newInterceptTargetHarness(t) + h.dns["Wi-Fi"] = []string{"192.0.2.53"} + original := targetDHCPReadFn + t.Cleanup(func() { targetDHCPReadFn = original }) + entered := make(chan struct{}) + targetDHCPReadFn = func(ctx context.Context, device string) ([]string, error) { + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) > nativeTargetReadBudget || device != "en1" { + t.Error("missing bounded context or wrong interface") + close(entered) + return nil, errors.New("invalid context") + } + close(entered) + <-ctx.Done() + return nil, ctx.Err() + } + interceptDHCPNameserversForInterfaceFn = readTargetDHCPNameservers + p := newInterceptTargetProg() + finished := make(chan struct{}) + go func() { p.ensureInterceptDNSTarget([]string{}); close(finished) }() + <-entered + cleaned := make(chan struct{}) + go func() { p.removeInterceptDNSTarget("test shutdown"); close(cleaned) }() + select { + case <-finished: + case <-time.After(5 * time.Second): + t.Fatal("DHCP lookup did not release reconciliation") + } + select { + case <-cleaned: + case <-time.After(time.Second): + t.Fatal("cleanup remained blocked after DHCP deadline") + } + if !slices.Equal(h.dns["Wi-Fi"], []string{"192.0.2.53"}) || len(h.setCalls) != 0 || len(h.resetCalls) != 0 { + t.Fatal("deadline changed existing static DNS") + } +} diff --git a/cmd/cli/dns_target_integration_darwin_test.go b/cmd/cli/dns_target_integration_darwin_test.go new file mode 100644 index 00000000..31838500 --- /dev/null +++ b/cmd/cli/dns_target_integration_darwin_test.go @@ -0,0 +1,97 @@ +//go:build darwin + +package cli + +import ( + "context" + "errors" + "net" + "testing" +) + +func TestNativeCLATDecisionJournalIntegration(t *testing.T) { + for _, local := range []bool{false, true} { + name := "target installed" + if local { + name = "local resolver preserved" + } + t.Run(name, func(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + h.dhcpErr = errors.New("no DHCP packet") + p.ensureInterceptDNSTarget([]string{}) + p.ensureInterceptDNSTarget([]string{}) + wantField(t, oneRecoveryEvent(t, logs, dnsTargetFailedMessage), "stage", "dhcp_dns") + h.nativeCLAT = true + reason := "dns_less_network" + action := "set" + if local { + h.dns["Wi-Fi"] = []string{"::1"} + reason = "static_loopback_dns" + action = "unchanged" + } + p.ensureInterceptDNSTarget([]string{}) + p.ensureInterceptDNSTarget([]string{}) + event := oneRecoveryEvent(t, logs, dnsTargetResolvedMessage) + wantField(t, event, "reason", reason) + wantField(t, event, "action", action) + wantField(t, event, "repeat_count", float64(1)) + if p.dnsInterceptState.(*pfState).targetDiagnostic.pending { + t.Fatal("native decision left stale failure") + } + if local && (len(h.setCalls) != 0 || len(h.resetCalls) != 0 || h.dns["Wi-Fi"][0] != "::1") { + t.Fatal("local resolver changed") + } + if !local && len(h.setCalls) != 1 { + t.Fatal("native target not installed exactly once") + } + }) + } +} + +func TestNativeCLATRecheckFailureJournal(t *testing.T) { + for _, static := range []bool{false, true} { + name := "native recheck" + if static { + name = "static recheck" + } + t.Run(name, func(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + h.dhcpErr = errors.New("no DHCP packet") + h.nativeCLAT = true + p := newInterceptTargetProg() + stage := "native_recheck" + calls := 0 + if static { + stage = "static_recheck" + orig := interceptNativeStaticDNSFn + interceptNativeStaticDNSFn = func(ctx context.Context, i *net.Interface) ([]string, error) { + calls++ + if calls%2 == 0 { + return nil, context.DeadlineExceeded + } + return orig(ctx, i) + } + } else { + orig := interceptNativeCLATDefaultServiceFn + interceptNativeCLATDefaultServiceFn = func(ctx context.Context, device string) (nativeTargetService, error) { + calls++ + if calls%2 == 0 { + return nativeTargetService{}, context.DeadlineExceeded + } + return orig(ctx, device) + } + } + p.ensureInterceptDNSTarget([]string{}) + p.ensureInterceptDNSTarget([]string{}) + event := oneRecoveryEvent(t, logs, dnsTargetFailedMessage) + wantField(t, event, "stage", stage) + wantField(t, event, "error_class", "timeout") + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 { + t.Fatal("failed recheck changed DNS") + } + }) + } +} diff --git a/cmd/cli/dns_target_native.go b/cmd/cli/dns_target_native.go new file mode 100644 index 00000000..f07e0337 --- /dev/null +++ b/cmd/cli/dns_target_native.go @@ -0,0 +1,317 @@ +package cli + +import ( + "bufio" + "bytes" + "context" + "errors" + "io" + "net" + "os" + "os/exec" + "regexp" + "strings" + "time" +) + +const ( + // One budget covers the entire native evidence transaction, including + // both identity/static rechecks; it is not renewed for each subprocess. + nativeTargetReadBudget = 2 * time.Second + nativeTargetMaxOutput = 64 * 1024 +) + +var errNativeTargetOutputLimit = errors.New("native DNS evidence output exceeds limit") + +type nativeTargetOutput struct{ buf bytes.Buffer } + +func (w *nativeTargetOutput) Write(p []byte) (int, error) { + if len(p) > nativeTargetMaxOutput-w.buf.Len() { + return 0, errNativeTargetOutputLimit + } + return w.buf.Write(p) +} + +func runNativeTargetCommand(ctx context.Context, input, path string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, path, args...) + cmd.Stdin = strings.NewReader(input) + var out nativeTargetOutput + cmd.Stdout = &out + cmd.Stderr = io.Discard + // Also bound waiting for inherited pipes if a child outlives the command. + cmd.WaitDelay = 100 * time.Millisecond + if err := cmd.Run(); err != nil { + return nil, err + } + if err := ctx.Err(); err != nil { + return nil, err + } + return out.buf.Bytes(), nil +} + +type nativeTargetService struct { + ID, Name, Device string +} + +type nativeTargetReader struct { + run func(context.Context, string, string, ...string) ([]byte, error) + interfaceByName func(string) (*net.Interface, error) + interfaceAddrs func(*net.Interface) ([]net.Addr, error) +} + +func (r nativeTargetReader) store(ctx context.Context, key string) (map[string]string, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + out, err := r.run(ctx, "show "+key+"\nquit\n", "/usr/sbin/scutil") + if err != nil { + return nil, err + } + return parseNativeIPv4Store(string(out)) +} + +// The service root contains only the configured name. Reject multiline or +// unfamiliar layouts rather than searching arbitrary DHCP/SSID/packet text. +// The UUID in the key comes exclusively from the validated Global dictionary. +func parseNativeServiceName(out string) (string, error) { + lines := strings.Split(strings.TrimSpace(out), "\n") + if len(lines) != 3 || lines[0] != " {" || lines[2] != "}" { + return "", errors.New("invalid native service dictionary") + } + name, ok := strings.CutPrefix(strings.TrimLeft(lines[1], " \t"), "UserDefinedName : ") + if !ok || name == "" || strings.ContainsAny(name, "\r\n\x00/{}") { + return "", errors.New("invalid native service name") + } + return name, nil +} + +// defaultService returns the actual primary service, not the first service +// networksetup happens to list for the device. Unknown or changing state is +// never absence evidence. The caller must carry and recheck the whole tuple. +func (r nativeTargetReader) defaultService(ctx context.Context, device string) (nativeTargetService, error) { + unknown := nativeTargetService{} + before, err := r.store(ctx, "State:/Network/Global/IPv4") + if err != nil { + return unknown, err + } + id := before["PrimaryService"] + if !nativeServiceID.MatchString(id) || before["PrimaryInterface"] != device { + return unknown, nil + } + state, err := r.store(ctx, "State:/Network/Service/"+id+"/IPv4") + if err != nil { + return unknown, err + } + if state["CLAT46"] != "TRUE" || state["InterfaceName"] != device || state["Router"] != "192.0.0.1" { + return unknown, nil + } + iface, err := r.interfaceByName(device) + if err != nil { + return unknown, err + } + if iface == nil || iface.Name != device || iface.Flags&net.FlagUp == 0 { + return unknown, nil + } + addrs, err := r.interfaceAddrs(iface) + if err != nil || !nativeCLATAddresses(addrs) { + return unknown, err + } + published := strings.Fields(state["AddressList"]) + if len(published) != 1 { + return unknown, nil + } + matched := false + for _, addr := range addrs { + ip, _, parseErr := net.ParseCIDR(addr.String()) + if parseErr == nil && ip.String() == published[0] { + matched = true + } + } + if !matched { + return unknown, nil + } + out, err := r.run(ctx, "show Setup:/Network/Service/"+id+"\nquit\n", "/usr/sbin/scutil") + if err != nil { + return unknown, err + } + name, err := parseNativeServiceName(string(out)) + if err != nil { + return unknown, err + } + after, err := r.store(ctx, "State:/Network/Global/IPv4") + if err != nil { + return unknown, err + } + if after["PrimaryService"] != id || after["PrimaryInterface"] != device { + return unknown, errors.New("default service changed during CLAT discovery") + } + return nativeTargetService{ID: id, Name: name, Device: device}, nil +} + +// saveTargetStaticDNSSnapshot saves only the already validated/rechecked +// snapshot. Never re-read via the legacy permissive static DNS reader, and +// never replace the original backup with ctrld's own (possibly non-loopback) +// target when a listener changes after restart. +func saveTargetStaticDNSSnapshot(path string, snapshot []string, owned string) error { + if isInterceptDNSTargetOnly(snapshot, owned) { + // Missing means the original service used automatic DNS. A present + // but unreadable/corrupt backup must not authorize a listener change. + data, err := os.ReadFile(path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if len(data) == 0 { + return nil + } + for _, s := range strings.Split(string(data), ",") { + if net.ParseIP(s) == nil || s == owned { + return errors.New("invalid retained static DNS backup") + } + } + return nil + } + var dns []string + for _, s := range snapshot { + ip := net.ParseIP(s) + if ip == nil { + return errors.New("invalid static DNS snapshot") + } + if s == owned { + continue + } + // The restore reader filters loopbacks. Refuse an unrestorable + // user snapshot rather than silently discarding a local resolver. + if ip.IsLoopback() { + return errors.New("non-owned loopback in static DNS snapshot") + } + dns = append(dns, ip.String()) + } + if len(dns) == 0 { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil + } + return os.WriteFile(path, []byte(strings.Join(dns, ",")), 0600) +} + +var nativeServiceID = regexp.MustCompile(`^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$`) + +// parseNativeIPv4Store reads only the two native IPv4 dictionaries, never +// ipconfig getsummary (which embeds unescaped, network-controlled DHCP text). +// Unknown layouts fail closed. These fields are generated by configd from +// interface names, service UUIDs, booleans and binary IP addresses. +func parseNativeIPv4Store(out string) (map[string]string, error) { + lines := strings.Split(strings.TrimSpace(out), "\n") + if len(lines) < 2 || lines[0] != " {" || lines[len(lines)-1] != "}" { + return nil, errors.New("invalid native IPv4 dictionary") + } + values := make(map[string]string) + depth := 1 + arrayKey := "" + for _, line := range lines[1 : len(lines)-1] { + s := strings.TrimSpace(line) + if s == "}" { + depth-- + if depth < 1 { + return nil, errors.New("unbalanced native IPv4 dictionary") + } + continue + } + key, value, ok := strings.Cut(s, " : ") + if !ok { + return nil, errors.New("invalid native IPv4 field") + } + if depth == 1 { + if _, exists := values[key]; exists { + return nil, errors.New("duplicate native IPv4 field") + } + switch key { + case "PrimaryService": + if !nativeServiceID.MatchString(value) { + return nil, errors.New("invalid service ID") + } + case "PrimaryInterface", "InterfaceName": + if value == "" || strings.ContainsAny(value, " \t\r\n{}") { + return nil, errors.New("invalid interface") + } + case "CLAT46": + if value != "TRUE" && value != "FALSE" { + return nil, errors.New("invalid CLAT flag") + } + case "Router": + if net.ParseIP(value).To4() == nil { + return nil, errors.New("invalid router") + } + case "Addresses", "SubnetMasks", "DestinationAddresses", "AdditionalRoutes", "ExcludedRoutes": + if value != " {" { + return nil, errors.New("invalid IPv4 array") + } + default: + return nil, errors.New("unknown native IPv4 field") + } + values[key] = value + arrayKey = key + } else if depth == 2 && arrayKey == "Addresses" { + ip := net.ParseIP(value) + if ip.To4() == nil { + return nil, errors.New("invalid native IPv4 address") + } + values["AddressList"] += ip.String() + " " + } + if strings.HasSuffix(value, "{") { + depth++ + } + } + if depth != 1 { + return nil, errors.New("unbalanced native IPv4 dictionary") + } + return values, nil +} + +// A successful networksetup exit alone is not absence evidence: reject all +// text except IP literals and its exact no-static-DNS response. +func parseTargetStaticDNS(out, service string) ([]string, error) { + out = strings.TrimSpace(out) + if out == "There aren't any DNS Servers set on "+service+"." { + return nil, nil + } + if out == "" { + return nil, errors.New("empty static DNS response") + } + var dns []string + scanner := bufio.NewScanner(strings.NewReader(out)) + for scanner.Scan() { + ip := net.ParseIP(strings.TrimSpace(scanner.Text())) + if ip == nil { + return nil, errors.New("invalid static DNS response") + } + dns = append(dns, ip.String()) + } + return dns, scanner.Err() +} + +func nativeCLATAddresses(addrs []net.Addr) bool { + clat, ipv6 := false, false + for _, addr := range addrs { + ip, _, err := net.ParseCIDR(addr.String()) + if err != nil { + return false + } + if v4 := ip.To4(); v4 != nil { + // Apple allocates one of these four CLAT addresses, not only 192.0.0.2. + if v4[0] != 192 || v4[1] != 0 || v4[2] != 0 || v4[3] < 2 || v4[3] > 5 { + return false + } + clat = true + } else if ip.IsGlobalUnicast() { + // ULA is legitimate IPv6 connectivity. Addresses alone never + // authorize a target: configd must independently publish CLAT. + ipv6 = true + } + } + return clat && ipv6 +} diff --git a/cmd/cli/dns_target_native_capture_test.go b/cmd/cli/dns_target_native_capture_test.go new file mode 100644 index 00000000..084e9f72 --- /dev/null +++ b/cmd/cli/dns_target_native_capture_test.go @@ -0,0 +1,84 @@ +package cli + +import ( + "context" + "net" + "strings" + "testing" +) + +// Captured macOS 15.7.7 CLAT dictionaries; UUID and interface IPv6 addresses +// are sanitized. Keep the native field shapes, including nested routes. +func TestCapturedCLATNativeReader(t *testing.T) { + const id = "11111111-2222-3333-4444-555555555555" + const global = ` { + PrimaryInterface : en0 + PrimaryService : 11111111-2222-3333-4444-555555555555 + Router : 192.0.0.1 +}` + const state = ` { + AdditionalRoutes : { + 0 : { + DestinationAddress : 192.0.0.2 + SubnetMask : 255.255.255.255 + } + } + Addresses : { + 0 : 192.0.0.2 + } + CLAT46 : TRUE + InterfaceName : en0 + Router : 192.0.0.1 +}` + const setup = ` { + UserDefinedName : Wi-Fi +}` + iface := &net.Interface{Name: "en0", Flags: net.FlagUp | net.FlagRunning} + var addrs []net.Addr + for _, s := range []string{"192.0.0.2/32", "fe80::1/64", "2001:db8::1/64", "2001:db8::2/64", "2001:db8::3/64"} { + ip, n, err := net.ParseCIDR(s) + if err != nil { + t.Fatal(err) + } + n.IP = ip + addrs = append(addrs, n) + } + calls := 0 + reader := nativeTargetReader{ + run: func(_ context.Context, input, path string, args ...string) ([]byte, error) { + calls++ + if path != "/usr/sbin/scutil" || len(args) != 0 { + t.Fatal("unexpected command") + } + switch strings.TrimSpace(input) { + case "show State:/Network/Global/IPv4\nquit": + return []byte(global), nil + case "show State:/Network/Service/" + id + "/IPv4\nquit": + return []byte(state), nil + case "show Setup:/Network/Service/" + id + "\nquit": + return []byte(setup), nil + default: + t.Fatal("unexpected key", input) + return nil, nil + } + }, + interfaceByName: func(name string) (*net.Interface, error) { + if name != "en0" { + t.Fatal(name) + } + return iface, nil + }, + interfaceAddrs: func(got *net.Interface) ([]net.Addr, error) { + if got != iface { + t.Fatal("wrong interface") + } + return addrs, nil + }, + } + got, err := reader.defaultService(context.Background(), "en0") + want := nativeTargetService{ID: id, Name: "Wi-Fi", Device: "en0"} + if err != nil || got != want || calls != 4 { + t.Fatalf("got=%+v err=%v reads=%d", got, err, calls) + } + t.Log("Captured native dictionaries accepted; primary Wi-Fi service on en0 selected") +} diff --git a/cmd/cli/dns_target_native_darwin.go b/cmd/cli/dns_target_native_darwin.go new file mode 100644 index 00000000..a8700a4d --- /dev/null +++ b/cmd/cli/dns_target_native_darwin.go @@ -0,0 +1,41 @@ +//go:build darwin + +package cli + +import ( + "context" + "net" + + ctrld "github.com/Control-D-Inc/ctrld" +) + +var targetDHCPReadFn = ctrld.DHCPNameserversForInterfaceContext + +// Bound the DHCP prerequisite separately from the native evidence transaction. +func readTargetDHCPNameservers(device string) ([]string, error) { + ctx, cancel := context.WithTimeout(context.Background(), nativeTargetReadBudget) + defer cancel() + return targetDHCPReadFn(ctx, device) +} + +func readTargetStaticDNS(iface *net.Interface) ([]string, error) { + ctx, cancel := context.WithTimeout(context.Background(), nativeTargetReadBudget) + defer cancel() + return readNativeTargetStaticDNS(ctx, iface) +} + +func readNativeTargetStaticDNS(ctx context.Context, iface *net.Interface) ([]string, error) { + out, err := runNativeTargetCommand(ctx, "", "/usr/sbin/networksetup", "-getdnsservers", iface.Name) + if err != nil { + return nil, err + } + return parseTargetStaticDNS(string(out), iface.Name) +} + +func nativeCLATDefaultService(ctx context.Context, device string) (nativeTargetService, error) { + return (nativeTargetReader{ + run: runNativeTargetCommand, + interfaceByName: net.InterfaceByName, + interfaceAddrs: (*net.Interface).Addrs, + }).defaultService(ctx, device) +} diff --git a/cmd/cli/dns_target_native_darwin_test.go b/cmd/cli/dns_target_native_darwin_test.go new file mode 100644 index 00000000..e6b866a2 --- /dev/null +++ b/cmd/cli/dns_target_native_darwin_test.go @@ -0,0 +1,380 @@ +//go:build darwin + +package cli + +import ( + "context" + "errors" + "net" + "os" + "path/filepath" + "slices" + "strings" + "testing" +) + +func TestNativeCLATTargetPreservesNonOwnedLoopback(t *testing.T) { + for _, static := range [][]string{{"::1"}, {"::1", "2001:db8::53"}, {"127.0.0.1"}, {"127.0.0.1", "::1"}} { + t.Run(strings.Join(static, ","), func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.nativeCLAT = true + h.dhcpErr = errors.New("no DHCPv4") + h.dns["Wi-Fi"] = slices.Clone(static) + backup := filepath.Join(t.TempDir(), ".dns_Wi-Fi") + interceptSaveStaticDNSSnapshotFn = func(_ *net.Interface, snapshot []string, owned string) error { + return saveTargetStaticDNSSnapshot(backup, snapshot, owned) + } + p := newInterceptTargetProg() + p.cfg.Listener["0"].Port = 53 + p.ensureInterceptDNSTarget([]string{}) + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 || p.interceptDNSTargetService != "" { + t.Fatalf("local resolver overwritten: set=%v reset=%v owner=%q", h.setCalls, h.resetCalls, p.interceptDNSTargetService) + } + p.removeInterceptDNSTarget("shutdown") + if !slices.Equal(h.dns["Wi-Fi"], static) { + t.Fatalf("lost user DNS: got %v, want %v", h.dns["Wi-Fi"], static) + } + if _, err := os.Stat(backup); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("backup unexpectedly changed: %v", err) + } + }) + } +} + +func TestNativeCLATTargetRejectsLoopbackBeforeSnapshot(t *testing.T) { + h := newInterceptTargetHarness(t) + h.nativeCLAT = true + h.dhcpErr = errors.New("no DHCPv4") + h.dns["Wi-Fi"] = []string{"::1", "2001:db8::53"} + h.dns["Old service"] = []string{"127.0.0.53"} + interceptSaveStaticDNSSnapshotFn = func(*net.Interface, []string, string) error { + t.Fatal("non-owned IPv6 loopback admitted to snapshot") + return nil + } + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Old service", "127.0.0.53") + p.ensureInterceptDNSTarget([]string{}) + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 || p.interceptDNSTargetService != "Old service" { + t.Fatal("rejected admission changed old ownership") + } +} + +func TestNativeCLATTargetLoopbackKeepsIPv4CleanupFirst(t *testing.T) { + h := newInterceptTargetHarness(t) + h.nativeCLAT = true + h.dhcpErr = errors.New("no DHCPv4") + h.dns["Wi-Fi"] = []string{"::1", "127.0.0.1"} + h.dns["Old service"] = []string{"127.0.0.53"} + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Old service", "127.0.0.53") + p.ensureInterceptDNSTarget([]string{}) + if !slices.Equal(h.resetCalls, []string{"Old service"}) || len(h.setCalls) != 0 || p.interceptDNSTargetService != "" { + t.Fatalf("IPv4 cleanup skipped: reset=%v set=%v owner=%q", h.resetCalls, h.setCalls, p.interceptDNSTargetService) + } + if !slices.Equal(h.dns["Wi-Fi"], []string{"::1", "127.0.0.1"}) { + t.Fatal("external local resolvers changed") + } +} + +func TestNativeCLATTargetSnapshotRejectsNonOwnedLoopback(t *testing.T) { + for _, snapshot := range [][]string{{"::1"}, {"2001:db8::53", "::1"}, {"127.0.0.1"}, {"127.0.0.53", "::1"}} { + t.Run(strings.Join(snapshot, ","), func(t *testing.T) { + path := filepath.Join(t.TempDir(), "backup") + const original = "2001:db8::54" + if err := os.WriteFile(path, []byte(original), 0600); err != nil { + t.Fatal(err) + } + if err := saveTargetStaticDNSSnapshot(path, snapshot, "127.0.0.53"); err == nil { + t.Fatal("accepted unrestorable loopback snapshot") + } + if got, err := os.ReadFile(path); err != nil || string(got) != original { + t.Fatalf("failed snapshot changed backup: %q %v", got, err) + } + }) + } + for _, owned := range []string{"127.0.0.53", "::1", "10.0.0.53"} { + t.Run("owned "+owned, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "backup") + const original = "2001:db8::54" + if err := os.WriteFile(path, []byte(original), 0600); err != nil { + t.Fatal(err) + } + if err := saveTargetStaticDNSSnapshot(path, []string{owned}, owned); err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(path); err != nil || string(got) != original { + t.Fatalf("owned target replaced backup: %q %v", got, err) + } + }) + } +} + +func TestNativeCLATTargetFallback(t *testing.T) { + for _, tc := range []struct { + name string + clat bool + nativeErr, readErr, saveErr error + static, dhcp []string + want bool + }{ + {name: "CLAT", clat: true, want: true}, + {name: "unknown"}, + {name: "native read error", clat: true, nativeErr: errors.New("read failed")}, + {name: "static read error", clat: true, readErr: errors.New("read failed")}, + {name: "backup error", clat: true, saveErr: errors.New("disk full")}, + {name: "static IPv4", clat: true, static: []string{"9.9.9.9"}}, + {name: "partial DHCP IPv4", clat: true, dhcp: []string{"1.1.1.1"}}, + } { + t.Run(tc.name, func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.dhcpErr = errors.New("getoption and getpacket exit 1") + h.nativeCLAT, h.nativeErr, h.readErr, h.saveErr = tc.clat, tc.nativeErr, tc.readErr, tc.saveErr + h.dns["Wi-Fi"], h.dhcp = tc.static, tc.dhcp + p := newInterceptTargetProg() + p.ensureInterceptDNSTarget([]string{}) + if (len(h.setCalls) > 0) != tc.want || len(h.resetCalls) != 0 { + t.Fatalf("set=%v reset=%v", h.setCalls, h.resetCalls) + } + }) + } +} + +func TestNativeCLATTargetUnknownPreservesStaticIPv4Cleanup(t *testing.T) { + h := newInterceptTargetHarness(t) + h.dhcpErr = errors.New("DHCP read failed") + h.nativeErr = errors.New("native state unavailable") + h.dns["Wi-Fi"] = []string{"192.0.2.53"} + h.dns["Old service"] = []string{"127.0.0.53"} + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Old service", "127.0.0.53") + p.ensureInterceptDNSTarget([]string{}) + if p.interceptDNSTargetService != "" || !slices.Equal(h.resetCalls, []string{"Old service"}) || len(h.setCalls) != 0 || !slices.Equal(h.dns["Wi-Fi"], []string{"192.0.2.53"}) { + t.Fatalf("existing static IPv4 cleanup changed: owner=%q reset=%v set=%v", p.interceptDNSTargetService, h.resetCalls, h.setCalls) + } +} + +func TestNativeCLATTargetTransitionAndReadFailure(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + h.nativeCLAT = true + h.dhcpErr = errors.New("no DHCPv4 packet") + h.dns["Wi-Fi"] = []string{"2001:db8::53"} + p.ensureInterceptDNSTarget([]string{}) + if p.interceptDNSTargetService != "Wi-Fi" { + t.Fatal("no target") + } + h.nativeErr = errors.New("store unavailable") + p.ensureInterceptDNSTarget([]string{}) + if p.interceptDNSTargetService != "Wi-Fi" || len(h.setCalls) != 1 || len(h.resetCalls) != 0 { + t.Fatal("unknown changed ownership") + } + h.nativeErr = nil + h.dhcpErr = nil + h.dhcp = []string{"192.168.1.1"} + p.ensureInterceptDNSTarget([]string{}) + if p.interceptDNSTargetService != "" || !slices.Equal(h.dns["Wi-Fi"], []string{"2001:db8::53"}) { + t.Fatal("dual stack did not restore IPv6 static DNS") + } +} + +// Use the real evidence reader: en1 has two configured services, and the +// second is primary. Legacy first-match mapping must not even be consulted. +func TestNativeCLATTargetExactPrimaryService(t *testing.T) { + for _, static := range [][]string{{"9.9.9.9"}, {"2001:db8::53"}} { + t.Run(strings.Join(static, ","), func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.dhcpErr = errors.New("no packet") + h.serviceByDev["en1"] = "First Wi-Fi" + h.dns["First Wi-Fi"] = nil + h.dns["Primary Wi-Fi"] = slices.Clone(static) + var calls []string + outputs := append(nativeTargetTestOutputs(), nativeTargetTestOutputs()...) + r := nativeTargetTestReader(t, outputs, &calls) + interceptNativeCLATDefaultServiceFn = r.defaultService + interceptPatchNetIfaceNameFn = func(*net.Interface) (bool, error) { + t.Fatal("used first-match mapping for native fallback") + return false, nil + } + var reads []string + read := interceptNativeStaticDNSFn + interceptNativeStaticDNSFn = func(ctx context.Context, iface *net.Interface) ([]string, error) { + reads = append(reads, iface.Name) + return read(ctx, iface) + } + p := newInterceptTargetProg() + p.ensureInterceptDNSTarget([]string{}) + if !slices.Equal(reads, []string{"Primary Wi-Fi", "Primary Wi-Fi"}) || len(calls) != 8 { + t.Fatalf("reads=%v calls=%v", reads, calls) + } + if len(h.dns["First Wi-Fi"]) != 0 { + t.Fatal("inactive service mutated") + } + if hasIPv4DNS(static) { + if !slices.Equal(h.dns["Primary Wi-Fi"], static) || len(h.setCalls) != 0 || len(h.resetCalls) != 0 { + t.Fatal("primary static IPv4 overwritten") + } + } else { + if !slices.Equal(h.setCalls, []string{"Primary Wi-Fi"}) || p.interceptDNSTargetService != "Primary Wi-Fi" { + t.Fatalf("set=%v ownership=%q", h.setCalls, p.interceptDNSTargetService) + } + p.removeInterceptDNSTarget("test shutdown") + if !slices.Equal(h.dns["Primary Wi-Fi"], static) { + t.Fatal("original static IPv6 not restored") + } + } + }) + } +} + +func TestNativeCLATTargetIdentityRecheck(t *testing.T) { + for _, field := range []string{"UUID", "name", "device", "read error", "static DNS"} { + t.Run(field, func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.nativeCLAT = true + h.dhcpErr = errors.New("no packet") + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Old service", "127.0.0.53") + h.dns["Old service"] = []string{"127.0.0.53"} + read := interceptNativeCLATDefaultServiceFn + calls := 0 + var shared context.Context + interceptNativeCLATDefaultServiceFn = func(ctx context.Context, device string) (nativeTargetService, error) { + calls++ + if calls == 1 { + shared = ctx + } else if shared != ctx { + t.Fatal("budget renewed on recheck") + } + svc, err := read(ctx, device) + if calls == 2 { + switch field { + case "UUID": + svc.ID = nativeTargetOtherServiceID + case "name": + svc.Name = "Other Wi-Fi" + case "device": + svc.Device = "en9" + case "read error": + err = errors.New("unknown") + case "static DNS": + h.dns["Wi-Fi"] = []string{"2001:db8::53"} + } + } + return svc, err + } + p.ensureInterceptDNSTarget([]string{}) + if calls != 2 || len(h.setCalls) != 0 || len(h.resetCalls) != 0 || p.interceptDNSTargetService != "Old service" { + t.Fatalf("uncertain recheck mutated state: calls=%d set=%v reset=%v owner=%q", calls, h.setCalls, h.resetCalls, p.interceptDNSTargetService) + } + }) + } +} + +func TestNativeCLATTargetRestartListenerChangePreservesBackup(t *testing.T) { + h := newInterceptTargetHarness(t) + h.nativeCLAT = true + h.dhcpErr = errors.New("no packet") + backup := filepath.Join(t.TempDir(), ".dns_Wi-Fi") + original := "2001:db8::53" + if err := os.WriteFile(backup, []byte(original), 0600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(h.statePath, []byte(`{"service":"Wi-Fi","value":"10.0.0.53"}`), 0600); err != nil { + t.Fatal(err) + } + h.dns["Wi-Fi"] = []string{"10.0.0.53"} + var snapshots [][]string + interceptSaveStaticDNSSnapshotFn = func(_ *net.Interface, snapshot []string, owned string) error { + snapshots = append(snapshots, slices.Clone(snapshot)) + return saveTargetStaticDNSSnapshot(backup, snapshot, owned) + } + interceptSaveCurrentStaticDNSFn = func(*net.Interface) error { t.Fatal("legacy unvalidated snapshot read"); return nil } + interceptSavedStaticNameserversFn = func(*net.Interface) []string { + data, err := os.ReadFile(backup) + if err != nil { + t.Fatal(err) + } + return strings.Split(string(data), ",") + } + p := newInterceptTargetProg() + p.cfg.Listener["0"].IP, p.cfg.Listener["0"].Port = "10.0.0.54", 53 + p.ensureInterceptDNSTarget([]string{}) + if !slices.Equal(h.dns["Wi-Fi"], []string{"10.0.0.54"}) || p.interceptDNSTargetSetValue != "10.0.0.54" || len(snapshots) != 1 { + t.Fatalf("listener migration failed: dns=%v target=%q snapshots=%v", h.dns["Wi-Fi"], p.interceptDNSTargetSetValue, snapshots) + } + if got, err := os.ReadFile(backup); err != nil || string(got) != original { + t.Fatalf("original backup lost: %q %v", got, err) + } + p.ensureInterceptDNSTarget([]string{}) + if len(h.setCalls) != 2 { + t.Fatalf("target churn after reconciliation: %v", h.setCalls) + } + // A second restart must recover new ownership and still restore the original. + restarted := newInterceptTargetProg() + restarted.removeInterceptDNSTarget("shutdown") + if !slices.Equal(h.dns["Wi-Fi"], []string{original}) || restarted.interceptDNSTargetService != "" { + t.Fatalf("shutdown restored owned target: %v", h.dns["Wi-Fi"]) + } + if _, err := os.Stat(h.statePath); !errors.Is(err, os.ErrNotExist) { + t.Fatal("ownership not removed", err) + } +} + +func TestNativeCLATTargetSnapshotFailureRetainsOwnership(t *testing.T) { + for _, mode := range []string{"write", "remove", "cleanup"} { + t.Run(mode, func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.nativeCLAT = true + h.dhcpErr = errors.New("no packet") + p := newInterceptTargetProg() + persistInterceptTargetForTest(t, p, "Old service", "10.0.0.53") + h.dns["Old service"] = []string{"10.0.0.53"} + path := t.TempDir() + if err := os.WriteFile(filepath.Join(path, "block"), nil, 0600); err != nil { + t.Fatal(err) + } + if mode == "write" { + h.dns["Wi-Fi"] = []string{"2001:db8::53"} + } + if mode == "cleanup" { + path = filepath.Join(t.TempDir(), "backup") + h.resetErr = errors.New("cleanup failed") + } + interceptSaveStaticDNSSnapshotFn = func(_ *net.Interface, snapshot []string, owned string) error { + return saveTargetStaticDNSSnapshot(path, snapshot, owned) + } + p.ensureInterceptDNSTarget([]string{}) + if p.interceptDNSTargetService != "Old service" || len(h.setCalls) != 0 { + t.Fatal("failed backup/cleanup lost old ownership") + } + if mode != "cleanup" && len(h.resetCalls) != 0 { + t.Fatal("backup failure attempted cleanup") + } + if got := h.dns["Old service"]; !slices.Equal(got, []string{"10.0.0.53"}) { + t.Fatal("old service changed", got) + } + if _, err := os.Stat(h.statePath); err != nil { + t.Fatal("ownership file lost", err) + } + }) + } +} + +func TestNativeCLATTargetRouteChange(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + h.nativeCLAT = true + h.dhcpErr = errors.New("no packet") + calls := 0 + interceptDefaultRouteInterfaceFn = func() (string, error) { + calls++ + if calls > 1 { + return "en9", nil + } + return "en1", nil + } + p.ensureInterceptDNSTarget([]string{}) + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 { + t.Fatal("route change mutated DNS") + } +} diff --git a/cmd/cli/dns_target_native_reader_test.go b/cmd/cli/dns_target_native_reader_test.go new file mode 100644 index 00000000..62c33e26 --- /dev/null +++ b/cmd/cli/dns_target_native_reader_test.go @@ -0,0 +1,312 @@ +package cli + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "path/filepath" + "slices" + "strings" + "testing" + "time" +) + +const nativeTargetTestServiceID = "22222222-2222-2222-2222-222222222222" +const nativeTargetOtherServiceID = "11111111-1111-1111-1111-111111111111" + +func nativeTargetTestGlobal(id, device string) string { + return fmt.Sprintf(" {\n PrimaryInterface : %s\n PrimaryService : %s\n Router : 192.0.0.1\n}", device, id) +} + +func nativeTargetTestState() string { + return " {\n CLAT46 : TRUE\n InterfaceName : en1\n Router : 192.0.0.1\n Addresses : {\n 0 : 192.0.0.2\n }\n}" +} + +func nativeTargetTestAddrs(ips ...string) []net.Addr { + var out []net.Addr + for _, s := range ips { + ip, n, err := net.ParseCIDR(s) + if err != nil { + panic(err) + } + n.IP = ip + out = append(out, n) + } + return out +} + +func nativeTargetTestReader(t *testing.T, outputs []string, calls *[]string) nativeTargetReader { + t.Helper() + return nativeTargetReader{ + run: func(ctx context.Context, input, path string, args ...string) ([]byte, error) { + if ctx.Err() != nil { + return nil, ctx.Err() + } + if path != "/usr/sbin/scutil" || len(args) != 0 { + t.Fatalf("unexpected command %q %v", path, args) + } + *calls = append(*calls, input) + if len(*calls) > len(outputs) { + t.Fatalf("unexpected read: %q", input) + } + return []byte(outputs[len(*calls)-1]), nil + }, + interfaceByName: func(name string) (*net.Interface, error) { return &net.Interface{Name: name, Flags: net.FlagUp}, nil }, + interfaceAddrs: func(*net.Interface) ([]net.Addr, error) { + return nativeTargetTestAddrs("192.0.0.2/32", "2001:db8::1/64"), nil + }, + } +} + +func nativeTargetTestOutputs() []string { + global := nativeTargetTestGlobal(nativeTargetTestServiceID, "en1") + return []string{global, nativeTargetTestState(), " {\n UserDefinedName : Primary Wi-Fi\n}", global} +} + +func TestNativeTargetReaderIdentityAndEvidence(t *testing.T) { + for _, tc := range []struct { + name string + change func(*nativeTargetReader, []string) + want bool + }{ + {name: "primary UUID exact service", want: true}, + {name: "ULA with native proof", want: true, change: func(r *nativeTargetReader, _ []string) { + r.interfaceAddrs = func(*net.Interface) ([]net.Addr, error) { + return nativeTargetTestAddrs("192.0.0.2/32", "fd12:3456::1/64"), nil + } + }}, + {name: "ULA without native proof", change: func(_ *nativeTargetReader, out []string) { out[1] = strings.Replace(out[1], "TRUE", "FALSE", 1) }}, + {name: "down interface", change: func(r *nativeTargetReader, _ []string) { + r.interfaceByName = func(name string) (*net.Interface, error) { return &net.Interface{Name: name}, nil } + }}, + {name: "missing interface", change: func(r *nativeTargetReader, _ []string) { + r.interfaceByName = func(string) (*net.Interface, error) { return nil, errors.New("gone") } + }}, + {name: "address error", change: func(r *nativeTargetReader, _ []string) { + r.interfaceAddrs = func(*net.Interface) ([]net.Addr, error) { return nil, errors.New("gone") } + }}, + {name: "ordinary IPv4", change: func(r *nativeTargetReader, _ []string) { + r.interfaceAddrs = func(*net.Interface) ([]net.Addr, error) { + return nativeTargetTestAddrs("192.0.0.2/32", "2001:db8::1/64", "10.0.0.2/24"), nil + } + }}, + {name: "published address mismatch", change: func(_ *nativeTargetReader, out []string) { + out[1] = strings.Replace(out[1], "192.0.0.2", "192.0.0.5", 1) + }}, + {name: "wrong device", change: func(_ *nativeTargetReader, out []string) { + out[0] = nativeTargetTestGlobal(nativeTargetTestServiceID, "en9") + }}, + {name: "wrong service device", change: func(_ *nativeTargetReader, out []string) { out[1] = strings.Replace(out[1], "en1", "en9", 1) }}, + {name: "invalid UUID", change: func(_ *nativeTargetReader, out []string) { out[0] = nativeTargetTestGlobal("untrusted", "en1") }}, + {name: "primary changed", change: func(_ *nativeTargetReader, out []string) { + out[3] = nativeTargetTestGlobal(nativeTargetOtherServiceID, "en1") + }}, + {name: "missing name", change: func(_ *nativeTargetReader, out []string) { out[2] = "No such key" }}, + {name: "DHCP embedded dictionary", change: func(_ *nativeTargetReader, out []string) { out[1] = "DHCPPacket : " + out[1] }}, + {name: "SSID embedded name", change: func(_ *nativeTargetReader, out []string) { + out[2] = " {\n SSID : network\n UserDefinedName : Primary Wi-Fi\n}" + }}, + } { + t.Run(tc.name, func(t *testing.T) { + out := nativeTargetTestOutputs() + var calls []string + r := nativeTargetTestReader(t, out, &calls) + if tc.change != nil { + tc.change(&r, out) + } + got, err := r.defaultService(context.Background(), "en1") + if (got.ID != "" && err == nil) != tc.want { + t.Fatalf("service=%+v err=%v calls=%v", got, err, calls) + } + if tc.want { + want := nativeTargetService{ID: nativeTargetTestServiceID, Name: "Primary Wi-Fi", Device: "en1"} + if got != want { + t.Fatalf("got %+v want %+v", got, want) + } + wantCalls := []string{"show State:/Network/Global/IPv4\nquit\n", "show State:/Network/Service/" + nativeTargetTestServiceID + "/IPv4\nquit\n", "show Setup:/Network/Service/" + nativeTargetTestServiceID + "\nquit\n", "show State:/Network/Global/IPv4\nquit\n"} + if !slices.Equal(calls, wantCalls) { + t.Fatalf("calls=%v", calls) + } + } + }) + } +} + +func TestNativeTargetReaderFailuresAndSharedBudget(t *testing.T) { + for failAt := 1; failAt <= 4; failAt++ { + t.Run(fmt.Sprint(failAt), func(t *testing.T) { + var calls []string + r := nativeTargetTestReader(t, nativeTargetTestOutputs(), &calls) + run := r.run + count := 0 + r.run = func(ctx context.Context, input, path string, args ...string) ([]byte, error) { + count++ + if count == failAt { + return nil, errors.New("read failed") + } + return run(ctx, input, path, args...) + } + if got, err := r.defaultService(context.Background(), "en1"); got.ID != "" || err == nil || count != failAt { + t.Fatalf("got=%v err=%v count=%d", got, err, count) + } + }) + } + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + var calls []string + r := nativeTargetTestReader(t, nativeTargetTestOutputs(), &calls) + run := r.run + r.run = func(actual context.Context, input, path string, args ...string) ([]byte, error) { + if actual != ctx { + t.Fatal("reader renewed the shared context") + } + out, err := run(actual, input, path, args...) + if len(calls) == 2 { + cancel() + } + return out, err + } + if got, err := r.defaultService(ctx, "en1"); got.ID != "" || !errors.Is(err, context.Canceled) { + t.Fatalf("got=%v err=%v", got, err) + } + if len(calls) != 2 { + t.Fatalf("reads after cancellation: %v", calls) + } +} + +func TestNativeServiceNameStrict(t *testing.T) { + for _, name := range []string{"Wi-Fi", "iPhone USB", "Office : Wi-Fi", "Téléphone", "Wi-Fi ", " Wi-Fi"} { + got, err := parseNativeServiceName(" {\n UserDefinedName : " + name + "\n}") + if err != nil || got != name { + t.Fatalf("%q: %q %v", name, got, err) + } + } + for _, out := range []string{"", "No such key", " {\n UserDefinedName : \n}", " {\n UserDefinedName : Wi-Fi\n UserDefinedName : Evil\n}", " {\n DHCPPacket : {\n UserDefinedName : Evil\n}\n}", " {\n UserDefinedName : ../bad\n}"} { + if _, err := parseNativeServiceName(out); err == nil { + t.Fatalf("accepted %q", out) + } + } +} + +// Exercise the real exec runner without macOS tools or host network effects. +func TestNativeTargetCommandHelper(t *testing.T) { + i := slices.Index(os.Args, "--native-target-helper") + if i < 0 { + return + } + switch os.Args[i+1] { + case "large": + _, _ = os.Stdout.Write([]byte(strings.Repeat("x", nativeTargetMaxOutput+1))) + case "exact": + _, _ = os.Stdout.Write([]byte(strings.Repeat("x", nativeTargetMaxOutput))) + case "slow": + time.Sleep(10 * time.Second) + case "fail": + fmt.Print(" {}") + os.Exit(1) + default: + fmt.Print("ok") + } + os.Exit(0) +} + +func TestNativeTargetCommandBounds(t *testing.T) { + for _, mode := range []string{"ok", "exact", "large", "slow", "fail"} { + t.Run(mode, func(t *testing.T) { + budget := 3 * time.Second + if mode == "slow" { + budget = 100 * time.Millisecond + } + ctx, cancel := context.WithTimeout(context.Background(), budget) + defer cancel() + start := time.Now() + out, err := runNativeTargetCommand(ctx, "", os.Args[0], "-test.run=^TestNativeTargetCommandHelper$", "--", "--native-target-helper", mode) + if mode == "ok" || mode == "exact" { + want := 2 + if mode == "exact" { + want = nativeTargetMaxOutput + } + if err != nil || len(out) != want { + t.Fatalf("len=%d err=%v", len(out), err) + } + } else if err == nil || out != nil { + t.Fatalf("accepted incomplete evidence len=%d err=%v", len(out), err) + } + if time.Since(start) > budget+2*time.Second { + t.Fatal("command exceeded deadline slack") + } + }) + } + var out nativeTargetOutput + if _, err := out.Write([]byte(strings.Repeat("x", nativeTargetMaxOutput))); err != nil { + t.Fatal(err) + } + if _, err := out.Write([]byte("x")); !errors.Is(err, errNativeTargetOutputLimit) || out.buf.Len() != nativeTargetMaxOutput { + t.Fatalf("unbounded writer: %d %v", out.buf.Len(), err) + } +} + +func TestSaveTargetStaticDNSSnapshotRetainedBackup(t *testing.T) { + for _, content := range []string{"", "2001:db8::53", "corrupt", "10.0.0.53"} { + t.Run(content, func(t *testing.T) { + path := filepath.Join(t.TempDir(), "backup") + if err := os.WriteFile(path, []byte(content), 0600); err != nil { + t.Fatal(err) + } + err := saveTargetStaticDNSSnapshot(path, []string{"10.0.0.53"}, "10.0.0.53") + wantErr := content == "corrupt" || content == "10.0.0.53" + if (err != nil) != wantErr { + t.Fatalf("retained %q: %v", content, err) + } + if got, readErr := os.ReadFile(path); readErr != nil || string(got) != content { + t.Fatalf("retained backup changed: %q %v", got, readErr) + } + }) + } + if err := saveTargetStaticDNSSnapshot(t.TempDir(), []string{"10.0.0.53"}, "10.0.0.53"); err == nil { + t.Fatal("unreadable retained backup accepted") + } + if err := saveTargetStaticDNSSnapshot(filepath.Join(t.TempDir(), "missing"), []string{"10.0.0.53"}, "10.0.0.53"); err != nil { + t.Fatal("original automatic DNS rejected", err) + } +} + +func TestSaveTargetStaticDNSSnapshot(t *testing.T) { + path := filepath.Join(t.TempDir(), "backup") + original := "2001:db8::53" + if err := saveTargetStaticDNSSnapshot(path, []string{original}, ""); err != nil { + t.Fatal(err) + } + if err := saveTargetStaticDNSSnapshot(path, []string{"10.0.0.53"}, "10.0.0.53"); err != nil { + t.Fatal(err) + } + if got, err := os.ReadFile(path); err != nil || string(got) != original { + t.Fatalf("owned value replaced original: %q %v", got, err) + } + if err := saveTargetStaticDNSSnapshot(path, []string{"bad"}, ""); err == nil { + t.Fatal("invalid snapshot accepted") + } + if err := saveTargetStaticDNSSnapshot(path, nil, ""); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) { + t.Fatal("empty snapshot left stale backup") + } + if err := saveTargetStaticDNSSnapshot(path, nil, ""); err != nil { + t.Fatal("missing backup removal should succeed", err) + } + if err := os.Mkdir(path, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(path, "block"), nil, 0600); err != nil { + t.Fatal(err) + } + if err := saveTargetStaticDNSSnapshot(path, nil, ""); err == nil { + t.Fatal("remove failure swallowed") + } + if err := saveTargetStaticDNSSnapshot(path, []string{original}, ""); err == nil { + t.Fatal("write failure swallowed") + } +} diff --git a/cmd/cli/dns_target_native_test.go b/cmd/cli/dns_target_native_test.go new file mode 100644 index 00000000..9d207029 --- /dev/null +++ b/cmd/cli/dns_target_native_test.go @@ -0,0 +1,55 @@ +package cli + +import ( + "net" + "strings" + "testing" +) + +func TestNativeIPv4Store(t *testing.T) { + valid := " {\n CLAT46 : TRUE\n InterfaceName : en7\n Router : 192.0.0.1\n Addresses : {\n 0 : 192.0.0.2\n }\n}" + if got, err := parseNativeIPv4Store(valid); err != nil || got["CLAT46"] != "TRUE" { + t.Fatalf("%v %v", got, err) + } + for _, bad := range []string{"", "No such key", strings.Replace(valid, "TRUE", "1", 1), strings.Replace(valid, "InterfaceName", "DHCPPacket", 1), strings.Replace(valid, " Router", " CLAT46 : TRUE\n Router", 1), valid + "\n {}", " {\nPrimaryService : bad\n}"} { + if _, err := parseNativeIPv4Store(bad); err == nil { + t.Fatalf("accepted %q", bad) + } + } +} + +func TestTargetStaticDNSStrict(t *testing.T) { + for _, good := range []string{"There aren't any DNS Servers set on Wi-Fi.", "2001:db8::53\n", "1.1.1.1\n2001:db8::53"} { + if _, err := parseTargetStaticDNS(good, "Wi-Fi"); err != nil { + t.Fatal(err) + } + } + for _, bad := range []string{"", "read failed", "There aren't any DNS Servers set on Other.", "1.1.1.1\nunexpected"} { + if _, err := parseTargetStaticDNS(bad, "Wi-Fi"); err == nil { + t.Fatalf("accepted %q", bad) + } + } +} + +func TestNativeCLATAddresses(t *testing.T) { + for _, tc := range []struct { + ips []string + want bool + }{ + {[]string{"192.0.0.2/32", "2001:db8::1/64"}, true}, + {[]string{"192.0.0.5/32", "2001:db8::1/64", "fe80::1/64"}, true}, + {[]string{"192.0.0.2/32", "2001:db8::1/64", "192.168.1.2/24"}, false}, + {[]string{"192.0.0.2/32", "fe80::1/64"}, false}, + {[]string{"2001:db8::1/64"}, false}, + } { + var addrs []net.Addr + for _, s := range tc.ips { + ip, n, _ := net.ParseCIDR(s) + n.IP = ip + addrs = append(addrs, n) + } + if got := nativeCLATAddresses(addrs); got != tc.want { + t.Fatalf("%v: %v", tc.ips, got) + } + } +} diff --git a/cmd/cli/pf_diagnostic.go b/cmd/cli/pf_diagnostic.go new file mode 100644 index 00000000..14969e67 --- /dev/null +++ b/cmd/cli/pf_diagnostic.go @@ -0,0 +1,192 @@ +package cli + +import ( + "bytes" + "context" + "errors" + "io" + "os/exec" + "regexp" + "strconv" + "strings" + "sync" + "time" +) + +const ( + pfDiagnosticInterval = time.Minute + pfDiagnosticTimeout = 500 * time.Millisecond + pfDiagnosticMaxBytes = 64 << 10 +) + +// Only these two exact, ctrld-generated rules are identified. PF's @N indexes +// are not stable identifiers. No rule text or other anchor contents are logged. +var pfDiagnosticRule = regexp.MustCompile(`^@[0-9]+ block drop out quick on ! ?lo0 inet6 proto (udp|tcp) from any to any port = 53$`) +var pfDiagnosticCounters = regexp.MustCompile(`^\[ Evaluations: +([0-9]+) +Packets: +([0-9]+) +Bytes: +([0-9]+) +States: +([0-9]+) +\]$`) + +type pfDiagnosticRuleCount struct { + Installed bool + Known bool + Packets uint64 +} + +type pfDiagnosticSnapshot struct { + Code string + UDP, TCP pfDiagnosticRuleCount + RouteV4, RouteV6 string + AddressCode string + NativeIPv4, CLAT bool +} + +// A missing or unrecognized rule is not a zero counter. A counter describes all +// traffic matching that rule since its last reset, not delivery of our UDP4 probe. +func parsePFDiagnostic(data []byte) pfDiagnosticSnapshot { + s := pfDiagnosticSnapshot{Code: "rules_not_observed"} + var current *pfDiagnosticRuleCount + duplicates := false + for _, raw := range strings.Split(string(data), "\n") { + line := strings.TrimSpace(raw) + if strings.HasPrefix(line, "@") { + current = nil + if m := pfDiagnosticRule.FindStringSubmatch(line); m != nil { + current = &s.UDP + if m[1] == "tcp" { + current = &s.TCP + } + if current.Installed { + duplicates = true + } + current.Installed = true + } + continue + } + if current == nil { + continue + } + if strings.HasPrefix(line, "[ Evaluations:") { + m := pfDiagnosticCounters.FindStringSubmatch(line) + if m != nil { + valid := true + for _, value := range m[1:] { + if _, err := strconv.ParseUint(value, 10, 64); err != nil { + valid = false + } + } + if valid { + current.Packets, _ = strconv.ParseUint(m[2], 10, 64) + current.Known = true + } + } + current = nil // never attach a later rule's counters to this rule + } + } + switch { + case duplicates: + s.Code = "ambiguous_rules" + s.UDP.Known, s.TCP.Known = false, false + case s.UDP.Known && s.TCP.Known: + s.Code = "ok" + case s.UDP.Installed || s.TCP.Installed: + s.Code = "partial_or_malformed" + } + return s +} + +var errPFDiagnosticLimit = errors.New("PF diagnostic output limit") + +type pfDiagnosticBuffer struct { + data bytes.Buffer // do not embed: promoted ReadFrom would bypass Write's cap + limited bool +} + +func (b *pfDiagnosticBuffer) Len() int { return b.data.Len() } + +func (b *pfDiagnosticBuffer) Write(p []byte) (int, error) { + if len(p) > pfDiagnosticMaxBytes-b.Len() { + b.limited = true + return 0, errPFDiagnosticLimit + } + return b.data.Write(p) +} + +// Stdout is bounded, stderr is discarded, and no raw command error escapes. +// WaitDelay also bounds inherited stdout pipes if a helper fails to close them. +func runPFDiagnosticCommand(ctx context.Context, cmd *exec.Cmd) pfDiagnosticSnapshot { + var out pfDiagnosticBuffer + cmd.Stdout, cmd.Stderr = &out, io.Discard + cmd.WaitDelay = 100 * time.Millisecond + err := cmd.Run() + code := "" + switch { + case ctx.Err() != nil: + code = pfProbeContextCode(ctx) + case out.limited: + code = "output_limit" + case err != nil: + code = "read_failed" + } + if code != "" { + return pfDiagnosticSnapshot{Code: code} + } + return parsePFDiagnostic(out.data.Bytes()) +} + +type pfDiagnosticEvent struct { + Snapshot pfDiagnosticSnapshot + Observation pfProbeObservation + Generation uint64 + Age time.Duration + Cached bool + UDPDelta, TCPDelta uint64 + UDPDeltaCode, TCPDeltaCode string +} + +// Deltas are shared-traffic observations, never per-probe attribution. Even an +// increasing counter may span an unobserved reload, so it is not an exact total. +func pfDiagnosticDelta(before, after pfDiagnosticRuleCount) (uint64, string) { + if !before.Known || !after.Known { + return 0, "unknown" + } + if after.Packets < before.Packets { + return 0, "reset" + } + return after.Packets - before.Packets, "shared_traffic" +} + +type pfDiagnosticState struct { + mu sync.Mutex + active bool + captured time.Time + generation uint64 + snapshot pfDiagnosticSnapshot + lastCondition pfProbeCondition +} + +// Healthy polls do no work. Failures capture at most once a minute; a restoration +// inside that interval names the cached capture and its age, not a fresh reading. +// Identical outcomes and snapshots are quiet. This state never controls repair. +func (s *pfDiagnosticState) observe(now time.Time, o pfProbeObservation, capture func() pfDiagnosticSnapshot) (pfDiagnosticEvent, bool) { + if o.code == "canceled" || !s.mu.TryLock() { + return pfDiagnosticEvent{}, false + } + defer s.mu.Unlock() + if o.result == pfProbeIntercepted && !s.active { + return pfDiagnosticEvent{}, false + } + condition := pfProbeCondition{o.result, o.stage, o.code, o.target} + e := pfDiagnosticEvent{Observation: o, Cached: true, UDPDeltaCode: "unknown", TCPDeltaCode: "unknown"} + changed := condition != s.lastCondition + if s.captured.IsZero() || now.Sub(s.captured) >= pfDiagnosticInterval { + next := capture() + changed = changed || next != s.snapshot + e.UDPDelta, e.UDPDeltaCode = pfDiagnosticDelta(s.snapshot.UDP, next.UDP) + e.TCPDelta, e.TCPDeltaCode = pfDiagnosticDelta(s.snapshot.TCP, next.TCP) + s.snapshot, s.captured = next, now + s.generation++ + e.Cached = false + } + s.active = o.result != pfProbeIntercepted + s.lastCondition = condition + e.Snapshot, e.Generation, e.Age = s.snapshot, s.generation, now.Sub(s.captured) + return e, changed +} diff --git a/cmd/cli/pf_diagnostic_darwin.go b/cmd/cli/pf_diagnostic_darwin.go new file mode 100644 index 00000000..51c57526 --- /dev/null +++ b/cmd/cli/pf_diagnostic_darwin.go @@ -0,0 +1,50 @@ +//go:build darwin + +package cli + +import ( + "context" + "os/exec" + "time" +) + +// This is the only command added by diagnostics: read only the exact ctrld +// anchor, never recurse or dump system rules/states. Tests replace this seam +// before driving the real probe, including on privileged native runners. +var capturePFDiagnosticFn = capturePFDiagnostic + +func capturePFDiagnostic() pfDiagnosticSnapshot { + ctx, cancel := context.WithTimeout(context.Background(), pfDiagnosticTimeout) + defer cancel() + s := runPFDiagnosticCommand(ctx, exec.CommandContext(ctx, "/sbin/pfctl", "-a", pfAnchorName, "-vvsr")) + v4, v6 := defaultRoutesFn() + s.RouteV4, s.RouteV6 = v4.Interface, v6.Interface + state, err := readNetworkSourceStateFn() + s.AddressCode = "read_failed" + if err == nil && state != nil { + s.CLAT = dns64StateHasCLAT(state) + s.AddressCode = "default_v4_unavailable" + if iface, ok := state.Interface[v4.Interface]; ok && iface.Interface != nil && iface.IsUp() { + s.AddressCode = "ok" + for _, prefix := range state.InterfaceIPs[v4.Interface] { + addr := prefix.Addr().Unmap() + if addr.Is4() && !clatPrefix.Contains(addr) && !addr.IsLoopback() && !addr.IsLinkLocalUnicast() && !addr.IsUnspecified() && !addr.IsMulticast() { + s.NativeIPv4 = true + } + } + } + } + return s +} + +func (p *prog) logPFIPv6Diagnostic(o pfProbeObservation, family string) { + state, ok := p.dnsInterceptState.(*pfState) + if !ok || state == nil { + return + } + e, emit := state.ipv6Diagnostic.observe(time.Now(), o, capturePFDiagnosticFn) + if !emit { + return + } + logPFDiagnostic(e, family) +} diff --git a/cmd/cli/pf_diagnostic_darwin_test.go b/cmd/cli/pf_diagnostic_darwin_test.go new file mode 100644 index 00000000..332e323c --- /dev/null +++ b/cmd/cli/pf_diagnostic_darwin_test.go @@ -0,0 +1,62 @@ +//go:build darwin + +package cli + +import ( + "encoding/json" + "strings" + "testing" +) + +func stubPFDiagnosticCapture(t *testing.T) *int { + t.Helper() + old := capturePFDiagnosticFn + t.Cleanup(func() { capturePFDiagnosticFn = old }) + calls := 0 + capturePFDiagnosticFn = func() pfDiagnosticSnapshot { + calls++ + s := parsePFDiagnostic([]byte(pfDiagnosticFixture)) + s.RouteV4, s.RouteV6, s.AddressCode = "en0", "en0", "ok" + s.CLAT = true + return s + } + return &calls +} + +// Exercise the production probe -> diagnostic handoff, without executing a PF +// command or sending DNS. Removing that handoff must make this test fail. +func TestPFDiagnosticProbeWiringAndPrivacy(t *testing.T) { + reads := stubPFDiagnosticCapture(t) + old := pfProbeNameservers + t.Cleanup(func() { pfProbeNameservers = old; pfProbeLogs = pfProbeLogState{} }) + pfProbeNameservers = func() []string { return []string{"[fe80::1%en0]:53", "76.76.2.0:53"} } + p := &prog{dnsInterceptState: &pfState{}} + p.recoveryGen.Store(7) + before := len(logOutput.String()) + p.probePFIntercept() + p.probePFIntercept() + p.logPFIPv6Diagnostic(pfProbeObservation{result: pfProbeIntercepted, recoveryGeneration: 7, stage: "received", target: "192.0.2.1:53"}, "ipv4") + p.logPFIPv6Diagnostic(pfProbeObservation{result: pfProbeIntercepted}, "ipv4") + var events []map[string]any + for _, line := range strings.Split(logOutput.String()[before:], "\n") { + var e map[string]any + if json.Unmarshal([]byte(line), &e) != nil || e["message"] != "DNS intercept IPv6 block diagnostic" { + continue + } + events = append(events, e) + for _, forbidden := range []string{"_pf-probe", "Owner", "Inserted", "Bytes:", "from any", "probe_id", "client_address"} { + if strings.Contains(line, forbidden) { + t.Fatalf("raw payload escaped: %s", forbidden) + } + } + if e["counter_scope"] != "shared_traffic_not_probe_attribution" || e["ipv6_dns_udp_packets"] != float64(812) || e["recovery_generation"] != float64(7) { + t.Fatalf("missing evidence: %v", e) + } + } + if *reads != 1 || len(events) != 2 { + t.Fatalf("reads=%d events=%v", *reads, events) + } + if events[0]["outcome"] != "indeterminate" || events[1]["outcome"] != "intercepted" || events[1]["capture_cached"] != true { + t.Fatalf("missing restore: %v", events) + } +} diff --git a/cmd/cli/pf_diagnostic_log.go b/cmd/cli/pf_diagnostic_log.go new file mode 100644 index 00000000..e328799b --- /dev/null +++ b/cmd/cli/pf_diagnostic_log.go @@ -0,0 +1,47 @@ +package cli + +import "github.com/rs/zerolog" + +func logPFDiagnosticRule(e *zerolog.Event, id string, r pfDiagnosticRuleCount, delta uint64, deltaCode string) { + // false means this exact rule was not observed, not proof no IPv6 block exists. + e.Bool(id+"_rule_observed", r.Installed).Bool(id+"_counter_known", r.Known) + if r.Known { + e.Uint64(id+"_packets", r.Packets) + } + e.Str(id+"_delta_status", deltaCode) + if deltaCode == "shared_traffic" { + e.Uint64(id+"_packet_delta", delta) + } +} + +func logPFDiagnostic(e pfDiagnosticEvent, family string) { + o := e.Observation + s := e.Snapshot + // Recovery generation and target associate this snapshot with the existing + // probe-result event without repeating the probe's DNS query name. + probeFamily := "none" + if o.stage != "target" && o.target != "" && family == "ipv4" { + probeFamily = "udp4" + } + log := mainLog.Load().Debug(). + Uint64("capture_generation", e.Generation). + Dur("capture_age_ms", e.Age).Bool("capture_cached", e.Cached). + Uint64("recovery_generation", o.recoveryGeneration). + Str("resolver_target", o.target).Str("resolver_family", family). + Str("probe_transport", probeFamily).Str("outcome", o.result.String()). + Str("stage", o.stage).Str("probe_error_code", o.code). + Str("pf_read_status", s.Code). + Str("counter_scope", "shared_traffic_not_probe_attribution"). + Str("default_route_v4_interface", s.RouteV4). + Str("default_route_v6_interface", s.RouteV6). + Str("address_status", s.AddressCode) + if s.AddressCode == "ok" || s.AddressCode == "default_v4_unavailable" { + log.Bool("clat_address_present", s.CLAT) + } + if s.AddressCode == "ok" { + log.Bool("native_ipv4_on_default_interface", s.NativeIPv4) + } + logPFDiagnosticRule(log, "ipv6_dns_udp", s.UDP, e.UDPDelta, e.UDPDeltaCode) + logPFDiagnosticRule(log, "ipv6_dns_tcp", s.TCP, e.TCPDelta, e.TCPDeltaCode) + log.Msg("DNS intercept IPv6 block diagnostic") +} diff --git a/cmd/cli/pf_diagnostic_log_test.go b/cmd/cli/pf_diagnostic_log_test.go new file mode 100644 index 00000000..0676fd0f --- /dev/null +++ b/cmd/cli/pf_diagnostic_log_test.go @@ -0,0 +1,54 @@ +package cli + +import ( + "encoding/json" + "io" + "strings" + "testing" +) + +func TestPFDiagnosticLogAllowlist(t *testing.T) { + // Windows tests can replace the package's initial logger before this test. + previous := mainLog.Load() + replacement := previous.Output(io.Discard) + mainLog.Store(&replacement) + t.Cleanup(func() { mainLog.Store(previous) }) + + for _, code := range []string{"ok", "read_failed", "partial_or_malformed"} { + t.Run(code, func(t *testing.T) { + s := parsePFDiagnostic([]byte(pfDiagnosticFixture)) + s.Code, s.AddressCode = code, "read_failed" + if code != "ok" { + s.UDP.Known, s.TCP.Known = false, false + } + e := pfDiagnosticEvent{Snapshot: s, Generation: 3, UDPDeltaCode: "unknown", TCPDeltaCode: "unknown", + Observation: pfProbeObservation{result: pfProbeNotIntercepted, probeID: "private-query.invalid", target: "192.0.2.1:53", recoveryGeneration: 7, stage: "delivery", code: "timeout"}} + output := captureDebugMainLog(t) + logPFDiagnostic(e, "ipv4") + line := strings.TrimSpace(output.String()) + var event map[string]any + if err := json.Unmarshal([]byte(line), &event); err != nil { + t.Fatal(err) + } + for _, forbidden := range []string{"private-query", "probe_id", "Owner", "Inserted", "from any", "Bytes:"} { + if strings.Contains(line, forbidden) { + t.Fatalf("raw payload escaped: %s", forbidden) + } + } + if event["pf_read_status"] != code || event["probe_transport"] != "udp4" || event["capture_generation"] != float64(3) || event["recovery_generation"] != float64(7) || event["counter_scope"] != "shared_traffic_not_probe_attribution" { + t.Fatalf("missing evidence: %v", event) + } + for _, field := range []string{"ipv6_dns_udp_packets", "ipv6_dns_tcp_packets"} { + _, exists := event[field] + if exists != (code == "ok") { + t.Fatalf("unknown counters serialized as zero: %v", event) + } + } + for _, field := range []string{"ipv6_dns_udp_packet_delta", "ipv6_dns_tcp_packet_delta", "clat_address_present", "native_ipv4_on_default_interface"} { + if _, exists := event[field]; exists { + t.Fatalf("unknown evidence serialized: %s", field) + } + } + }) + } +} diff --git a/cmd/cli/pf_diagnostic_test.go b/cmd/cli/pf_diagnostic_test.go new file mode 100644 index 00000000..2dcd12f0 --- /dev/null +++ b/cmd/cli/pf_diagnostic_test.go @@ -0,0 +1,208 @@ +package cli + +import ( + "context" + "fmt" + "os" + "os/exec" + "strings" + "testing" + "time" +) + +// Native-format provenance: macOS pfctl's @N / Owner / Evaluations / Inserted +// layout is published at https://apple.stackexchange.com/questions/335019 +// (pfctl -vvv -s all); the counter layout is also reported on macOS 26.3 at +// https://apple.stackexchange.com/questions/486150 (pfctl -s rules -v). +// This is an ADAPTED fixture, not a capture from a ctrld host: headers use the +// two expanded forms of generatePFRules' IPv6 DNS block, and numbers are test +// values. Keep the native spacing/metadata; do not use pf.conf input as output. +const pfDiagnosticFixture = `@30 block drop out quick on ! lo0 inet6 proto udp from any to any port = 53 + [ Owner : nil Priority : 0 ] + [ Evaluations: 574091 Packets: 812 Bytes: 51968 States: 0 ] + [ Inserted: uid 0 pid 68 ] +@31 block drop out quick on ! lo0 inet6 proto tcp from any to any port = 53 + [ Owner : nil Priority : 0 ] + [ Evaluations: 573678 Packets: 210 Bytes: 13152 States: 0 ] + [ Inserted: uid 0 pid 68 ] +` + +func TestPFDiagnosticNativeFormat(t *testing.T) { + s := parsePFDiagnostic([]byte(pfDiagnosticFixture)) + if s.Code != "ok" || !s.UDP.Installed || !s.TCP.Installed || !s.UDP.Known || !s.TCP.Known || s.UDP.Packets != 812 || s.TCP.Packets != 210 { + t.Fatalf("native-format fixture: %+v", s) + } + // IDs and metadata do not become logged identity, and need not be stable. + changed := strings.NewReplacer("@30", "@1", "@31", "@2", "! lo0", "!lo0", "pid 68", "pid 99").Replace(pfDiagnosticFixture) + if got := parsePFDiagnostic([]byte(changed)); got != s { + t.Fatalf("unstable identity: %+v", got) + } +} + +func TestPFDiagnosticUnknownIsNotZero(t *testing.T) { + for _, tc := range []struct { + name, input, code string + installed bool + }{ + {"empty", "", "rules_not_observed", false}, + {"unrelated", strings.ReplaceAll(pfDiagnosticFixture, "inet6", "inet"), "rules_not_observed", false}, + {"other interface", strings.ReplaceAll(pfDiagnosticFixture, "! lo0", "en0"), "rules_not_observed", false}, + {"not exact", strings.ReplaceAll(pfDiagnosticFixture, "port = 53", "port = 53 label private"), "rules_not_observed", false}, + {"malformed", strings.ReplaceAll(pfDiagnosticFixture, "Packets:", "Packets: invalid"), "partial_or_malformed", true}, + {"overflow", strings.ReplaceAll(pfDiagnosticFixture, "Packets:", "Packets: 18446744073709551616"), "partial_or_malformed", true}, + {"duplicate", pfDiagnosticFixture + pfDiagnosticFixture, "ambiguous_rules", true}, + {"no counters", strings.Split(pfDiagnosticFixture, "\n")[0], "partial_or_malformed", true}, + } { + t.Run(tc.name, func(t *testing.T) { + s := parsePFDiagnostic([]byte(tc.input)) + if s.Code != tc.code || s.UDP.Installed != tc.installed || s.UDP.Known || s.TCP.Known { + t.Fatalf("unknown misreported: %+v", s) + } + }) + } + zero := strings.NewReplacer("812", "0", "210", "0").Replace(pfDiagnosticFixture) + if s := parsePFDiagnostic([]byte(zero)); s.Code != "ok" || !s.UDP.Known || s.UDP.Packets != 0 { + t.Fatalf("real zero: %+v", s) + } + // A missing UDP counter must not consume the following unrelated rule's one. + input := strings.Split(pfDiagnosticFixture, "\n")[0] + "\n@32 pass out all\n [ Evaluations: 1 Packets: 900 Bytes: 100 States: 0 ]\n" + if s := parsePFDiagnostic([]byte(input)); s.UDP.Known { + t.Fatalf("borrowed unrelated counter: %+v", s) + } +} + +func TestPFDiagnosticProcess(t *testing.T) { + switch os.Getenv("CTRLD_PF_DIAGNOSTIC_TEST") { + case "": + return + case "ok": + fmt.Print(pfDiagnosticFixture) + case "failure": + fmt.Print(pfDiagnosticFixture) + fmt.Fprint(os.Stderr, "private rules/client-address") + os.Exit(1) + case "limit": + fmt.Print(strings.Repeat("x", pfDiagnosticMaxBytes+1)) + case "hang": + time.Sleep(time.Minute) + } + os.Exit(0) +} + +func pfDiagnosticTestCommand(ctx context.Context, mode string) *exec.Cmd { + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestPFDiagnosticProcess$") + cmd.Env = append(os.Environ(), "CTRLD_PF_DIAGNOSTIC_TEST="+mode) + return cmd +} + +func TestPFDiagnosticCommandBounds(t *testing.T) { + for _, tc := range []struct{ mode, code string }{{"ok", "ok"}, {"failure", "read_failed"}, {"limit", "output_limit"}, {"hang", "timeout"}} { + t.Run(tc.mode, func(t *testing.T) { + d := 5 * time.Second + if tc.mode == "hang" { + d = 100 * time.Millisecond + } + ctx, cancel := context.WithTimeout(context.Background(), d) + defer cancel() + start := time.Now() + s := runPFDiagnosticCommand(ctx, pfDiagnosticTestCommand(ctx, tc.mode)) + if s.Code != tc.code { + t.Fatalf("got %+v, want %s", s, tc.code) + } + if tc.mode != "ok" && (s.UDP.Known || s.TCP.Known || s.UDP.Installed) { + t.Fatalf("failed read trusted: %+v", s) + } + if tc.mode == "hang" && time.Since(start) > time.Second { + t.Fatal("deadline not bounded") + } + }) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if got := runPFDiagnosticCommand(ctx, pfDiagnosticTestCommand(ctx, "ok")); got.Code != "canceled" { + t.Fatalf("cancellation: %+v", got) + } + if got := runPFDiagnosticCommand(context.Background(), exec.Command(t.TempDir()+"/missing")); got.Code != "read_failed" { + t.Fatalf("start failure: %+v", got) + } + var b pfDiagnosticBuffer + _, _ = b.Write(make([]byte, pfDiagnosticMaxBytes)) + if _, err := b.Write([]byte{1}); err == nil || b.Len() != pfDiagnosticMaxBytes { + t.Fatal("buffer is unbounded") + } +} + +func TestPFDiagnosticCadenceTransitionsAndReset(t *testing.T) { + var s pfDiagnosticState + now := time.Unix(1, 0) + reads := 0 + fixture := parsePFDiagnostic([]byte(pfDiagnosticFixture)) + capture := func() pfDiagnosticSnapshot { reads++; return fixture } + o := pfProbeObservation{result: pfProbeIntercepted, target: "192.0.2.1:53"} + if _, emit := s.observe(now, o, capture); emit || reads != 0 { + t.Fatal("healthy poll did work") + } + o.result, o.stage, o.code = pfProbeNotIntercepted, "delivery", "timeout" + e, emit := s.observe(now, o, capture) + if !emit || e.Cached || e.Generation != 1 || e.UDPDeltaCode != "unknown" { + t.Fatalf("first failure: %+v %v", e, emit) + } + for i := 0; i < 100; i++ { + o.probeID = fmt.Sprint(i) + o.recoveryGeneration++ + if _, emit := s.observe(now.Add(time.Second), o, capture); emit { + t.Fatal("duplicate failure noisy") + } + } + if reads != 1 { + t.Fatalf("unbounded reads: %d", reads) + } + if _, emit := s.observe(now.Add(time.Minute), o, capture); emit || reads != 2 { + t.Fatal("unchanged refresh noisy") + } + fixture.UDP.Packets++ + e, emit = s.observe(now.Add(2*time.Minute), o, capture) + if !emit || e.UDPDelta != 1 || e.UDPDeltaCode != "shared_traffic" { + t.Fatalf("delta: %+v", e) + } + fixture.UDP.Packets = 0 + e, _ = s.observe(now.Add(3*time.Minute), o, capture) + if e.UDPDeltaCode != "reset" || e.UDPDelta != 0 { + t.Fatalf("reset: %+v", e) + } + o.result = pfProbeIntercepted + e, emit = s.observe(now.Add(3*time.Minute+time.Second), o, capture) + if !emit || !e.Cached || e.Age != time.Second || e.Generation != 4 || e.UDPDeltaCode != "unknown" { + t.Fatalf("restored: %+v", e) + } + if _, emit := s.observe(now.Add(4*time.Minute), o, capture); emit || reads != 4 { + t.Fatal("healthy repeats did work") + } + o.code = "canceled" + o.result = pfProbeIndeterminate + if _, emit := s.observe(now.Add(5*time.Minute), o, capture); emit || reads != 4 { + t.Fatal("shutdown did work") + } + fixture.Code = "read_failed" + fixture.UDP.Known = false + fixture.TCP.Known = false + o.code = "unavailable" + e, emit = s.observe(now.Add(6*time.Minute), o, capture) + if !emit || e.UDPDeltaCode != "unknown" { + t.Fatalf("failed read: %+v", e) + } +} + +func TestPFDiagnosticConcurrentCaptureDoesNotBlock(t *testing.T) { + var s pfDiagnosticState + started, release, done := make(chan struct{}), make(chan struct{}), make(chan struct{}) + go func() { + defer close(done) + s.observe(time.Now(), pfProbeObservation{}, func() pfDiagnosticSnapshot { close(started); <-release; return pfDiagnosticSnapshot{} }) + }() + <-started + defer func() { close(release); <-done }() + if _, emit := s.observe(time.Now(), pfProbeObservation{}, func() pfDiagnosticSnapshot { t.Error("overlapping capture"); return pfDiagnosticSnapshot{} }); emit { + t.Fatal("concurrent capture should stand down") + } +} diff --git a/cmd/cli/pf_probe_darwin_test.go b/cmd/cli/pf_probe_darwin_test.go index 87f4d721..9fa3f6da 100644 --- a/cmd/cli/pf_probe_darwin_test.go +++ b/cmd/cli/pf_probe_darwin_test.go @@ -118,6 +118,7 @@ func TestPFProbeRepairCorrelation(t *testing.T) { } func TestPFProbeMissingTargetLogTransitions(t *testing.T) { + stubPFDiagnosticCapture(t) old := pfProbeNameservers t.Cleanup(func() { pfProbeNameservers = old; pfProbeLogs = pfProbeLogState{} }) pfProbeLogs = pfProbeLogState{} @@ -146,6 +147,7 @@ func TestPFProbeMissingTargetLogTransitions(t *testing.T) { } func TestPFProbeUnsentDoesNotReload(t *testing.T) { + stubPFDiagnosticCapture(t) oldServers, oldCommand, oldReload := pfProbeNameservers, newPFProbeCommand, forceReloadPFInterceptFn t.Cleanup(func() { pfProbeNameservers, newPFProbeCommand, forceReloadPFInterceptFn = oldServers, oldCommand, oldReload @@ -203,6 +205,7 @@ func TestPFProbeWatchdogSkipsIndeterminateAnchor(t *testing.T) { } func TestPFProbeNoIPv4TargetIsIndeterminate(t *testing.T) { + stubPFDiagnosticCapture(t) old := pfProbeNameservers t.Cleanup(func() { pfProbeNameservers = old }) pfProbeNameservers = func() []string { return []string{"[fe80::1%en0]:53", "76.76.2.0:53"} } diff --git a/docs/ipv6-only-dns-target-qa.md b/docs/ipv6-only-dns-target-qa.md new file mode 100644 index 00000000..e37d2f92 --- /dev/null +++ b/docs/ipv6-only-dns-target-qa.md @@ -0,0 +1,29 @@ +# IPv6-only DNS-target validation + +Relates to #617. Native CLAT/PF behavior is not proven by parser tests or cross-compilation. This change must not disable IPv6 DNS enforcement, change PF routing, or reclassify every failed DHCP command as empty DNS. + +## Regression gates + +- Ordinary static IPv4 DNS and successful DHCPv4 discovery retain their existing behavior. +- An explicit non-owned IPv6 loopback resolver (`::1`), alone or mixed with other static DNS, prevents the expanded fallback. Its settings and backup must remain unchanged through reconciliation and cleanup. +- The target-specific DHCP lookup shares a two-second deadline across both commands, with a 100 ms pipe-cleanup grace. Timeout remains an error, not confirmed absence. This does not impose a deadline on legacy global discovery or every operation under the target lock. +- An unknown, truncated, malformed, timed-out or permission-denied native-state read authorizes no new DNS mutation. +- A CLAT-looking address alone is insufficient. Positive native publication must match the current default interface/service, with no contradictory native IPv4 state. +- Confirmed IPv6-only state can reach the existing listener-aware target installation path despite unavailable DHCPv4 data. +- Repeated reconciliation is idempotent. Existing targets, external DNS changes, default-service switches, DHCP return and restart/stop cleanup preserve ownership. +- A failed preservation snapshot cannot authorize the new fallback write. +- Logs distinguish configured IPv6 blocking, observed rule counters, missing evidence, counter resets and successful local receipt. Shared counters cannot identify which packet was blocked. +- Diagnostics are bounded, omit raw PF dumps/query names and do not cause DNS/PF changes or repeated repair actions. + +## Native comparison + +Use an owned disposable Mac with a local console, an identified baseline/candidate and the same profile/listener. Save exact original DNS settings. Do not run host-mutating tests on shared CI hosts; native unit tests must intercept the first mutation boundary. + +1. Verify a healthy baseline on ordinary IPv4/DHCP, then establish actual host-side CLAT using an owned hotspot or isolated NAT64/PREF64 fixture. Record the interface/service mapping, native state, routes, static DNS, raw scutil DNS and both DHCP commands' exit/output. A carrier label or synthetic address assignment is not a fixture. +2. Confirm upstream and direct-listener resolution independently. Keep a pre-existing manual loopback DNS override from masking the target decision, without deleting unrelated configuration. +3. Compare startup already attached, disconnected startup followed by attachment, and a same-process transition to the IPv6-only network. Test sleep/wake separately and exclude intentionally offline/sleep periods from recovery measurements. +4. On the candidate, require the appropriate temporary target and fresh system-resolution success while still on CLAT. Record any independent PF-probe failure rather than treating working system DNS as proof of repaired transparent interception. +5. Return to IPv4/DHCP; verify target removal/restoration. Repeat with explicit static DNS, an existing tracked target, external DNS changes and an injected unknown/read-error state. Stop/uninstall must not leave DNS pointing to a dead listener. +6. Read the retained diagnostics after a failed probe and after restored receipt. Distinguish missing rules/read errors from a present IPv6 block with cumulative traffic counters. A reload or rising shared counter does not prove a particular probe's fate. + +Preserve captures privately and record the exact artifact. A successful owned-lab case supports the scoped fix; a customer retest remains a separate validation gate. diff --git a/docs/network-recovery-diagnostics.md b/docs/network-recovery-diagnostics.md index ed02baaa..224fa939 100644 --- a/docs/network-recovery-diagnostics.md +++ b/docs/network-recovery-diagnostics.md @@ -252,6 +252,44 @@ A probe-triggered reload does not count as a missing-anchor repair. See [PF DNS interception](pf-dns-intercept.md) and [DNS intercept mode](dns-intercept-mode.md) for the rule lifecycle. +## IPv6-only target fallback and PF diagnostics + +On macOS, a failed DHCPv4 read remains unknown unless configd positively publishes +CLAT for the current primary service/interface and the live interface addresses +agree. A CLAT-looking address alone is insufficient. Unknown or unsupported native +state leaves DNS unchanged. The fallback reuses the existing listener-aware target +and cleanup paths; it does not change PF enforcement or treat every command error +as empty DNS. Native output compatibility and the real hotspot scenario require +[owned-Mac validation](ipv6-only-dns-target-qa.md). + +`DNS intercept IPv6 block diagnostic` is a change-only debug event after unsuccessful +probes and their first restored receipt. It reads only the ctrld anchor using +`pfctl -a com.controld.ctrld -vvsr`, at most once per minute. The command has a +500 ms deadline and a 64 KiB output cap. A restored receipt inside the interval can +refer to the cached capture; `capture_cached`, `capture_age_ms` and +`capture_generation` make this explicit. Collection is synchronous before the +caller can repair: the 500 ms bound covers the command, not its pipe-cleanup grace +or the subsequent kernel route/interface reads. + +`pf_read_status` distinguishes unavailable/failed/unrecognized data from recognized +rules and counters. `ipv6_dns_udp_rule_observed` and its TCP equivalent mean the +exact generated rule was observed, not that it received the probe. A missing rule +match does not prove that no other IPv6 blocking exists. Packet fields are emitted +only when their counters are known. Delta status is `unknown`, `reset` or +`shared_traffic`; decreases are never reported as negative packet counts. + +Counters include all matching traffic, not just the probe, and unobserved reloads +can reset them between captures. A counter increase does not establish that CLAT +translated or PF blocked a particular packet. The event names the probe transport, +resolver target/family, recovery generation and default-route interfaces. +`clat_address_present` is an address hint, not the positive native evidence used to +authorize DNS changes. Raw rules, customer query names and interface-address lists +are not included. + +These debug events do not guarantee journal retention. !1020 separately handles +retained DNS-target decision failures. Reload completion, a configured block rule, +and successful local probe receipt remain distinct observations. + ## Reproduction boundaries Compare the affected release with the exact candidate artifact on an owned host. diff --git a/nameservers_darwin.go b/nameservers_darwin.go index 0133dc4e..2ae1c297 100644 --- a/nameservers_darwin.go +++ b/nameservers_darwin.go @@ -100,21 +100,17 @@ func getDHCPNameservers(iface string) ([]string, error) { return nil, fmt.Errorf("ipconfig not available on mobile") } - // getoption returns the selected interface's DHCP option directly and does - // not expose unrelated packet addresses to the parser. - output, err := exec.Command("ipconfig", "getoption", iface, "domain_name_server").Output() - if err == nil { - return parseDHCPOptionNameservers(output), nil - } + return dhcpNameserversFromCommands(context.Background(), iface, func(_ context.Context, args ...string) ([]byte, error) { + return exec.Command("ipconfig", args...).Output() + }) +} - // Older macOS releases can fail getoption while still exposing the packet. - // Parse the real macOS field shape, for example: - // domain_name_server (ip_mult): {192.168.1.1, 8.8.8.8} - output, packetErr := exec.Command("ipconfig", "getpacket", iface).Output() - if packetErr != nil { - return nil, fmt.Errorf("error reading DHCP DNS option: getoption: %w; getpacket: %w", err, packetErr) - } - return parseDHCPPacketNameservers(output), nil +// DHCPNameserversForInterfaceContext bounds both commands with the caller's +// context. Other system-discovery callers retain their existing behavior. +func DHCPNameserversForInterfaceContext(ctx context.Context, iface string) ([]string, error) { + return dhcpNameserversFromCommands(ctx, iface, func(ctx context.Context, args ...string) ([]byte, error) { + return dhcpCommandOutput(ctx, "/usr/sbin/ipconfig", args...) + }) } // DHCPNameserversForInterface returns DHCP option 6 for exactly iface. diff --git a/nameservers_darwin_deadline_test.go b/nameservers_darwin_deadline_test.go new file mode 100644 index 00000000..32caee37 --- /dev/null +++ b/nameservers_darwin_deadline_test.go @@ -0,0 +1,18 @@ +//go:build darwin + +package ctrld + +import ( + "context" + "errors" + "testing" +) + +func TestDHCPInterfaceContextCanceled(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + got, err := DHCPNameserversForInterfaceContext(ctx, "unused-test-interface") + if got != nil || !errors.Is(err, context.Canceled) { + t.Fatalf("canceled discovery: got=%v err=%v", got, err) + } +} diff --git a/nameservers_dhcp_commands.go b/nameservers_dhcp_commands.go new file mode 100644 index 00000000..493d4990 --- /dev/null +++ b/nameservers_dhcp_commands.go @@ -0,0 +1,41 @@ +package ctrld + +import ( + "context" + "fmt" + "os/exec" + "time" +) + +// dhcpNameserversFromCommands shares one context across both discovery attempts. +// Cancellation is unknown state, never a successful empty DNS result. +func dhcpNameserversFromCommands(ctx context.Context, iface string, run func(context.Context, ...string) ([]byte, error)) ([]string, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + output, optionErr := run(ctx, "getoption", iface, "domain_name_server") + if err := ctx.Err(); err != nil { + return nil, err + } + if optionErr == nil { + return parseDHCPOptionNameservers(output), nil + } + output, packetErr := run(ctx, "getpacket", iface) + if err := ctx.Err(); err != nil { + return nil, err + } + if packetErr != nil { + return nil, fmt.Errorf("error reading DHCP DNS option: getoption: %w; getpacket: %w", optionErr, packetErr) + } + return parseDHCPPacketNameservers(output), nil +} + +func dhcpCommandOutput(ctx context.Context, path string, args ...string) ([]byte, error) { + cmd := exec.CommandContext(ctx, path, args...) + cmd.WaitDelay = 100 * time.Millisecond + output, err := cmd.Output() + if ctx.Err() != nil { + return nil, ctx.Err() + } + return output, err +} diff --git a/nameservers_dhcp_commands_test.go b/nameservers_dhcp_commands_test.go new file mode 100644 index 00000000..acc81595 --- /dev/null +++ b/nameservers_dhcp_commands_test.go @@ -0,0 +1,112 @@ +package ctrld + +import ( + "context" + "errors" + "os" + "os/exec" + "slices" + "testing" + "time" +) + +func TestDHCPNameserversCommandFallback(t *testing.T) { + for _, tc := range []struct { + name, option, packet string + optionErr, packetErr error + want []string + calls int + wantErr bool + }{ + {name: "option", option: "192.0.2.53\n", want: []string{"192.0.2.53"}, calls: 1}, + {name: "empty option", calls: 1}, + {name: "packet", optionErr: errors.New("exit 1"), packet: "domain_name_server (ip_mult): {192.0.2.53, 192.0.2.54}", want: []string{"192.0.2.53", "192.0.2.54"}, calls: 2}, + {name: "empty packet", optionErr: errors.New("exit 1"), calls: 2}, + {name: "both fail", optionErr: errors.New("exit 1"), packetErr: errors.New("exit 1"), calls: 2, wantErr: true}, + } { + t.Run(tc.name, func(t *testing.T) { + calls := 0 + got, err := dhcpNameserversFromCommands(context.Background(), "en0", func(_ context.Context, args ...string) ([]byte, error) { + calls++ + if calls == 1 { + if !slices.Equal(args, []string{"getoption", "en0", "domain_name_server"}) { + t.Fatal(args) + } + return []byte(tc.option), tc.optionErr + } + if !slices.Equal(args, []string{"getpacket", "en0"}) { + t.Fatal(args) + } + return []byte(tc.packet), tc.packetErr + }) + if calls != tc.calls || (err != nil) != tc.wantErr || !slices.Equal(got, tc.want) { + t.Fatalf("calls=%d got=%v err=%v", calls, got, err) + } + }) + } +} + +func TestDHCPNameserversRetainsCommandCauses(t *testing.T) { + option, packet := &exec.ExitError{}, &exec.ExitError{} + calls := 0 + _, err := dhcpNameserversFromCommands(context.Background(), "en0", func(context.Context, ...string) ([]byte, error) { + calls++ + if calls == 1 { + return nil, option + } + return nil, packet + }) + var exit *exec.ExitError + if !errors.As(err, &exit) || !errors.Is(err, option) || !errors.Is(err, packet) { + t.Fatalf("DHCP command causes were flattened: %T: %v", err, err) + } +} + +func TestDHCPNameserversCancellation(t *testing.T) { + for _, cancelAt := range []int{0, 1, 2} { + ctx, cancel := context.WithCancel(context.Background()) + calls := 0 + if cancelAt == 0 { + cancel() + } + got, err := dhcpNameserversFromCommands(ctx, "en0", func(received context.Context, _ ...string) ([]byte, error) { + if received != ctx { + t.Fatal("discovery renewed the context") + } + calls++ + if calls == cancelAt { + cancel() + } + return nil, errors.New("unavailable") + }) + cancel() + if !errors.Is(err, context.Canceled) || got != nil || calls != cancelAt { + t.Fatalf("cancel at %d: got=%v err=%v calls=%d", cancelAt, got, err, calls) + } + } +} + +func TestDHCPCommandDeadlineHelper(t *testing.T) { + if os.Getenv("CTRLD_DHCP_DEADLINE_HELPER") != "1" { + return + } + time.Sleep(4 * time.Second) +} + +func TestDHCPCommandDeadline(t *testing.T) { + t.Setenv("CTRLD_DHCP_DEADLINE_HELPER", "1") + exe, err := os.Executable() + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 750*time.Millisecond) + defer cancel() + started := time.Now() + _, err = dhcpCommandOutput(ctx, exe, "-test.run=^TestDHCPCommandDeadlineHelper$") + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("err=%v", err) + } + if elapsed := time.Since(started); elapsed >= 3*time.Second { + t.Fatalf("command ignored deadline: %v", elapsed) + } +} From d779b5374cd6e31c379238d26b5e83072c211d72 Mon Sep 17 00:00:00 2001 From: Dev Scribe Date: Thu, 24 Sep 2026 00:24:31 +0000 Subject: [PATCH 23/29] macOS: use the networksetup service name for USB CLAT targets --- cmd/cli/dns_target_native.go | 10 ++- cmd/cli/dns_target_native_capture_test.go | 5 +- cmd/cli/dns_target_native_darwin_test.go | 10 ++- cmd/cli/dns_target_native_reader_test.go | 15 ++-- cmd/cli/dns_target_service.go | 73 ++++++++++++++++++ cmd/cli/dns_target_service_test.go | 94 +++++++++++++++++++++++ cmd/cli/dns_target_usb_darwin_test.go | 89 +++++++++++++++++++++ 7 files changed, 285 insertions(+), 11 deletions(-) create mode 100644 cmd/cli/dns_target_service.go create mode 100644 cmd/cli/dns_target_service_test.go create mode 100644 cmd/cli/dns_target_usb_darwin_test.go diff --git a/cmd/cli/dns_target_native.go b/cmd/cli/dns_target_native.go index f07e0337..bf8431bc 100644 --- a/cmd/cli/dns_target_native.go +++ b/cmd/cli/dns_target_native.go @@ -134,7 +134,15 @@ func (r nativeTargetReader) defaultService(ctx context.Context, device string) ( if err != nil { return unknown, err } - name, err := parseNativeServiceName(string(out)) + _, err = parseNativeServiceName(string(out)) + if err != nil { + return unknown, err + } + out, err = r.run(ctx, "", "/usr/sbin/networksetup", "-listnetworkserviceorder") + if err != nil { + return unknown, err + } + name, err := targetNetworkServiceName(string(out), device) if err != nil { return unknown, err } diff --git a/cmd/cli/dns_target_native_capture_test.go b/cmd/cli/dns_target_native_capture_test.go index 084e9f72..76051062 100644 --- a/cmd/cli/dns_target_native_capture_test.go +++ b/cmd/cli/dns_target_native_capture_test.go @@ -47,6 +47,9 @@ func TestCapturedCLATNativeReader(t *testing.T) { reader := nativeTargetReader{ run: func(_ context.Context, input, path string, args ...string) ([]byte, error) { calls++ + if path == "/usr/sbin/networksetup" && len(args) == 1 && args[0] == "-listnetworkserviceorder" && input == "" { + return []byte(nativeTargetTestServiceOrder("Wi-Fi", "en0")), nil + } if path != "/usr/sbin/scutil" || len(args) != 0 { t.Fatal("unexpected command") } @@ -77,7 +80,7 @@ func TestCapturedCLATNativeReader(t *testing.T) { } got, err := reader.defaultService(context.Background(), "en0") want := nativeTargetService{ID: id, Name: "Wi-Fi", Device: "en0"} - if err != nil || got != want || calls != 4 { + if err != nil || got != want || calls != 5 { t.Fatalf("got=%+v err=%v reads=%d", got, err, calls) } t.Log("Captured native dictionaries accepted; primary Wi-Fi service on en0 selected") diff --git a/cmd/cli/dns_target_native_darwin_test.go b/cmd/cli/dns_target_native_darwin_test.go index e6b866a2..dca408ca 100644 --- a/cmd/cli/dns_target_native_darwin_test.go +++ b/cmd/cli/dns_target_native_darwin_test.go @@ -177,8 +177,8 @@ func TestNativeCLATTargetTransitionAndReadFailure(t *testing.T) { } } -// Use the real evidence reader: en1 has two configured services, and the -// second is primary. Legacy first-match mapping must not even be consulted. +// Use the real evidence reader: the native primary UUID is still checked, +// and the command-facing service is unambiguously mapped to its device. func TestNativeCLATTargetExactPrimaryService(t *testing.T) { for _, static := range [][]string{{"9.9.9.9"}, {"2001:db8::53"}} { t.Run(strings.Join(static, ","), func(t *testing.T) { @@ -188,7 +188,9 @@ func TestNativeCLATTargetExactPrimaryService(t *testing.T) { h.dns["First Wi-Fi"] = nil h.dns["Primary Wi-Fi"] = slices.Clone(static) var calls []string - outputs := append(nativeTargetTestOutputs(), nativeTargetTestOutputs()...) + out := nativeTargetTestOutputs() + out[3] = nativeTargetTestServiceOrder("First Wi-Fi", "en9") + "(2) Primary Wi-Fi\n(Hardware Port: Wi-Fi, Device: en1)\n" + outputs := append(out, out...) r := nativeTargetTestReader(t, outputs, &calls) interceptNativeCLATDefaultServiceFn = r.defaultService interceptPatchNetIfaceNameFn = func(*net.Interface) (bool, error) { @@ -203,7 +205,7 @@ func TestNativeCLATTargetExactPrimaryService(t *testing.T) { } p := newInterceptTargetProg() p.ensureInterceptDNSTarget([]string{}) - if !slices.Equal(reads, []string{"Primary Wi-Fi", "Primary Wi-Fi"}) || len(calls) != 8 { + if !slices.Equal(reads, []string{"Primary Wi-Fi", "Primary Wi-Fi"}) || len(calls) != 10 { t.Fatalf("reads=%v calls=%v", reads, calls) } if len(h.dns["First Wi-Fi"]) != 0 { diff --git a/cmd/cli/dns_target_native_reader_test.go b/cmd/cli/dns_target_native_reader_test.go index 62c33e26..1acf7a17 100644 --- a/cmd/cli/dns_target_native_reader_test.go +++ b/cmd/cli/dns_target_native_reader_test.go @@ -44,7 +44,9 @@ func nativeTargetTestReader(t *testing.T, outputs []string, calls *[]string) nat if ctx.Err() != nil { return nil, ctx.Err() } - if path != "/usr/sbin/scutil" || len(args) != 0 { + if path == "/usr/sbin/networksetup" && slices.Equal(args, []string{"-listnetworkserviceorder"}) && input == "" { + input = "networksetup -listnetworkserviceorder" + } else if path != "/usr/sbin/scutil" || len(args) != 0 { t.Fatalf("unexpected command %q %v", path, args) } *calls = append(*calls, input) @@ -62,7 +64,7 @@ func nativeTargetTestReader(t *testing.T, outputs []string, calls *[]string) nat func nativeTargetTestOutputs() []string { global := nativeTargetTestGlobal(nativeTargetTestServiceID, "en1") - return []string{global, nativeTargetTestState(), " {\n UserDefinedName : Primary Wi-Fi\n}", global} + return []string{global, nativeTargetTestState(), " {\n UserDefinedName : Primary Wi-Fi\n}", nativeTargetTestServiceOrder("Primary Wi-Fi", "en1"), global} } func TestNativeTargetReaderIdentityAndEvidence(t *testing.T) { @@ -72,6 +74,9 @@ func TestNativeTargetReaderIdentityAndEvidence(t *testing.T) { want bool }{ {name: "primary UUID exact service", want: true}, + {name: "matching native name does not disambiguate another service on same device", want: false, change: func(_ *nativeTargetReader, out []string) { + out[3] += "(2) Secondary Wi-Fi\n(Hardware Port: Wi-Fi, Device: en1)\n" + }}, {name: "ULA with native proof", want: true, change: func(r *nativeTargetReader, _ []string) { r.interfaceAddrs = func(*net.Interface) ([]net.Addr, error) { return nativeTargetTestAddrs("192.0.0.2/32", "fd12:3456::1/64"), nil @@ -101,7 +106,7 @@ func TestNativeTargetReaderIdentityAndEvidence(t *testing.T) { {name: "wrong service device", change: func(_ *nativeTargetReader, out []string) { out[1] = strings.Replace(out[1], "en1", "en9", 1) }}, {name: "invalid UUID", change: func(_ *nativeTargetReader, out []string) { out[0] = nativeTargetTestGlobal("untrusted", "en1") }}, {name: "primary changed", change: func(_ *nativeTargetReader, out []string) { - out[3] = nativeTargetTestGlobal(nativeTargetOtherServiceID, "en1") + out[4] = nativeTargetTestGlobal(nativeTargetOtherServiceID, "en1") }}, {name: "missing name", change: func(_ *nativeTargetReader, out []string) { out[2] = "No such key" }}, {name: "DHCP embedded dictionary", change: func(_ *nativeTargetReader, out []string) { out[1] = "DHCPPacket : " + out[1] }}, @@ -125,7 +130,7 @@ func TestNativeTargetReaderIdentityAndEvidence(t *testing.T) { if got != want { t.Fatalf("got %+v want %+v", got, want) } - wantCalls := []string{"show State:/Network/Global/IPv4\nquit\n", "show State:/Network/Service/" + nativeTargetTestServiceID + "/IPv4\nquit\n", "show Setup:/Network/Service/" + nativeTargetTestServiceID + "\nquit\n", "show State:/Network/Global/IPv4\nquit\n"} + wantCalls := []string{"show State:/Network/Global/IPv4\nquit\n", "show State:/Network/Service/" + nativeTargetTestServiceID + "/IPv4\nquit\n", "show Setup:/Network/Service/" + nativeTargetTestServiceID + "\nquit\n", "networksetup -listnetworkserviceorder", "show State:/Network/Global/IPv4\nquit\n"} if !slices.Equal(calls, wantCalls) { t.Fatalf("calls=%v", calls) } @@ -135,7 +140,7 @@ func TestNativeTargetReaderIdentityAndEvidence(t *testing.T) { } func TestNativeTargetReaderFailuresAndSharedBudget(t *testing.T) { - for failAt := 1; failAt <= 4; failAt++ { + for failAt := 1; failAt <= 5; failAt++ { t.Run(fmt.Sprint(failAt), func(t *testing.T) { var calls []string r := nativeTargetTestReader(t, nativeTargetTestOutputs(), &calls) diff --git a/cmd/cli/dns_target_service.go b/cmd/cli/dns_target_service.go new file mode 100644 index 00000000..273cc8a7 --- /dev/null +++ b/cmd/cli/dns_target_service.go @@ -0,0 +1,73 @@ +package cli + +import ( + "errors" + "regexp" + "strings" +) + +var targetServiceOrderEntry = regexp.MustCompile(`^\(([1-9][0-9]*|\*)\) (.+)$`) +var targetServiceDevice = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9]*$`) + +// networksetup uses its own service name, which can differ from the Dynamic +// Store's UserDefinedName (for example, iPhone USB versus iPhone). +// Its listing has no UUID, so require exactly one service on the proven +// device, and require that service to be enabled. +func targetNetworkServiceName(output, device string) (string, error) { + const header = "An asterisk (*) denotes that a network service is disabled." + lines := strings.Split(strings.TrimSpace(output), "\n") + if len(lines) == 0 || lines[0] != header || !targetServiceDevice.MatchString(device) { + return "", errors.New("invalid network service listing") + } + seen := make(map[string]bool) + name, pending := "", "" + disabled := false + matches := 0 + for _, line := range lines[1:] { + if strings.TrimSpace(line) == "" { + continue + } + if pending == "" { + m := targetServiceOrderEntry.FindStringSubmatch(line) + if m == nil { + return "", errors.New("invalid network service entry") + } + pending = m[2] + disabled = m[1] == "*" + if strings.HasPrefix(pending, "*") { + pending = strings.TrimPrefix(pending, "*") + disabled = true + } + if pending == "" || strings.ContainsAny(pending, "\r\n\x00") || seen[strings.ToLower(pending)] { + return "", errors.New("ambiguous network service name") + } + seen[strings.ToLower(pending)] = true + continue + } + if !strings.HasPrefix(line, "(Hardware Port: ") || !strings.HasSuffix(line, ")") { + return "", errors.New("invalid network service device") + } + at := strings.LastIndex(line, ", Device: ") + if at < 0 { + return "", errors.New("missing network service device") + } + dev := line[at+len(", Device: ") : len(line)-1] + if dev != "" && !targetServiceDevice.MatchString(dev) { + return "", errors.New("invalid network device name") + } + if dev == device { + if strings.ContainsAny(pending, "/{}") { + return "", errors.New("unsafe target service name") + } + matches++ + if !disabled { + name = pending + } + } + pending = "" + } + if pending != "" || matches != 1 || name == "" { + return "", errors.New("network service mapping is missing or ambiguous") + } + return name, nil +} diff --git a/cmd/cli/dns_target_service_test.go b/cmd/cli/dns_target_service_test.go new file mode 100644 index 00000000..83a84d95 --- /dev/null +++ b/cmd/cli/dns_target_service_test.go @@ -0,0 +1,94 @@ +package cli + +import ( + "context" + "fmt" + "strings" + "testing" +) + +func nativeTargetTestServiceOrder(name, device string) string { + return fmt.Sprintf("An asterisk (*) denotes that a network service is disabled.\n(1) %s\n(Hardware Port: Test, Device: %s)\n", name, device) +} + +// networksetup output captured on the failing USB fixture. The Dynamic Store +// name below models the name observed in ctrld's native-reader diagnostic. +const capturedUSBServiceOrder = `An asterisk (*) denotes that a network service is disabled. +(1) USB 10/100/1000 LAN +(Hardware Port: USB 10/100/1000 LAN, Device: en7) + +(2) Thunderbolt Bridge +(Hardware Port: Thunderbolt Bridge, Device: bridge0) + +(3) Wi-Fi +(Hardware Port: Wi-Fi, Device: en0) + +(4) iPhone USB +(Hardware Port: iPhone USB, Device: en8) + +(5) Tailscale +(Hardware Port: io.tailscale.ipn.macos, Device: ) +` + +func TestNativeUSBServiceName(t *testing.T) { + global := nativeTargetTestGlobal(nativeTargetTestServiceID, "en8") + outputs := []string{global, strings.ReplaceAll(nativeTargetTestState(), "en1", "en8"), " {\n UserDefinedName : iPhone\n}", capturedUSBServiceOrder, global} + var calls []string + reader := nativeTargetTestReader(t, outputs, &calls) + got, err := reader.defaultService(context.Background(), "en8") + if err != nil || got.ID != nativeTargetTestServiceID || got.Device != "en8" || got.Name != "iPhone USB" { + t.Fatalf("service=%+v err=%v", got, err) + } + dns, err := parseTargetStaticDNS("There aren't any DNS Servers set on iPhone USB.\n", got.Name) + if err != nil || len(dns) != 0 { + t.Fatalf("recognized USB DNS response rejected: %v %v", dns, err) + } +} + +func TestTargetNetworkServiceName(t *testing.T) { + base := nativeTargetTestServiceOrder("iPhone USB", "en8") + for _, tc := range []struct{ name, listing, want string }{ + {"captured USB", capturedUSBServiceOrder, "iPhone USB"}, + {"renamed", nativeTargetTestServiceOrder("Téléphone personnel", "en8"), "Téléphone personnel"}, + {"spaces preserved", nativeTargetTestServiceOrder(" iPhone USB ", "en8"), " iPhone USB "}, + {"enabled and disabled same device", base + "(*) Old USB\n(Hardware Port: Test, Device: en8)\n", ""}, + {"disabled duplicate on another device", base + "(*) iPhone USB\n(Hardware Port: Test, Device: en9)\n", ""}, + {"device boundary", nativeTargetTestServiceOrder("Other", "en80"), ""}, + {"disabled", strings.Replace(base, "(1)", "(*)", 1), ""}, + {"two enabled same device", base + "(2) Backup USB\n(Hardware Port: Test, Device: en8)\n", ""}, + {"duplicate command name", base + "(2) iPhone USB\n(Hardware Port: Test, Device: en9)\n", ""}, + {"truncated", base + "(2) Incomplete\n", ""}, + {"malformed device", strings.Replace(base, "Device: en8", "Device: en8 extra", 1), ""}, + {"unrecognized output", "Error: invalid parameter", ""}, + {"unsafe name", nativeTargetTestServiceOrder("../other", "en8"), ""}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := targetNetworkServiceName(tc.listing, "en8") + if tc.want == "" { + if err == nil { + t.Fatalf("accepted %q", got) + } + return + } + if err != nil || got != tc.want { + t.Fatalf("got=%q err=%v", got, err) + } + }) + } +} + +func TestNativeUSBServiceMappingRechecked(t *testing.T) { + outputs := nativeTargetTestOutputs() + var calls []string + reader := nativeTargetTestReader(t, outputs, &calls) + first, err := reader.defaultService(context.Background(), "en1") + if err != nil { + t.Fatal(err) + } + outputs[3] = nativeTargetTestServiceOrder("Renamed Wi-Fi", "en1") + calls = nil + second, err := reader.defaultService(context.Background(), "en1") + if err != nil || first == second || second.Name != "Renamed Wi-Fi" { + t.Fatalf("mapping change invisible: %+v %+v %v", first, second, err) + } +} diff --git a/cmd/cli/dns_target_usb_darwin_test.go b/cmd/cli/dns_target_usb_darwin_test.go new file mode 100644 index 00000000..b86a914a --- /dev/null +++ b/cmd/cli/dns_target_usb_darwin_test.go @@ -0,0 +1,89 @@ +//go:build darwin + +package cli + +import ( + "context" + "fmt" + "net" + "slices" + "strings" + "testing" +) + +func usbTargetTestReader(t *testing.T, listing string) nativeTargetReader { + t.Helper() + global := nativeTargetTestGlobal(nativeTargetTestServiceID, "en8") + outputs := []string{global, strings.ReplaceAll(nativeTargetTestState(), "en1", "en8"), " {\n UserDefinedName : iPhone\n}", listing, global} + var calls []string + return nativeTargetTestReader(t, outputs, &calls) +} + +func TestUSBServiceReaderDrivesTargetLifecycle(t *testing.T) { + for _, initial := range [][]string{nil, {"2001:db8::53"}} { + for _, port := range []int{53, 5353, 5354} { + t.Run(fmt.Sprintf("%d/%v", port, initial), func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.serviceByDev["en8"] = "iPhone USB" + h.dns["iPhone USB"] = slices.Clone(initial) + interceptDefaultRouteInterfaceFn = func() (string, error) { return "en8", nil } + interceptNativeCLATDefaultServiceFn = func(ctx context.Context, device string) (nativeTargetService, error) { + return usbTargetTestReader(t, capturedUSBServiceOrder).defaultService(ctx, device) + } + interceptNativeStaticDNSFn = func(_ context.Context, iface *net.Interface) ([]string, error) { + if iface.Name != "iPhone USB" { + t.Fatalf("invalid command service %q", iface.Name) + } + return slices.Clone(h.dns[iface.Name]), nil + } + p := newInterceptTargetProg() + p.cfg.Listener["0"].IP = "127.0.0.2" + p.cfg.Listener["0"].Port = port + p.ensureInterceptDNSTarget([]string{}) + p.ensureInterceptDNSTarget([]string{}) + if !slices.Equal(h.dns["iPhone USB"], []string{p.interceptDNSTargetValue()}) || len(h.setCalls) != 1 { + t.Fatalf("USB target=%v writes=%v", h.dns, h.setCalls) + } + if _, ok := h.dns["iPhone"]; ok { + t.Fatal("wrote display-name service") + } + p.removeInterceptDNSTarget("USB test cleanup") + if !slices.Equal(h.dns["iPhone USB"], initial) || p.interceptDNSTargetService != "" { + t.Fatal("USB DNS not restored") + } + }) + } + } +} + +func TestUSBServiceMappingChangeAndAmbiguityDoNotWrite(t *testing.T) { + for _, scenario := range []string{"renamed on recheck", "ambiguous device"} { + t.Run(scenario, func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.serviceByDev["en8"] = "iPhone USB" + original := []string{"2001:db8::53"} + h.dns["iPhone USB"] = slices.Clone(original) + interceptDefaultRouteInterfaceFn = func() (string, error) { return "en8", nil } + reads := 0 + interceptNativeCLATDefaultServiceFn = func(ctx context.Context, device string) (nativeTargetService, error) { + reads++ + listing := capturedUSBServiceOrder + if scenario == "renamed on recheck" && reads == 2 { + listing = strings.Replace(listing, "(4) iPhone USB", "(4) Renamed USB", 1) + } + if scenario == "ambiguous device" { + listing += "(6) iPhone\n(Hardware Port: iPhone USB, Device: en8)\n" + } + return usbTargetTestReader(t, listing).defaultService(ctx, device) + } + p := newInterceptTargetProg() + p.ensureInterceptDNSTarget([]string{}) + if len(h.setCalls) != 0 || len(h.resetCalls) != 0 || p.interceptDNSTargetService != "" || !slices.Equal(h.dns["iPhone USB"], original) { + t.Fatalf("uncertain mapping changed DNS: %+v", h) + } + if scenario == "renamed on recheck" && reads != 2 { + t.Fatal("did not exercise final mapping recheck") + } + }) + } +} From 44a638d15827cad28643b04cd1f16b43afc3fe0a Mon Sep 17 00:00:00 2001 From: Dev Scribe Date: Thu, 24 Sep 2026 09:06:42 -0400 Subject: [PATCH 24/29] macOS: share network service-order parsing Decode networksetup entries once for interface lookup and DNS-target selection. Preserve first-enabled lookup and fail-closed DNS uniqueness as separate policies. Treat only the (*) index as disabled, preserving asterisks and whitespace in enabled service names. Add shared-parser, caller-policy and incomplete-read regressions. --- cmd/cli/dns_target_service.go | 70 ++++++------------ cmd/cli/net_darwin.go | 61 --------------- cmd/cli/network_service_order.go | 101 +++++++++++++++++++++++++ cmd/cli/network_service_order_test.go | 102 ++++++++++++++++++++++++++ 4 files changed, 224 insertions(+), 110 deletions(-) create mode 100644 cmd/cli/network_service_order.go create mode 100644 cmd/cli/network_service_order_test.go diff --git a/cmd/cli/dns_target_service.go b/cmd/cli/dns_target_service.go index 273cc8a7..441984b9 100644 --- a/cmd/cli/dns_target_service.go +++ b/cmd/cli/dns_target_service.go @@ -2,71 +2,43 @@ package cli import ( "errors" - "regexp" "strings" ) -var targetServiceOrderEntry = regexp.MustCompile(`^\(([1-9][0-9]*|\*)\) (.+)$`) -var targetServiceDevice = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9]*$`) - // networksetup uses its own service name, which can differ from the Dynamic // Store's UserDefinedName (for example, iPhone USB versus iPhone). // Its listing has no UUID, so require exactly one service on the proven -// device, and require that service to be enabled. +// device, and require that service to be enabled. Do not use patchNetIfaceName: +// its first-enabled-match policy cannot prove a unique target for a DNS write. func targetNetworkServiceName(output, device string) (string, error) { - const header = "An asterisk (*) denotes that a network service is disabled." - lines := strings.Split(strings.TrimSpace(output), "\n") - if len(lines) == 0 || lines[0] != header || !targetServiceDevice.MatchString(device) { - return "", errors.New("invalid network service listing") + if !serviceOrderDeviceName.MatchString(device) { + return "", errors.New("invalid target device") + } + entries, err := parseNetworkServiceOrder(strings.NewReader(output)) + if err != nil { + return "", err } seen := make(map[string]bool) - name, pending := "", "" - disabled := false + name := "" matches := 0 - for _, line := range lines[1:] { - if strings.TrimSpace(line) == "" { - continue + for _, entry := range entries { + key := strings.ToLower(entry.Name) + if seen[key] { + return "", errors.New("ambiguous network service name") } - if pending == "" { - m := targetServiceOrderEntry.FindStringSubmatch(line) - if m == nil { - return "", errors.New("invalid network service entry") - } - pending = m[2] - disabled = m[1] == "*" - if strings.HasPrefix(pending, "*") { - pending = strings.TrimPrefix(pending, "*") - disabled = true - } - if pending == "" || strings.ContainsAny(pending, "\r\n\x00") || seen[strings.ToLower(pending)] { - return "", errors.New("ambiguous network service name") - } - seen[strings.ToLower(pending)] = true + seen[key] = true + if entry.Device != device { continue } - if !strings.HasPrefix(line, "(Hardware Port: ") || !strings.HasSuffix(line, ")") { - return "", errors.New("invalid network service device") - } - at := strings.LastIndex(line, ", Device: ") - if at < 0 { - return "", errors.New("missing network service device") - } - dev := line[at+len(", Device: ") : len(line)-1] - if dev != "" && !targetServiceDevice.MatchString(dev) { - return "", errors.New("invalid network device name") + if strings.ContainsAny(entry.Name, "/{}") { + return "", errors.New("unsafe target service name") } - if dev == device { - if strings.ContainsAny(pending, "/{}") { - return "", errors.New("unsafe target service name") - } - matches++ - if !disabled { - name = pending - } + matches++ + if !entry.Disabled { + name = entry.Name } - pending = "" } - if pending != "" || matches != 1 || name == "" { + if matches != 1 || name == "" { return "", errors.New("network service mapping is missing or ambiguous") } return name, nil diff --git a/cmd/cli/net_darwin.go b/cmd/cli/net_darwin.go index 4856150c..35b0f44a 100644 --- a/cmd/cli/net_darwin.go +++ b/cmd/cli/net_darwin.go @@ -1,7 +1,6 @@ package cli import ( - "bufio" "bytes" "context" "errors" @@ -32,9 +31,6 @@ const networksetupTimeout = 5 * time.Second // below it, so more than this is a broken command. const maxNetworksetupOutput = 1 << 20 -// maxNetworksetupLine bounds one line, so one token fills no memory. -const maxNetworksetupLine = 64 * 1024 - var errNetworksetupOutputTooLarge = errors.New("networksetup output above the bound") var ( @@ -65,63 +61,6 @@ func networkServiceName(ifaceName string, r io.Reader) string { return serviceNamesByDevice(r)[ifaceName] } -// serviceNamesByDevice maps each device of "networksetup -// -listnetworkserviceorder" to the name of the service that owns it. -func serviceNamesByDevice(r io.Reader) map[string]string { - names := make(map[string]string) - scanner := scanNetworksetup(r) - prevLine := "" - for scanner.Scan() { - line := scanner.Text() - if strings.Contains(line, "*") { - // A disabled service names no device. Without the reset, the device - // line that follows takes the name of the service before it. - prevLine = "" - continue - } - device := serviceOrderDevice(line) - if device == "" { - prevLine = line - continue - } - if _, seen := names[device]; !seen { - names[device] = serviceOrderName(prevLine) - } - prevLine = line - } - if scanner.Err() != nil { - // A truncated read names fewer services than the host has. - return nil - } - return names -} - -// scanNetworksetup reads the output line by line under a bounded buffer. -func scanNetworksetup(r io.Reader) *bufio.Scanner { - scanner := bufio.NewScanner(r) - scanner.Buffer(nil, maxNetworksetupLine) - return scanner -} - -// serviceOrderDevice reads the device out of a line like -// "(Hardware Port: Wi-Fi, Device: en0)". -func serviceOrderDevice(line string) string { - _, after, found := strings.Cut(line, "Device:") - if !found { - return "" - } - return strings.TrimSpace(strings.TrimSuffix(strings.TrimSpace(after), ")")) -} - -// serviceOrderName reads the service name out of a line like "(1) Wi-Fi". -func serviceOrderName(line string) string { - parts := strings.SplitN(line, " ", 2) - if len(parts) != 2 { - return "" - } - return strings.TrimSpace(parts[1]) -} - // parseHardwarePorts maps each device of "networksetup -listallhardwareports" // to its hardware port. Each block names the port before the device. func parseHardwarePorts(r io.Reader) map[string]hardwarePortInfo { diff --git a/cmd/cli/network_service_order.go b/cmd/cli/network_service_order.go new file mode 100644 index 00000000..b025fee5 --- /dev/null +++ b/cmd/cli/network_service_order.go @@ -0,0 +1,101 @@ +package cli + +import ( + "bufio" + "errors" + "io" + "regexp" + "strings" +) + +// maxNetworksetupLine bounds one line, so one token fills no memory. +const maxNetworksetupLine = 64 * 1024 + +var serviceOrderEntry = regexp.MustCompile(`^\(([1-9][0-9]*|\*)\) (.+)$`) +var serviceOrderDeviceName = regexp.MustCompile(`^[A-Za-z][A-Za-z0-9]*$`) + +type networkServiceEntry struct { + Name, Device string + Disabled bool +} + +// scanNetworksetup reads the output line by line under a bounded buffer. +func scanNetworksetup(r io.Reader) *bufio.Scanner { + scanner := bufio.NewScanner(r) + scanner.Buffer(nil, maxNetworksetupLine) + return scanner +} + +// parseNetworkServiceOrder decodes networksetup -listnetworkserviceorder. +// Selection policy belongs to the caller: retain disabled and duplicate-device +// entries so a DNS writer can reject mappings a first-match reader accepts. +func parseNetworkServiceOrder(r io.Reader) ([]networkServiceEntry, error) { + const header = "An asterisk (*) denotes that a network service is disabled." + scanner := scanNetworksetup(r) + var entries []networkServiceEntry + var pending *networkServiceEntry + headerSeen := false + for scanner.Scan() { + line := scanner.Text() + if strings.TrimSpace(line) == "" { + continue + } + if !headerSeen { + if strings.TrimSpace(line) != header { + return nil, errors.New("invalid network service listing") + } + headerSeen = true + continue + } + if pending == nil { + m := serviceOrderEntry.FindStringSubmatch(line) + if m == nil || strings.ContainsAny(m[2], "\r\n\x00") { + return nil, errors.New("invalid network service entry") + } + // Only the index marks a disabled service. An enabled name may + // itself start with an asterisk; preserve it and its whitespace. + pending = &networkServiceEntry{Name: m[2], Disabled: m[1] == "*"} + continue + } + if !strings.HasPrefix(line, "(Hardware Port: ") || !strings.HasSuffix(line, ")") { + return nil, errors.New("invalid network service device") + } + at := strings.LastIndex(line, ", Device: ") + if at < 0 { + return nil, errors.New("missing network service device") + } + device := line[at+len(", Device: ") : len(line)-1] + if device != "" && !serviceOrderDeviceName.MatchString(device) { + return nil, errors.New("invalid network device name") + } + pending.Device = device + entries = append(entries, *pending) + pending = nil + } + if err := scanner.Err(); err != nil { + return nil, err + } + if !headerSeen || pending != nil { + return nil, errors.New("incomplete network service listing") + } + return entries, nil +} + +// serviceNamesByDevice keeps the first enabled service on each device for +// interface-name lookup and hardware-port metadata, not DNS-target admission. +func serviceNamesByDevice(r io.Reader) map[string]string { + entries, err := parseNetworkServiceOrder(r) + if err != nil { + return nil + } + names := make(map[string]string) + for _, entry := range entries { + if entry.Disabled || entry.Device == "" { + continue + } + if _, seen := names[entry.Device]; !seen { + names[entry.Device] = entry.Name + } + } + return names +} diff --git a/cmd/cli/network_service_order_test.go b/cmd/cli/network_service_order_test.go new file mode 100644 index 00000000..f82fed82 --- /dev/null +++ b/cmd/cli/network_service_order_test.go @@ -0,0 +1,102 @@ +package cli + +import ( + "errors" + "io" + "reflect" + "strings" + "testing" +) + +func TestNetworkServiceOrderEntries(t *testing.T) { + const listing = `An asterisk (*) denotes that a network service is disabled. +(1) *Téléphone personnel +(Hardware Port: USB, Device: en8) +(*) Disabled USB +(Hardware Port: USB, Device: en8) +(2) Wi-Fi +(Hardware Port: Wi-Fi, Device: en0) +(3) Tunnel +(Hardware Port: VPN, Device: ) +` + want := []networkServiceEntry{ + {Name: "*Téléphone personnel", Device: "en8"}, + {Name: "Disabled USB", Device: "en8", Disabled: true}, + {Name: " Wi-Fi", Device: "en0"}, + {Name: "Tunnel"}, + } + got, err := parseNetworkServiceOrder(strings.NewReader(listing)) + if err != nil || !reflect.DeepEqual(got, want) { + t.Fatalf("entries=%+v err=%v", got, err) + } +} + +func TestNetworkServiceOrderSelectionPolicies(t *testing.T) { + base := nativeTargetTestServiceOrder("*iPhone USB", "en8") + for _, tc := range []struct { + name, listing, lookup, target string + }{ + {"enabled asterisk name", base, "*iPhone USB", "*iPhone USB"}, + {"embedded asterisk", nativeTargetTestServiceOrder("My * USB", "en8"), "My * USB", "My * USB"}, + {"name whitespace", nativeTargetTestServiceOrder(" USB ", "en8"), " USB ", " USB "}, + {"disabled", strings.Replace(base, "(1)", "(*)", 1), "", ""}, + {"disabled neighbor", base + "(*) Ethernet\n(Hardware Port: Ethernet, Device: en9)\n", "*iPhone USB", "*iPhone USB"}, + {"enabled then disabled same device", base + "(*) Old USB\n(Hardware Port: USB, Device: en8)\n", "*iPhone USB", ""}, + {"disabled then enabled same device", strings.Replace(base, "(1)", "(*)", 1) + "(2) New USB\n(Hardware Port: USB, Device: en8)\n", "New USB", ""}, + {"two enabled same device", base + "(2) Other USB\n(Hardware Port: USB, Device: en8)\n", "*iPhone USB", ""}, + {"duplicate name on another device", base + "(2) *iPhone USB\n(Hardware Port: USB, Device: en9)\n", "*iPhone USB", ""}, + {"case-insensitive duplicate", base + "(*) *IPHONE USB\n(Hardware Port: USB, Device: en9)\n", "*iPhone USB", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + names := serviceNamesByDevice(strings.NewReader(tc.listing)) + if names["en8"] != tc.lookup { + t.Fatalf("lookup=%v, want %q", names, tc.lookup) + } + if tc.name == "disabled neighbor" { + if _, exists := names["en9"]; exists { + t.Fatal("disabled neighbor inherited an enabled service name") + } + } + got, err := targetNetworkServiceName(tc.listing, "en8") + if got != tc.target || (err != nil) != (tc.target == "") { + t.Fatalf("target=%q err=%v, want %q", got, err, tc.target) + } + }) + } +} + +func TestNetworkServiceOrderRejectsIncompleteReads(t *testing.T) { + base := nativeTargetTestServiceOrder("USB", "en8") + for _, listing := range []string{ + "", "unexpected output", base + "(2) Incomplete\n", + base + "(2) \n(Hardware Port: USB, Device: en9)\n", + base + "(2) Broken\nmissing device line\n", + base + strings.Repeat("x", maxNetworksetupLine+1), + strings.Replace(base, "Device: en8", "Device: en8 extra", 1), + strings.Replace(base, "(1) USB", "(0) USB", 1), + strings.Replace(base, "(1) USB", "(1) USB\x00", 1), + } { + entries, err := parseNetworkServiceOrder(strings.NewReader(listing)) + if err == nil || entries != nil { + t.Fatalf("invalid listing returned entries=%+v err=%v", entries, err) + } + if got := serviceNamesByDevice(strings.NewReader(listing)); got != nil { + t.Fatalf("invalid listing returned partial names: %v", got) + } + if got, err := targetNetworkServiceName(listing, "en8"); err == nil || got != "" { + t.Fatalf("invalid listing admitted target=%q err=%v", got, err) + } + } + readErr := errors.New("interrupted service-order read") + reader := func() io.Reader { return io.MultiReader(strings.NewReader(base), serviceOrderErrorReader{readErr}) } + if entries, err := parseNetworkServiceOrder(reader()); !errors.Is(err, readErr) || entries != nil { + t.Fatalf("partial read returned entries=%+v err=%v", entries, err) + } + if names := serviceNamesByDevice(reader()); names != nil { + t.Fatalf("failed read returned partial names: %v", names) + } +} + +type serviceOrderErrorReader struct{ err error } + +func (r serviceOrderErrorReader) Read([]byte) (int, error) { return 0, r.err } From 462f912495c1593f20ed3e946025600272b22923 Mon Sep 17 00:00:00 2001 From: Dev Scribe Date: Thu, 24 Sep 2026 13:13:31 +0000 Subject: [PATCH 25/29] Harden DNS64, VPN routing and DNS recovery ownership --- cmd/cli/cli.go | 11 +- cmd/cli/dns64.go | 29 +++- cmd/cli/dns64_dnssec_test.go | 139 +++++++++++++++ cmd/cli/dns_intercept_darwin.go | 30 +++- cmd/cli/dns_proxy.go | 7 +- cmd/cli/dns_target_darwin.go | 100 +++++++++-- cmd/cli/dns_target_darwin_test.go | 3 + cmd/cli/dns_target_diagnostic_darwin_test.go | 13 +- cmd/cli/dns_target_ownership_darwin_test.go | 168 +++++++++++++++++++ cmd/cli/dns_target_stub.go | 2 + cmd/cli/listener_5353_darwin_test.go | 67 ++++++++ cmd/cli/listener_5353_linux_test.go | 32 ++++ cmd/cli/listener_5353_target_darwin_test.go | 41 +++++ cmd/cli/listener_5353_test.go | 34 ++++ cmd/cli/prog.go | 27 +++ cmd/cli/vpn_dns.go | 33 ++-- cmd/cli/vpn_dns_convergence_test.go | 133 +++++++++++++++ docs/619-target-ownership-limits.md | 17 ++ nameservers_darwin.go | 36 +--- nameservers_scutil.go | 45 +++++ nameservers_scutil_darwin_test.go | 51 ++++++ nameservers_scutil_test.go | 80 +++++++++ resolver.go | 9 +- 23 files changed, 1021 insertions(+), 86 deletions(-) create mode 100644 cmd/cli/dns64_dnssec_test.go create mode 100644 cmd/cli/dns_target_ownership_darwin_test.go create mode 100644 cmd/cli/listener_5353_darwin_test.go create mode 100644 cmd/cli/listener_5353_linux_test.go create mode 100644 cmd/cli/listener_5353_target_darwin_test.go create mode 100644 cmd/cli/listener_5353_test.go create mode 100644 cmd/cli/vpn_dns_convergence_test.go create mode 100644 docs/619-target-ownership-limits.md create mode 100644 nameservers_scutil.go create mode 100644 nameservers_scutil_darwin_test.go create mode 100644 nameservers_scutil_test.go diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index b6cf64f1..1f5374fd 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -1482,12 +1482,19 @@ func uninstall(p *prog, s service.Service) { } } +var restoreSavedStaticDNSInterfacesFn = withEachPhysicalInterfaces +var restoreSavedStaticDNSRestoreFn = restoreDNS + // restoreSavedStaticDNS restores DNS from saved static config files on physical interfaces. func restoreSavedStaticDNS(excludeIfaceName string, removeSaved bool) { - withEachPhysicalInterfaces(excludeIfaceName, "restore static DNS", func(i *net.Interface) error { + restoreSavedStaticDNSInterfacesFn(excludeIfaceName, "restore static DNS", func(i *net.Interface) error { + if !interceptTargetAllowsStaticRestore(i.Name) { + mainLog.Load().Debug().Msgf("Saved static DNS restore skipped on interface %s: intercept target cleanup is pending or ownership is unreadable", i.Name) + return nil + } file := savedStaticDnsSettingsFilePath(i) if _, err := os.Stat(file); err == nil { - if err := restoreDNS(i); err != nil { + if err := restoreSavedStaticDNSRestoreFn(i); err != nil { mainLog.Load().Error().Err(err).Msgf("Could not restore static DNS on interface %s", i.Name) } else { mainLog.Load().Debug().Msgf("Restored static DNS on interface %s successfully", i.Name) diff --git a/cmd/cli/dns64.go b/cmd/cli/dns64.go index b8d39899..88545753 100644 --- a/cmd/cli/dns64.go +++ b/cmd/cli/dns64.go @@ -175,6 +175,9 @@ func synthesizeAAAAFromA(req *dns.Msg, aAnswer *dns.Msg, prefix netip.Prefix) *d out.SetReply(req) out.Rcode = aAnswer.Rcode out.Compress = true + // An authenticated A RRset alone cannot authenticate synthesized AAAA. + // maybeDNS64 also checks the original AAAA denial before setting AD. + out.AuthenticatedData = false answers := make([]dns.RR, 0, len(aAnswer.Answer)) pb := prefix.Addr().As16() for _, rr := range aAnswer.Answer { @@ -228,10 +231,7 @@ func answerHasAAAA(answer *dns.Msg) bool { // rcodes are never synthesized (RFC 6147 §5.1.2: the name genuinely does // not exist or the query failed). func dns64Eligible(req, answer *dns.Msg) bool { - if req == nil || answer == nil || len(req.Question) == 0 || req.CheckingDisabled { - return false - } - if req.Question[0].Qtype != dns.TypeAAAA { + if !dns64RequestAllowed(req) || answer == nil { return false } if answer.Rcode != dns.RcodeSuccess { @@ -240,6 +240,13 @@ func dns64Eligible(req, answer *dns.Msg) bool { return !answerHasAAAA(answer) } +// Apply the same client gate to cold synthesis and fresh/stale variants. +// In particular, RFC 6147 section 5.5 forbids synthesis for CD+DO clients. +// Retain the existing conservative bypass for all CD requests. +func dns64RequestAllowed(req *dns.Msg) bool { + return req != nil && len(req.Question) > 0 && req.Question[0].Qtype == dns.TypeAAAA && !req.CheckingDisabled +} + // dns64Active reports whether synthesis should currently run, re-evaluating // network class and (if needed) kicking off background prefix discovery at // most every dns64RecheckInterval. @@ -307,7 +314,13 @@ func dns64CacheVariant(prefix netip.Prefix) string { } func dns64CacheKey(msg *dns.Msg, upstream string, prefix netip.Prefix) dnscache.Key { - return dnscache.NewVariantKey(msg, upstream, dns64CacheVariant(prefix)) + variant := dns64CacheVariant(prefix) + // DO changes the authentication and DNSSEC data in a synthesized reply. + // Do not reuse a non-DO result for a DO client (or the reverse). + if opt := msg.IsEdns0(); opt != nil && opt.Do() { + variant += ":do" + } + return dnscache.NewVariantKey(msg, upstream, variant) } func (p *prog) resetDNS64State() { @@ -476,6 +489,12 @@ func (p *prog) maybeDNS64(ctx context.Context, req *dns.Msg, answer *dns.Msg, re // DNS64 variant to avoid repeating both upstream lookups. return answer, prefix } + // RFC 6147 section 5.5: AD may describe synthesis only when both the + // AAAA denial and the A response were authenticated by our upstream. + // Without DO, synthesized responses must not assert AD. + if opt := req.IsEdns0(); opt != nil && opt.Do() { + synth.AuthenticatedData = answer.AuthenticatedData && aAnswer.AuthenticatedData + } ctrld.Log(ctx, mainLog.Load().Debug(), "dns64: synthesized AAAA from A records via NAT64 prefix %s", prefix) return synth, prefix } diff --git a/cmd/cli/dns64_dnssec_test.go b/cmd/cli/dns64_dnssec_test.go new file mode 100644 index 00000000..6c862db7 --- /dev/null +++ b/cmd/cli/dns64_dnssec_test.go @@ -0,0 +1,139 @@ +package cli + +import ( + "context" + "fmt" + "net" + "testing" + "time" + + "github.com/miekg/dns" + ctrld "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/dnscache" +) + +func TestDNS64DNSSECCacheAndSynthesis(t *testing.T) { + captureDebugMainLog(t) + for _, do := range []bool{false, true} { + for _, denialAD := range []bool{false, true} { + for _, aAD := range []bool{false, true} { + t.Run(fmt.Sprintf("DO=%v/denialAD=%v/A_AD=%v", do, denialAD, aAD), func(t *testing.T) { + req := mkAAAAReq("dnssec.example.") + req.SetEdns0(1232, do) + denial := new(dns.Msg) + denial.SetReply(req) + denial.AuthenticatedData = denialAD + cache, err := dnscache.NewLRUCache(10) + if err != nil { + t.Fatal(err) + } + p := &prog{cache: cache} + p.dns64.active = true + p.dns64.prefix = dns64WellKnownPrefix + p.dns64.checkedAt = time.Now() + calls := 0 + resolveA := func(q *dns.Msg) *dns.Msg { + calls++ + if q.Question[0].Qtype != dns.TypeA { + t.Fatal("not an A lookup") + } + a := new(dns.Msg) + a.SetReply(q) + a.AuthenticatedData = aAD + a.Answer = []dns.RR{&dns.A{Hdr: dns.RR_Header{Name: q.Question[0].Name, Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}, A: net.ParseIP("198.51.100.7")}} + return a + } + out, prefix := p.maybeDNS64(context.Background(), req, denial, resolveA) + wantAD := do && denialAD && aAD + if calls != 1 || !answerHasAAAA(out) || !prefix.IsValid() || out.AuthenticatedData != wantAD { + t.Fatalf("cold: calls=%d AAAA=%v prefix=%v AD=%v wantAD=%v", calls, answerHasAAAA(out), prefix, out.AuthenticatedData, wantAD) + } + now := time.Now() + for _, stale := range []bool{false, true} { + expiry := now.Add(time.Minute) + if stale { + expiry = now.Add(-time.Minute) + } + cache.Add(dns64CacheKey(req, "upstream.0", prefix), dnscache.NewValue(out, expiry)) + answer, old, hit, variant, _ := p.cachedResponse(req, "upstream.0", prefix, true, now) + got := answer + if stale { + got = old + } + if got == nil || !answerHasAAAA(got) || got.AuthenticatedData != wantAD || hit == stale || variant == stale { + t.Fatalf("normal cache stale=%v: answer=%v old=%v hit=%v variant=%v", stale, answer, old, hit, variant) + } + cd := req.Copy() + cd.CheckingDisabled = true + answer, old, hit, variant, _ = p.cachedResponse(cd, "upstream.0", prefix, true, now) + if answerHasAAAA(answer) || answerHasAAAA(old) || hit || variant { + t.Errorf("CD cache stale=%v leaked synthesis: answer=%v old=%v hit=%v variant=%v", stale, answer, old, hit, variant) + } + cold, coldPrefix := p.maybeDNS64(context.Background(), cd, denial, resolveA) + if cold != denial || coldPrefix.IsValid() || calls != 1 { + t.Fatal("CD cold path performed synthesis") + } + } + if denial.AuthenticatedData != denialAD { + t.Fatal("mutated denial") + } + }) + } + } + } +} + +func TestDNS64DNSSECOrdinaryCacheFallback(t *testing.T) { + req := mkAAAAReq("dnssec.example.") + req.CheckingDisabled = true + req.SetEdns0(1232, true) + ordinary := new(dns.Msg) + ordinary.SetReply(req) + cache, err := dnscache.NewLRUCache(10) + if err != nil { + t.Fatal(err) + } + p := &prog{cache: cache} + now := time.Now() + cache.Add(dnscache.NewKey(req, "upstream.0"), dnscache.NewValue(ordinary, now.Add(time.Minute))) + answer, _, hit, variant, bypass := p.cachedResponse(req, "upstream.0", dns64WellKnownPrefix, true, now) + if !hit || variant || bypass || answerHasAAAA(answer) { + t.Fatalf("ordinary cache: hit=%v variant=%v bypass=%v answer=%v", hit, variant, bypass, answer) + } +} + +func TestRecoveryOSOnlyCandidatePool(t *testing.T) { + for _, reason := range []RecoveryReason{RecoveryReasonNetworkChange, RecoveryReasonRegularFailure} { + p := &prog{cfg: &ctrld.Config{Upstream: map[string]*ctrld.UpstreamConfig{"0": {Type: ctrld.ResolverTypeOS}, "nil": nil}}} + pool := p.buildRecoveryUpstreams(reason) + if len(pool) != 1 || pool[upstreamOS] != osUpstreamConfig { + t.Errorf("reason=%v: OS-only recovery candidates=%v", reason, pool) + } + } +} + +func TestDNS64DNSSECCacheDOIsolation(t *testing.T) { + for _, do := range []bool{false, true} { + req := mkAAAAReq("dnssec.example.") + req.SetEdns0(1232, do) + response := new(dns.Msg) + response.SetReply(req) + response.AuthenticatedData = do + response.Answer = []dns.RR{&dns.AAAA{Hdr: dns.RR_Header{Name: req.Question[0].Name, Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}, AAAA: net.ParseIP("64:ff9b::c633:6407")}} + cache, err := dnscache.NewLRUCache(10) + if err != nil { + t.Fatal(err) + } + p := &prog{cache: cache} + now := time.Now() + other := req.Copy() + other.IsEdns0().SetDo(!do) + for _, expiry := range []time.Time{now.Add(time.Minute), now.Add(-time.Minute)} { + cache.Add(dns64CacheKey(req, "upstream.0", dns64WellKnownPrefix), dnscache.NewValue(response, expiry)) + answer, stale, hit, variant, _ := p.cachedResponse(other, "upstream.0", dns64WellKnownPrefix, true, now) + if answer != nil || stale != nil || hit || variant { + t.Fatalf("DO=%v result reused for DO=%v: answer=%v stale=%v", do, !do, answer, stale) + } + } + } +} diff --git a/cmd/cli/dns_intercept_darwin.go b/cmd/cli/dns_intercept_darwin.go index 68a7dbbe..b4ddc8be 100644 --- a/cmd/cli/dns_intercept_darwin.go +++ b/cmd/cli/dns_intercept_darwin.go @@ -696,6 +696,21 @@ func flushPFStates() { } } +// pfNameserverAddress keeps socket zones out of group-scoped PF addresses. +// ctrld already has a blanket group exemption in the same anchor. +func pfNameserverAddress(server string) (string, string, bool) { + host, _, err := net.SplitHostPort(server) + if err != nil { + host = server + } + addr, err := netip.ParseAddr(host) + if err != nil { + return "", "", false + } + host = addr.WithZone("").Unmap().String() + return host, pfAddressFamily(host), true +} + func pfAddressFamily(ip string) string { if addr := net.ParseIP(ip); addr != nil && addr.To4() == nil { return "inet6" @@ -886,11 +901,10 @@ func (p *prog) buildPFAnchorRulesForTunnels(vpnExemptions []vpnDNSExemption, tun rules.WriteString("# Scoped to group " + pfGroupName + " so only ctrld's own queries are exempted,\n") rules.WriteString("# preventing other processes from bypassing the redirect by querying these IPs.\n") for _, ns := range osNS { - host, _, _ := net.SplitHostPort(ns) - if host == "" { - host = ns + host, af, ok := pfNameserverAddress(ns) + if !ok { + continue } - af := pfAddressFamily(host) rules.WriteString(fmt.Sprintf("pass out quick on ! lo0 %s proto { udp, tcp } from any to %s port 53 group %s\n", af, host, pfGroupName)) } rules.WriteString("\n") @@ -932,10 +946,10 @@ func (p *prog) buildPFAnchorRulesForTunnels(vpnExemptions []vpnDNSExemption, tun rules.WriteString("# Exempt VPN DNS servers: ctrld's own queries (group-scoped).\n") seen := make(map[string]bool) for _, ex := range vpnExemptions { - if !seen[ex.Server] { - seen[ex.Server] = true - af := pfAddressFamily(ex.Server) - rules.WriteString(fmt.Sprintf("pass out quick on ! lo0 %s proto { udp, tcp } from any to %s port 53 group %s\n", af, ex.Server, pfGroupName)) + host, af, ok := pfNameserverAddress(ex.Server) + if ok && !seen[host] { + seen[host] = true + rules.WriteString(fmt.Sprintf("pass out quick on ! lo0 %s proto { udp, tcp } from any to %s port 53 group %s\n", af, host, pfGroupName)) } } rules.WriteString("\n") diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index d2099ee0..87e0b1b8 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -124,7 +124,7 @@ func (p *prog) addCachedResponse(key dnscache.Key, answer *dns.Msg) { } func (p *prog) cachedResponse(req *dns.Msg, upstream string, dns64Prefix netip.Prefix, dns64Active bool, now time.Time) (answer, stale *dns.Msg, hit, dns64Hit, dns64Bypass bool) { - if dns64Active { + if dns64Active && dns64RequestAllowed(req) { if cachedValue := p.cache.Get(dns64CacheKey(req, upstream, dns64Prefix)); cachedValue != nil { answer = cachedValue.Msg.Copy() ctrld.SetCacheReply(answer, req, answer.Rcode) @@ -2719,6 +2719,11 @@ func (p *prog) buildRecoveryUpstreams(reason RecoveryReason) map[string]*ctrld.U upstreams[upstreamPrefix+k] = uc } } + // An OS-only configuration still needs a recovery worker. Do not add + // OS to mixed pools: their existing recovery boundaries stay intact. + if len(upstreams) == 0 { + upstreams[upstreamOS] = osUpstreamConfig + } } return upstreams } diff --git a/cmd/cli/dns_target_darwin.go b/cmd/cli/dns_target_darwin.go index 4f2fb323..95cb79b2 100644 --- a/cmd/cli/dns_target_darwin.go +++ b/cmd/cli/dns_target_darwin.go @@ -8,6 +8,7 @@ import ( "errors" "net" "os" + "path/filepath" "slices" "tailscale.com/net/netmon" @@ -30,13 +31,38 @@ var ( interceptSaveStaticDNSSnapshotFn = func(iface *net.Interface, dns []string, owned string) error { return saveTargetStaticDNSSnapshot(savedStaticDnsSettingsFilePath(iface), dns, owned) } - interceptSaveCurrentStaticDNSFn = saveCurrentStaticDNS + interceptSaveCurrentStaticDNSFn = saveCurrentStaticDNS + interceptRemoveSavedDNSFn = func(iface *net.Interface) error { + err := os.Remove(savedStaticDnsSettingsFilePath(iface)) + if errors.Is(err, os.ErrNotExist) { + return nil + } + return err + } interceptSetDNSFn = setDNS interceptSavedStaticNameserversFn = savedStaticNameservers interceptResetDNSIgnoreUnusableIfaceFn = resetDnsIgnoreUnusableInterface interceptDHCPNameserversForInterfaceFn = readTargetDHCPNameservers ) +// A retained target record means cleanup was not completed. The broad saved +// static sweep must not bypass the target's ownership check or retry failures +// through an unguarded restore. Unknown state is conservatively left alone. +func interceptTargetAllowsStaticRestore(service string) bool { + data, err := os.ReadFile(interceptDNSTargetStatePathFn()) + if errors.Is(err, os.ErrNotExist) { + return true + } + if err != nil { + return false + } + var st interceptDNSTargetState + if json.Unmarshal(data, &st) != nil || st.Service == "" { + return false + } + return st.Service != service +} + type interceptDNSTargetState struct { Service string `json:"service"` Value string `json:"value"` @@ -64,23 +90,34 @@ func (p *prog) loadInterceptDNSTargetStateLocked() { // persistInterceptDNSTargetStateLocked writes (or clears) the state file to // match in-memory tracking. Callers must hold interceptDNSTargetMu. -func (p *prog) persistInterceptDNSTargetStateLocked() { +func (p *prog) persistInterceptDNSTargetStateLocked() error { file := interceptDNSTargetStatePathFn() if p.interceptDNSTargetService == "" { - _ = os.Remove(file) - return + if err := os.Remove(file); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + return nil } data, err := json.Marshal(interceptDNSTargetState{Service: p.interceptDNSTargetService, Value: p.interceptDNSTargetSetValue}) if err != nil { - return + return err } - if err := os.WriteFile(file, data, 0600); err != nil { - mainLog.Load().Debug().Err(err).Msg("intercept DNS target: could not persist state file") + // Replace atomically; a failed write must not truncate prior ownership. + tmp, err := os.CreateTemp(filepath.Dir(file), ".intercept-target-*") + if err != nil { + return err } + defer os.Remove(tmp.Name()) + _, writeErr := tmp.Write(data) + err = errors.Join(writeErr, tmp.Close()) + if err != nil { + return err + } + return os.Rename(tmp.Name(), file) } -// ensureInterceptDNSTarget guarantees macOS always has an emittable DNS -// target while DNS intercept mode is active. +// ensureInterceptDNSTarget provides an emittable DNS target when ownership +// and discovery permit it; externally changed DNS is not permission to reinstall. // // Intercept mode deliberately never manages interface DNS: pf redirects DNS // packets in flight. But pf can only redirect packets macOS actually sends, @@ -192,6 +229,14 @@ func (p *prog) ensureInterceptDNSTarget(systemDiscovery []string) { return } } + // Tracking is not proof that the target is still installed. An external + // edit (including clearing DNS) is not permission to reinstall it. Retain + // the cleanup record until shutdown, preventing periodic reconciliation + // from reinterpreting the same edit as a new DNS-less admission. + if p.interceptDNSTargetService == iface.Name && !isInterceptDNSTargetOnly(snapshot, p.interceptDNSTargetSetValue) && !hasIPv4DNS(filterOwnTarget(snapshot, p.interceptDNSTargetSetValue)) { + diagnostic.failed(decision, "owned_target_changed", nil) + return + } // Never count ctrld's own previously-set entry as network-provided DNS, // or the next recovery on the same DNS-less network would remove it and // the one after re-add it. @@ -256,12 +301,21 @@ func (p *prog) ensureInterceptDNSTarget(systemDiscovery []string) { mainLog.Load().Debug().Err(err).Msgf("intercept DNS target: could not save static DNS for %q", iface.Name) } } + p.setInterceptDNSTargetLocked(iface.Name, target) + if err := p.persistInterceptDNSTargetStateLocked(); err != nil { + p.setInterceptDNSTargetLocked("", "") + resolvedReason = "" + diagnostic.failed(decision, "ownership_persistence", err) + mainLog.Load().Warn().Err(err).Msg("intercept DNS target: not changing DNS without persisted ownership") + return + } if err := interceptSetDNSFn(iface, []string{target}); err != nil { + // A command error can follow a partial OS mutation. Keep the durable + // cleanup record; the next observation decides what still belongs to us. + resolvedReason = "" mainLog.Load().Warn().Err(err).Msgf("intercept DNS target: could not set %s on %q", target, iface.Name) return } - p.setInterceptDNSTargetLocked(iface.Name, target) - p.persistInterceptDNSTargetStateLocked() journal(mainLog.Load().Warn()).Str("service", iface.Name).Str("target", target). Bool("native_clat_fallback", nativeFallback). Str("reason", "dns_less_network"). @@ -295,7 +349,15 @@ func (p *prog) removeInterceptDNSTargetLocked(reason string) { return } if !isInterceptDNSTargetOnly(cur, val) { - p.clearInterceptDNSTargetStateLocked() + // The generic stop/uninstall sweep must not replay a stale snapshot + // over the external edit. On failure keep ownership so its guard skips. + if err := interceptRemoveSavedDNSFn(iface); err != nil { + mainLog.Load().Warn().Err(err).Msg("intercept DNS target: could not discard stale backup; retaining cleanup state") + return + } + if !p.clearInterceptDNSTargetStateLocked() { + return + } // ctrld owns the DNS of the service no longer, and a later outage // report needs the moment that ownership ended. journal(mainLog.Load().Info()).Str("service", svc).Str("target", val). @@ -312,7 +374,9 @@ func (p *prog) removeInterceptDNSTargetLocked(reason string) { mainLog.Load().Warn().Err(err).Msgf("intercept DNS target: could not reset DNS on %q; retaining cleanup state", svc) return } - p.clearInterceptDNSTargetStateLocked() + if !p.clearInterceptDNSTargetStateLocked() { + return + } journal(mainLog.Load().Info()).Str("service", svc).Str("target", val).Str("reason", reason). Msgf("intercept DNS target: removed %s from %q (%s)", val, svc, reason) } @@ -321,9 +385,15 @@ func (p *prog) interceptDNSTargetOwnershipLocked() dnsTargetOwnership { return dnsTargetOwnership{p.interceptDNSTargetService, p.interceptDNSTargetSetValue} } -func (p *prog) clearInterceptDNSTargetStateLocked() { +func (p *prog) clearInterceptDNSTargetStateLocked() bool { + service, value := p.interceptDNSTargetService, p.interceptDNSTargetSetValue p.setInterceptDNSTargetLocked("", "") - p.persistInterceptDNSTargetStateLocked() + if err := p.persistInterceptDNSTargetStateLocked(); err != nil { + p.setInterceptDNSTargetLocked(service, value) + mainLog.Load().Warn().Err(err).Msg("intercept DNS target: could not clear ownership; retaining cleanup state") + return false + } + return true } // setInterceptDNSTargetLocked stores the service that ctrld owns and the value diff --git a/cmd/cli/dns_target_darwin_test.go b/cmd/cli/dns_target_darwin_test.go index b22d9dd3..c7e882c0 100644 --- a/cmd/cli/dns_target_darwin_test.go +++ b/cmd/cli/dns_target_darwin_test.go @@ -41,6 +41,9 @@ func newInterceptTargetHarness(t *testing.T) *interceptTargetHarness { statePath: filepath.Join(t.TempDir(), interceptDNSTargetStateFile), } + origHome := homedir + homedir = t.TempDir() + t.Cleanup(func() { homedir = origHome }) origPath := interceptDNSTargetStatePathFn origRoute := interceptDefaultRouteInterfaceFn origIface := interceptInterfaceByNameFn diff --git a/cmd/cli/dns_target_diagnostic_darwin_test.go b/cmd/cli/dns_target_diagnostic_darwin_test.go index c83f2984..2dd48924 100644 --- a/cmd/cli/dns_target_diagnostic_darwin_test.go +++ b/cmd/cli/dns_target_diagnostic_darwin_test.go @@ -240,14 +240,11 @@ func TestEnsureInterceptDNSTargetResolutionDoesNotClaimWriteSuccess(t *testing.T h.dhcpErr = nil h.setErr = errors.New("synthetic set failure") p.ensureInterceptDNSTarget([]string{}) - event := oneRecoveryEvent(t, logs, dnsTargetResolvedMessage) - wantField(t, event, "outcome", "decision_available") - wantField(t, event, "reason", "dns_less_network") - wantField(t, event, "action", "unchanged") - wantField(t, event, "ownership_before", false) - wantField(t, event, "ownership_after", false) - if len(h.setCalls) != 1 || p.interceptDNSTargetService != "" { - t.Fatal("failed target write was not exercised or acquired ownership") + if events := jsonLogEvents(t, logs, dnsTargetResolvedMessage); len(events) != 0 { + t.Fatal("failed target write reported a resolved decision") + } + if len(h.setCalls) != 1 || p.interceptDNSTargetService != "Wi-Fi" { + t.Fatal("failed target write did not retain cleanup state for possible partial mutation") } } diff --git a/cmd/cli/dns_target_ownership_darwin_test.go b/cmd/cli/dns_target_ownership_darwin_test.go new file mode 100644 index 00000000..aefaaa52 --- /dev/null +++ b/cmd/cli/dns_target_ownership_darwin_test.go @@ -0,0 +1,168 @@ +//go:build darwin + +package cli + +import ( + "errors" + "fmt" + "net" + "os" + "path/filepath" + "slices" + "testing" +) + +func TestInterceptDNSTargetOwnershipPersistenceBeforeMutation(t *testing.T) { + h := newInterceptTargetHarness(t) + h.nativeCLAT = true + h.dhcpErr = errors.New("no DHCPv4") + h.dns["Wi-Fi"] = []string{"2001:db8::53"} + h.statePath = filepath.Join(t.TempDir(), "absent", "ownership") + p := newInterceptTargetProg() + p.cfg.Listener["0"].IP = "127.0.0.9" + p.cfg.Listener["0"].Port = 5353 + p.ensureInterceptDNSTarget([]string{}) + if len(h.setCalls) != 0 || p.interceptDNSTargetService != "" || !slices.Equal(h.dns["Wi-Fi"], []string{"2001:db8::53"}) { + t.Fatalf("persistence failure mutated DNS: calls=%v dns=%v owner=%s", h.setCalls, h.dns["Wi-Fi"], p.interceptDNSTargetService) + } +} + +func TestInterceptDNSTargetExternalEditInvalidatesBackup(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + p.cfg.Listener["0"].IP = "127.0.0.9" + p.cfg.Listener["0"].Port = 5353 + p.ensureInterceptDNSTarget([]string{}) + target := p.interceptDNSTargetValue() + if !slices.Equal(h.dns["Wi-Fi"], []string{target}) { + t.Fatal("listener-aware target not installed") + } + backup := savedStaticDnsSettingsFilePath(&net.Interface{Name: "Wi-Fi"}) + if err := os.WriteFile(backup, []byte("2001:db8::old"), 0600); err != nil { + t.Fatal(err) + } + external := []string{"2001:db8::54"} + h.dns["Wi-Fi"] = slices.Clone(external) + for range 2 { + p.ensureInterceptDNSTarget([]string{}) + } + if !slices.Equal(h.dns["Wi-Fi"], external) || len(h.setCalls) != 1 { + t.Fatal("reconciliation overwrote external DNS") + } + p.removeInterceptDNSTarget("stop") + oldEach, oldRestore := restoreSavedStaticDNSInterfacesFn, restoreSavedStaticDNSRestoreFn + t.Cleanup(func() { restoreSavedStaticDNSInterfacesFn, restoreSavedStaticDNSRestoreFn = oldEach, oldRestore }) + restoreSavedStaticDNSInterfacesFn = func(_, _ string, f func(*net.Interface) error) { _ = f(&net.Interface{Name: "Wi-Fi"}) } + restoreSavedStaticDNSRestoreFn = func(*net.Interface) error { t.Fatal("full stop sweep replayed stale DNS"); return nil } + restoreSavedStaticDNS("", false) + if _, err := os.Stat(backup); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("stale backup remains available to stop sweep: %v", err) + } + if !slices.Equal(h.dns["Wi-Fi"], external) { + t.Fatal("stop overwrote external DNS") + } +} + +func TestInterceptDNSTargetFailedOwnershipClearRetainsState(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + p.ensureInterceptDNSTarget([]string{}) + if err := os.Remove(h.statePath); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(h.statePath, 0700); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(h.statePath, "busy"), nil, 0600); err != nil { + t.Fatal(err) + } + p.removeInterceptDNSTarget("stop") + if p.interceptDNSTargetService != "Wi-Fi" { + t.Fatal("failed ownership clear reported completed cleanup") + } +} + +func TestInterceptDNSTargetTrackedValueIsNotActualDNS(t *testing.T) { + for _, port := range []int{53, 5353} { + t.Run(fmt.Sprint(port), func(t *testing.T) { + logs := captureDebugMainLog(t) + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + p.cfg.Listener["0"].IP = "127.0.0.9" + p.cfg.Listener["0"].Port = port + p.ensureInterceptDNSTarget([]string{}) + h.dns["Wi-Fi"] = nil + p.ensureInterceptDNSTarget([]string{}) + event := oneRecoveryEvent(t, logs, dnsTargetFailedMessage) + wantField(t, event, "stage", "owned_target_changed") + if len(h.setCalls) != 1 || len(h.dns["Wi-Fi"]) != 0 { + t.Fatal("reinstalled after external clear") + } + }) + } +} + +func TestInterceptDNSTargetFailedCleanupGuardsFullSweep(t *testing.T) { + h := newInterceptTargetHarness(t) + p := newInterceptTargetProg() + p.ensureInterceptDNSTarget([]string{}) + backup := savedStaticDnsSettingsFilePath(&net.Interface{Name: "Wi-Fi"}) + if err := os.WriteFile(backup, []byte("2001:db8::53"), 0600); err != nil { + t.Fatal(err) + } + h.resetErr = errors.New("reset failed") + p.removeInterceptDNSTarget("stop") + oldEach, oldRestore := restoreSavedStaticDNSInterfacesFn, restoreSavedStaticDNSRestoreFn + t.Cleanup(func() { restoreSavedStaticDNSInterfacesFn, restoreSavedStaticDNSRestoreFn = oldEach, oldRestore }) + restoreSavedStaticDNSInterfacesFn = func(_, _ string, f func(*net.Interface) error) { _ = f(&net.Interface{Name: "Wi-Fi"}) } + restoreSavedStaticDNSRestoreFn = func(*net.Interface) error { t.Fatal("sweep bypassed failed guarded cleanup"); return nil } + restoreSavedStaticDNS("", true) + if _, err := os.Stat(backup); err != nil { + t.Fatal("failed cleanup lost backup:", err) + } + if p.interceptDNSTargetService != "Wi-Fi" { + t.Fatal("failed cleanup lost state") + } +} + +func TestInterceptDNSTargetStaticRestoreSkipIsNotFailure(t *testing.T) { + for _, state := range []string{`{"service":"Wi-Fi","value":"127.0.0.53"}`, `invalid ownership`} { + t.Run(state, func(t *testing.T) { + h := newInterceptTargetHarness(t) + logs := captureDebugMainLog(t) + if err := os.WriteFile(h.statePath, []byte(state), 0600); err != nil { + t.Fatal(err) + } + backup := savedStaticDnsSettingsFilePath(&net.Interface{Name: "Wi-Fi"}) + const savedDNS = "2001:db8::53" + if err := os.WriteFile(backup, []byte(savedDNS), 0600); err != nil { + t.Fatal(err) + } + oldEach, oldRestore := restoreSavedStaticDNSInterfacesFn, restoreSavedStaticDNSRestoreFn + t.Cleanup(func() { restoreSavedStaticDNSInterfacesFn, restoreSavedStaticDNSRestoreFn = oldEach, oldRestore }) + calls := 0 + restoreSavedStaticDNSInterfacesFn = func(_, _ string, f func(*net.Interface) error) { + calls++ + if err := f(&net.Interface{Name: "Wi-Fi"}); err != nil { + t.Fatalf("ownership skip reported as restore failure: %v", err) + } + } + restoreSavedStaticDNSRestoreFn = func(*net.Interface) error { + t.Fatal("ownership skip restored DNS") + return nil + } + restoreSavedStaticDNS("", true) + if calls != 1 { + t.Fatalf("interface sweeps=%d, want 1", calls) + } + event := oneRecoveryEvent(t, logs, "Saved static DNS restore skipped on interface Wi-Fi: intercept target cleanup is pending or ownership is unreadable") + wantField(t, event, "level", "debug") + if data, err := os.ReadFile(backup); err != nil || string(data) != savedDNS { + t.Fatalf("ownership skip changed backup: %q, %v", data, err) + } + if data, err := os.ReadFile(h.statePath); err != nil || string(data) != state { + t.Fatalf("ownership skip changed state: %q, %v", data, err) + } + }) + } +} diff --git a/cmd/cli/dns_target_stub.go b/cmd/cli/dns_target_stub.go index 680ede3b..459e7b2f 100644 --- a/cmd/cli/dns_target_stub.go +++ b/cmd/cli/dns_target_stub.go @@ -2,6 +2,8 @@ package cli +func interceptTargetAllowsStaticRestore(_ string) bool { return true } + // ensureInterceptDNSTarget is a no-op on non-Darwin platforms: the DNS-less // network problem it solves is specific to macOS pf interception blocking // IPv6 port 53 with no IPv4 fallback (issue #533). Windows intercept mode diff --git a/cmd/cli/listener_5353_darwin_test.go b/cmd/cli/listener_5353_darwin_test.go new file mode 100644 index 00000000..24abcc67 --- /dev/null +++ b/cmd/cli/listener_5353_darwin_test.go @@ -0,0 +1,67 @@ +//go:build darwin + +package cli + +import ( + "fmt" + ctrld "github.com/Control-D-Inc/ctrld" + "strings" + "testing" +) + +func TestPFScopedNameserverAddress(t *testing.T) { + for _, tc := range []struct{ server, host, family string }{ + {"[fe80::1%en0]:53", "fe80::1", "inet6"}, + {"2001:db8::53", "2001:db8::53", "inet6"}, + {"192.0.2.53:53", "192.0.2.53", "inet"}, + } { + host, family, ok := pfNameserverAddress(tc.server) + if !ok || host != tc.host || family != tc.family { + t.Fatalf("PF nameserver %q: %q %q %v", tc.server, host, family, ok) + } + } + if _, _, ok := pfNameserverAddress("not an address"); ok { + t.Fatal("invalid PF address accepted") + } +} + +func TestPFRecoveryScopedNameserverRules(t *testing.T) { + captureDebugMainLog(t) + p := &prog{cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.2", Port: 5353}}}} + // Recovery converts OS-discovered addresses into this exemption list. + rules := p.buildPFAnchorRulesForTunnels([]vpnDNSExemption{ + {Server: "fe80::1%en0"}, {Server: "fe80::1%en1"}, {Server: "192.0.2.53"}, + }, nil) + want := "pass out quick on ! lo0 inet6 proto { udp, tcp } from any to fe80::1 port 53 group " + pfGroupName + if strings.Count(rules, want) != 1 { + t.Fatalf("scoped recovery exemption missing or duplicated: %s", rules) + } + if strings.Contains(rules, "%en0") || strings.Contains(rules, "%en1") || strings.Contains(rules, "inet proto { udp, tcp } from any to fe80:") { + t.Fatal("socket zone or IPv4 family leaked into recovery PF rules") + } + if !strings.Contains(rules, "inet proto { udp, tcp } from any to 192.0.2.53 port 53 group "+pfGroupName) { + t.Fatal("IPv4 recovery exemption changed") + } +} + +func TestCustomListener5353PFRules(t *testing.T) { + captureDebugMainLog(t) + for _, ip := range []string{"127.0.0.1", "127.0.0.2", "127.0.0.53", "192.0.2.10"} { + t.Run(ip, func(t *testing.T) { + p := &prog{cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: ip, Port: 5353}}}} + rules := p.buildPFAnchorRulesForTunnels(nil, []string{"utun9"}) + for _, proto := range []string{"udp", "tcp"} { + want := fmt.Sprintf("rdr on lo0 inet proto %s from any to ! %s port 53 -> %s port 5353", proto, ip, ip) + if !strings.Contains(rules, want) { + t.Errorf("missing redirect %s", want) + } + } + if strings.Contains(rules, "port 5354") { + t.Fatal("explicit port replaced with automatic fallback") + } + if p.interceptDNSTargetValue() == ip { + t.Fatal("target equals rdr exclusion: no port translation") + } + }) + } +} diff --git a/cmd/cli/listener_5353_linux_test.go b/cmd/cli/listener_5353_linux_test.go new file mode 100644 index 00000000..273cf53f --- /dev/null +++ b/cmd/cli/listener_5353_linux_test.go @@ -0,0 +1,32 @@ +//go:build linux + +package cli + +import ( + ctrld "github.com/Control-D-Inc/ctrld" + "net" + "testing" +) + +// Socket-only Linux control. Native macOS PF delivery is a separate QA gate. +func TestCustomListener5353Binding(t *testing.T) { + captureDebugMainLog(t) + for _, ip := range []string{"127.0.0.1", "127.0.0.2"} { + t.Run(ip, func(t *testing.T) { + cfg := &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: ip, Port: 5353}}} + changed, ok := tryUpdateListenerConfigIntercept(cfg, func() {}, false) + if !ok || changed { + t.Fatalf("custom listener did not bind unchanged: changed=%v ok=%v", changed, ok) + } + occupied, err := net.Listen("tcp4", net.JoinHostPort(ip, "5353")) + if err != nil { + t.Fatal(err) + } + defer occupied.Close() + changed, ok = tryUpdateListenerConfigIntercept(cfg, func() {}, false) + if ok || changed || cfg.Listener["0"].IP != ip || cfg.Listener["0"].Port != 5353 { + t.Fatal("occupied explicit listener silently fell back") + } + }) + } +} diff --git a/cmd/cli/listener_5353_target_darwin_test.go b/cmd/cli/listener_5353_target_darwin_test.go new file mode 100644 index 00000000..8259e599 --- /dev/null +++ b/cmd/cli/listener_5353_target_darwin_test.go @@ -0,0 +1,41 @@ +//go:build darwin + +package cli + +import ( + "errors" + "slices" + "testing" +) + +func TestNativeCLATCustomListener5353Lifecycle(t *testing.T) { + for _, ip := range []string{"127.0.0.1", "127.0.0.2", "127.0.0.53", "192.0.2.10"} { + t.Run(ip, func(t *testing.T) { + h := newInterceptTargetHarness(t) + h.nativeCLAT = true + h.dhcpErr = errors.New("DHCPv4 intentionally absent") + p := newInterceptTargetProg() + p.cfg.Listener["0"].IP = ip + p.cfg.Listener["0"].Port = 5353 + target := p.interceptDNSTargetValue() + p.ensureInterceptDNSTarget([]string{}) + p.ensureInterceptDNSTarget([]string{}) + if !slices.Equal(h.dns["Wi-Fi"], []string{target}) || len(h.setCalls) != 1 { + t.Fatalf("target was not installed idempotently: %v writes %v", h.dns, h.setCalls) + } + if target == ip { + t.Fatal("OS DNS would hit port53 at listener instead of PF port5353 redirect") + } + h.dhcpErr = nil + h.dhcp = []string{"192.0.2.53"} + h.nativeCLAT = false + p.ensureInterceptDNSTarget([]string{"192.0.2.53"}) + if len(h.dns["Wi-Fi"]) != 0 || p.interceptDNSTargetService != "" { + t.Fatalf("DHCP return failed to remove target: %+v", h.dns) + } + if p.cfg.Listener["0"].IP != ip || p.cfg.Listener["0"].Port != 5353 { + t.Fatal("target lifecycle changed configured listener") + } + }) + } +} diff --git a/cmd/cli/listener_5353_test.go b/cmd/cli/listener_5353_test.go new file mode 100644 index 00000000..611b4437 --- /dev/null +++ b/cmd/cli/listener_5353_test.go @@ -0,0 +1,34 @@ +package cli + +import ( + ctrld "github.com/Control-D-Inc/ctrld" + "testing" +) + +func TestCustomListener5353Contract(t *testing.T) { + for _, tc := range []struct{ ip, target string }{ + {"127.0.0.1", "127.0.0.53"}, {"127.0.0.2", "127.0.0.53"}, + {"127.0.0.53", "127.0.0.54"}, {"192.0.2.10", "127.0.0.53"}, + } { + t.Run(tc.ip, func(t *testing.T) { + if !isExplicitInterceptListener(tc.ip, 5353) { + t.Fatal("5353 was not explicit") + } + p := &prog{cfg: &ctrld.Config{Listener: map[string]*ctrld.ListenerConfig{"0": {IP: tc.ip, Port: 5353}}}} + if got := p.interceptDNSTargetValue(); got != tc.target { + t.Fatalf("target=%s want %s", got, tc.target) + } + current := p.cfg.Listener + generated := map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 53}} + preserveBoundListeners(generated, current) + if generated["0"].IP != tc.ip || generated["0"].Port != 5353 { + t.Fatalf("generated reload lost bound listener: %+v", generated["0"]) + } + explicit := map[string]*ctrld.ListenerConfig{"0": {IP: tc.ip, Port: 5353}} + preserveBoundListeners(explicit, map[string]*ctrld.ListenerConfig{"0": {IP: "127.0.0.1", Port: 5354}}) + if explicit["0"].IP != tc.ip || explicit["0"].Port != 5353 { + t.Fatalf("explicit reload normalized: %+v", explicit["0"]) + } + }) + } +} diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 0a3cc85c..7957fa21 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -323,6 +323,8 @@ type prog struct { // VPN DNS manager for split DNS routing when intercept mode is active. vpnDNS *vpnDNSManager + // Serializes journal reads with publication after an intercept startup retry. + vpnDNSJournalMu sync.Mutex started chan struct{} onStartedDone chan struct{} @@ -1178,6 +1180,27 @@ func (p *prog) startNetworkJournal() func() { // drop. The search suffixes name the organization of the endpoint, so the // journal holds their number only. func (p *prog) logDNSConfigChanges(entries []dnsResolverEntry) { + // The poller emits one batch per changed snapshot (at most once per poll), + // not one discovery per resolver. Late DNS publication on an unchanged + // tunnel otherwise has no network event to refresh split routing. + // Join the existing shutdown fence before touching the manager or PF/WFP: + // closeNetMonitor drains this callback before restoring the host's DNS. + if len(entries) > 0 && p.beginNetworkActivity() { + defer p.netMonitorWG.Done() + // setDNS publishes the manager before signalling completion. The journal + // starts earlier; never race initial manager publication or initialize a + // manager in traditional/hard intercept mode. + select { + case <-p.csSetDnsDone: + p.vpnDNSJournalMu.Lock() + manager := p.vpnDNS + p.vpnDNSJournalMu.Unlock() + if manager != nil { + manager.Refresh(true) + } + default: + } + } for _, entry := range entries { journal(mainLog.Load().Info()). Strs("nameservers", entry.Nameservers). @@ -1546,8 +1569,10 @@ func (p *prog) setDNS(systemNameservers []string) { // Discovers search domains from virtual/VPN interfaces and forwards // matching queries to the DNS server on that interface. // Skipped in --intercept-mode hard where all DNS goes through ctrld. + p.vpnDNSJournalMu.Lock() p.vpnDNS = newVPNDNSManager(p.exemptVPNDNSServers) p.vpnDNS.Refresh(true) + p.vpnDNSJournalMu.Unlock() } setDnsOK = true @@ -1802,7 +1827,9 @@ func (p *prog) resetDNS(isStart bool, restoreStatic bool) { } // Clean up VPN DNS manager + p.vpnDNSJournalMu.Lock() p.vpnDNS = nil + p.vpnDNSJournalMu.Unlock() return } diff --git a/cmd/cli/vpn_dns.go b/cmd/cli/vpn_dns.go index 2c8516ae..2bcf1026 100644 --- a/cmd/cli/vpn_dns.go +++ b/cmd/cli/vpn_dns.go @@ -176,11 +176,7 @@ func (m *vpnDNSManager) refreshOnce(guardAgainstNoNameservers bool) { config.InterfaceName, len(config.Domains), len(config.Servers)) for _, domain := range config.Domains { - // Normalize domain: remove leading dot, Linux routing domain prefix (~), - // and convert to lowercase. - domain = strings.TrimPrefix(domain, "~") - domain = strings.TrimPrefix(domain, ".") - domain = strings.ToLower(domain) + domain = normalizeVPNDomain(domain) if domain != "" { m.routes[domain] = append([]string{}, config.Servers...) @@ -288,9 +284,7 @@ func (m *vpnDNSManager) RefreshRoutesOnly() (routes, domainlessServers, exemptio for _, config := range configs { for _, domain := range config.Domains { - domain = strings.TrimPrefix(domain, "~") - domain = strings.TrimPrefix(domain, ".") - domain = strings.ToLower(domain) + domain = normalizeVPNDomain(domain) if domain != "" { m.routes[domain] = append([]string{}, config.Servers...) } @@ -389,6 +383,14 @@ func (m *vpnDNSManager) VPNDNSReachable() { m.retainedAfterEmptyDiscovery = false } +// normalizeVPNDomain accepts OS suffix and Linux routing-domain spellings. +// The root alone is not a split-DNS route (including the Linux ~. catch-all). +func normalizeVPNDomain(domain string) string { + domain = strings.TrimPrefix(domain, "~") + domain = strings.TrimPrefix(domain, ".") + return strings.ToLower(strings.TrimSuffix(domain, ".")) +} + // UpstreamForDomain checks if the domain matches any VPN search domain. // Returns VPN DNS servers if matched, nil otherwise. func (m *vpnDNSManager) UpstreamForDomain(domain string) []string { @@ -402,14 +404,17 @@ func (m *vpnDNSManager) UpstreamForDomain(domain string) []string { domain = strings.TrimSuffix(domain, ".") domain = strings.ToLower(domain) - if servers, ok := m.routes[domain]; ok { - return append([]string{}, servers...) - } - - for vpnDomain, servers := range m.routes { - if strings.HasSuffix(domain, "."+vpnDomain) { + // Walk label boundaries from most to least specific. Map iteration order + // must never choose a parent VPN's resolver over a matching child zone. + for domain != "" { + if servers, ok := m.routes[domain]; ok { return append([]string{}, servers...) } + i := strings.IndexByte(domain, '.') + if i < 0 { + break + } + domain = domain[i+1:] } return nil diff --git a/cmd/cli/vpn_dns_convergence_test.go b/cmd/cli/vpn_dns_convergence_test.go new file mode 100644 index 00000000..d07cfbae --- /dev/null +++ b/cmd/cli/vpn_dns_convergence_test.go @@ -0,0 +1,133 @@ +package cli + +import ( + "context" + "reflect" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/Control-D-Inc/ctrld" +) + +func TestVPNDNSLongestNormalizedSuffixAcrossRefreshes(t *testing.T) { + for _, routeOnly := range []bool{false, true} { + t.Run(map[bool]string{false: "full", true: "route-only"}[routeOnly], func(t *testing.T) { + m := newVPNDNSManager(nil) + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + return []ctrld.VPNDNSConfig{ + {Domains: []string{".EXAMPLE.com."}, Servers: []string{"10.0.0.1"}}, + {Domains: []string{"~Child.Example.COM."}, Servers: []string{"10.0.0.2", "10.0.0.3"}}, + {Domains: []string{".", "~."}, Servers: []string{"10.0.0.4"}}, + } + } + if routeOnly { + m.RefreshRoutesOnly() + } else { + m.Refresh(true) + } + for query, want := range map[string][]string{ + "example.com.": {"10.0.0.1"}, "other.example.com": {"10.0.0.1"}, + "child.example.com.": {"10.0.0.2", "10.0.0.3"}, + "HOST.Deep.Child.Example.COM.": {"10.0.0.2", "10.0.0.3"}, + "notexample.com.": nil, "example.com.other.": nil, "public.test.": nil, + } { + for i := 0; i < 1000; i++ { + if got := m.UpstreamForDomain(query); !reflect.DeepEqual(got, want) { + t.Fatalf("%s: got %v want %v", query, got, want) + } + } + } + got := m.UpstreamForDomain("child.example.com") + got[0] = "mutated" + if m.UpstreamForDomain("child.example.com")[0] != "10.0.0.2" { + t.Fatal("lookup leaked mutable slice") + } + }) + } +} + +// Drive the real poller diff -> journal callback -> manager discovery path. +// Both OS boundaries are fixtures, so this test cannot touch host DNS or PF. +func TestVPNDNSJournalLatePublicationConverges(t *testing.T) { + m := newVPNDNSManager(nil) + var discoveryCalls, updates int + var snapshot []ctrld.VPNDNSConfig + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { discoveryCalls++; return snapshot } + m.onServersChanged = func([]vpnDNSExemption) error { updates++; return nil } + p := &prog{vpnDNS: m, csSetDnsDone: make(chan struct{})} + close(p.csSetDnsDone) + var table []dnsResolverEntry + poller := newDNSConfigPoller(func() ([]dnsResolverEntry, error) { return table, nil }) + emit := func() { p.logDNSConfigChanges(poller.pollOnce(time.Now())) } + emit() // empty startup baseline + for _, server := range []string{"10.0.0.1", "10.0.0.2"} { + snapshot = []ctrld.VPNDNSConfig{{InterfaceName: "utun-fixture", Domains: []string{"corp.example."}, Servers: []string{server}}} + table = []dnsResolverEntry{{Interface: "utun-fixture", Nameservers: []string{server}}} + emit() + if got := m.UpstreamForDomain("host.corp.example."); len(got) != 1 || got[0] != server { + t.Fatalf("late DNS-only change not published: %v", got) + } + for i := 0; i < 100; i++ { + emit() + } + } + if discoveryCalls != 2 || updates != 2 { + t.Fatalf("unchanged tables caused refresh storm: discoveries=%d updates=%d", discoveryCalls, updates) + } + // Removal is a DNS-only delta too; keep Windows' one-empty-snapshot guard. + snapshot = nil + table = nil + emit() + if vpnDNSSettlingEnabled { // Preserve Windows' one-empty-snapshot guard. + m.Refresh(true) + } + if got := m.UpstreamForDomain("host.corp.example"); len(got) != 0 { + t.Fatalf("removed DNS retained: %v", got) + } + p.closeNetMonitor() +} + +func TestVPNDNSJournalShutdownDrainsAndRejectsRefresh(t *testing.T) { + m := newVPNDNSManager(nil) + entered, release := make(chan struct{}), make(chan struct{}) + var calls, updates atomic.Int32 + m.discoverVPNDNS = func(context.Context) []ctrld.VPNDNSConfig { + calls.Add(1) + close(entered) + <-release + return []ctrld.VPNDNSConfig{{Domains: []string{"corp.example"}, Servers: []string{"10.0.0.1"}}} + } + m.onServersChanged = func([]vpnDNSExemption) error { updates.Add(1); return nil } + p := &prog{vpnDNS: m, csSetDnsDone: make(chan struct{})} + entries := []dnsResolverEntry{{Action: resolverActionChanged}} + p.logDNSConfigChanges(entries) + if calls.Load() != 0 { + t.Fatal("discovery ran before DNS initialization") + } + close(p.csSetDnsDone) + callbackDone := make(chan struct{}) + go func() { defer close(callbackDone); p.logDNSConfigChanges(entries) }() + <-entered + stopDone := make(chan struct{}) + go func() { p.closeNetMonitor(); close(stopDone) }() + <-p.networkActivityDone() + select { + case <-stopDone: + t.Fatal("shutdown did not drain in-flight refresh") + default: + } + close(release) + <-callbackDone + <-stopDone + var wg sync.WaitGroup + for i := 0; i < 100; i++ { + wg.Add(1) + go func() { defer wg.Done(); p.logDNSConfigChanges(entries) }() + } + wg.Wait() + if calls.Load() != 1 || updates.Load() != 1 { + t.Fatalf("post-stop mutation: discovery=%d updates=%d", calls.Load(), updates.Load()) + } +} diff --git a/docs/619-target-ownership-limits.md b/docs/619-target-ownership-limits.md new file mode 100644 index 00000000..7ccad15a --- /dev/null +++ b/docs/619-target-ownership-limits.md @@ -0,0 +1,17 @@ +# Target ownership safety (relates to #619) + +The target cleanup record is not proof of actual service DNS. Reconciliation now reports a tracked/actual mismatch instead of returning an implicit already-installed decision. An external clear or IPv6-only edit does not authorize reinstalling a target; the record is retained to prevent repeated watchdog admission. Usable external IPv4 keeps the existing guarded cleanup path. + +Ownership is written by atomic replacement before target installation. Failure refuses installation, rather than depending on a future restart to recover an untracked target. A failed DNS command may have partly mutated OS state, so its cleanup record is retained. Failed ownership-file removal also retains memory state and does not emit the target-removed success event. + +On external edits, cleanup discards the stale saved-static backup before releasing the record. The full saved-static sweep skips a service with retained target cleanup state; unreadable/malformed ownership state conservatively skips the sweep. No unbounded retry, fatal exit, or machine-stop policy is introduced. + +## Evidence and limits + +- Linux-adapted Darwin tests execute the real target lifecycle and saved-static sweep with only OS boundaries replaced. New regression tests cover persistence failure before mutation, external edit through reconciliation/cleanup/full sweep, ownership clear failure, actual-vs-tracked discrepancy, and failed-cleanup sweep guards. Custom listener values are configured and assertions derive the target from the production selector. +- This is not native macOS PF/CLAT proof. Existing native CLAT admission/removal behavior still needs native validation on the integrated artifact; no PF rules or target selector were changed here. +- A missing target is not automatically repaired: external clearing and accidental loss cannot safely be distinguished from these observations. The diagnostic reports the uncertainty; it does not claim resolution. +- Once explicit cleanup releases a record, a later new admission uses the existing policy for IPv6-only static DNS. This is not a persistent per-service operator opt-out mechanism. +- Retaining cleanup state does not guarantee another cleanup attempt after process exit. The void shutdown API still cannot report aggregate stop failure to its caller; no guaranteed retry or issue closure is claimed. +- Startup/running-interface reset and callback shutdown ordering live in prog.go and are outside this patch. Persistence failure was not a universal restart orphan on the baseline: startup can rescue the current interface, while old/noncurrent-service cases are conditional. +- Network configuration commands are not compare-and-swap transactions. A concurrent external edit after the last read remains a native race; these tests do not prove otherwise. Atomic file replacement does not claim power-loss durability. diff --git a/nameservers_darwin.go b/nameservers_darwin.go index 2ae1c297..65ac705b 100644 --- a/nameservers_darwin.go +++ b/nameservers_darwin.go @@ -22,6 +22,8 @@ func dnsFns() []dnsFn { return []dnsFn{dnsFromResolvConf, getDNSFromScutil, getAllDHCPNameservers} } +var scutilLocalAddresses = netmon.LocalAddresses + func getDNSFromScutil() []string { logger := *ProxyLogger.Load() @@ -36,7 +38,7 @@ func getDNSFromScutil() []string { retryInterval = 100 * time.Millisecond ) - regularIPs, loopbackIPs, _ := netmon.LocalAddresses() + regularIPs, loopbackIPs, _ := scutilLocalAddresses() var nameservers []string for attempt := 0; attempt < maxRetries; attempt++ { @@ -51,36 +53,8 @@ func getDNSFromScutil() []string { continue } - var localDNS []string - seen := make(map[string]bool) - - scanner := bufio.NewScanner(bytes.NewReader(output)) - for scanner.Scan() { - line := strings.TrimSpace(scanner.Text()) - if strings.HasPrefix(line, "nameserver[") { - parts := strings.Split(line, ":") - if len(parts) == 2 { - ns := strings.TrimSpace(parts[1]) - if ip := net.ParseIP(ns); ip != nil { - // skip loopback IPs - isLocal := false - for _, v := range slices.Concat(regularIPs, loopbackIPs) { - ipStr := v.String() - if ip.String() == ipStr { - isLocal = true - break - } - } - if !isLocal && !seen[ip.String()] { - seen[ip.String()] = true - localDNS = append(localDNS, ip.String()) - } - } - } - } - } - - if err := scanner.Err(); err != nil { + localDNS, err := parseScutilNameservers(output, slices.Concat(regularIPs, loopbackIPs)) + if err != nil { Log(context.Background(), logger.Error(), "error scanning scutil output (attempt %d/%d): %v", attempt+1, maxRetries, err) continue } diff --git a/nameservers_scutil.go b/nameservers_scutil.go new file mode 100644 index 00000000..cf745915 --- /dev/null +++ b/nameservers_scutil.go @@ -0,0 +1,45 @@ +package ctrld + +import ( + "bufio" + "bytes" + "net/netip" + "strings" +) + +// Link-local addresses identify a host only within their interface's zone. +// In particular, an unzoned local fe80::1 must not exclude fe80::1%en0. +func scutilLocalAddressKey(addr netip.Addr) netip.Addr { + addr = addr.Unmap() + if addr.IsLinkLocalUnicast() { + return addr + } + return addr.WithZone("") +} + +// parseScutilNameservers retains IPv6 zones for the eventual socket dial. +func parseScutilNameservers(output []byte, local []netip.Addr) ([]string, error) { + excluded := make(map[netip.Addr]bool, len(local)) + for _, addr := range local { + excluded[scutilLocalAddressKey(addr)] = true + } + var servers []string + seen := make(map[netip.Addr]bool) + scanner := bufio.NewScanner(bytes.NewReader(output)) + for scanner.Scan() { + key, value, ok := strings.Cut(strings.TrimSpace(scanner.Text()), ":") + if !ok || !strings.HasPrefix(key, "nameserver[") { + continue + } + addr, err := netip.ParseAddr(strings.TrimSpace(value)) + if err != nil || excluded[scutilLocalAddressKey(addr)] || seen[addr] { + continue + } + seen[addr] = true + servers = append(servers, addr.String()) + } + if err := scanner.Err(); err != nil { + return nil, err + } + return servers, nil +} diff --git a/nameservers_scutil_darwin_test.go b/nameservers_scutil_darwin_test.go new file mode 100644 index 00000000..55b5e0c6 --- /dev/null +++ b/nameservers_scutil_darwin_test.go @@ -0,0 +1,51 @@ +//go:build darwin + +package ctrld + +import ( + "net" + "net/netip" + "os" + "path/filepath" + "slices" + "testing" +) + +func TestScopedOSResolverDoesNotBindDefaultSource(t *testing.T) { + old4, old6 := GetDefaultLocalIPv4(), GetDefaultLocalIPv6() + SetDefaultLocalIPv4(net.ParseIP("192.0.2.10")) + SetDefaultLocalIPv6(net.ParseIP("2001:db8::10")) + t.Cleanup(func() { SetDefaultLocalIPv4(old4); SetDefaultLocalIPv6(old6) }) + if got := defaultLocalIPForServer("[fe80::1%en0]:53"); got != nil { + t.Fatalf("scoped resolver bound unrelated source %v", got) + } + if got := defaultLocalIPForServer("[2001:db8::53]:53"); !got.Equal(net.ParseIP("2001:db8::10")) { + t.Fatalf("global IPv6 source changed: %v", got) + } + if got := defaultLocalIPForServer("192.0.2.53:53"); !got.Equal(net.ParseIP("192.0.2.10")) { + t.Fatalf("IPv4 source changed: %v", got) + } +} + +func TestScutilIPv6ProductionReader(t *testing.T) { + old := scutilLocalAddresses + t.Cleanup(func() { scutilLocalAddresses = old }) + localReads := 0 + scutilLocalAddresses = func() ([]netip.Addr, []netip.Addr, error) { + localReads++ + return []netip.Addr{netip.MustParseAddr("2001:db8::10"), netip.MustParseAddr("fe80::2%en0")}, + []netip.Addr{netip.MustParseAddr("::1"), netip.MustParseAddr("fe80::1")}, nil + } + dir := t.TempDir() + script := "#!/bin/sh\nprintf '%s\\n' 'nameserver[0] : 2001:db8::53' 'nameserver[1] : fe80::1%en0' 'nameserver[2] : fe80::2%en0' 'nameserver[3] : ::1' 'nameserver[4] : 2001:db8::10'\n" + if err := os.WriteFile(filepath.Join(dir, "scutil"), []byte(script), 0700); err != nil { + t.Fatal(err) + } + t.Setenv("PATH", dir) + if got := getDNSFromScutil(); !slices.Equal(got, []string{"2001:db8::53", "fe80::1%en0"}) { + t.Fatalf("production reader lost IPv6: %v", got) + } + if localReads != 1 { + t.Fatalf("local-address reads=%d, want 1", localReads) + } +} diff --git a/nameservers_scutil_test.go b/nameservers_scutil_test.go new file mode 100644 index 00000000..7c2b0904 --- /dev/null +++ b/nameservers_scutil_test.go @@ -0,0 +1,80 @@ +package ctrld + +import ( + "net/netip" + "slices" + "strings" + "testing" +) + +func TestScutilNameserversIPv6AndZones(t *testing.T) { + output := []byte(`DNS configuration +resolver #1 + nameserver[0] : 192.0.2.53 + nameserver[1] : 2001:db8::53 + nameserver[2] : fe80::1%en0 + nameserver[3] : 2001:db8::53 + nameserver[4] : fe80::1%en1 + nameserver[5] : 127.0.0.1 + nameserver[6] : ::1 + nameserver[7] : fe80::2%en0 + nameserver[8] : not-an-address + search domain[0] : internal.example +`) + local := []netip.Addr{netip.MustParseAddr("127.0.0.1"), netip.MustParseAddr("::1"), netip.MustParseAddr("fe80::2%en0")} + got, err := parseScutilNameservers(output, local) + want := []string{"192.0.2.53", "2001:db8::53", "fe80::1%en0", "fe80::1%en1"} + if err != nil || !slices.Equal(got, want) { + t.Fatalf("nameservers=%v err=%v", got, err) + } + effective := initializeOsResolver(got) + r := newResolverWithNameserver(effective) + if !slices.Contains(*r.lanServers.Load(), "[fe80::1%en0]:53") || !slices.Contains(*r.lanServers.Load(), "[fe80::1%en1]:53") { + t.Fatalf("zones lost in effective resolver: %v", effective) + } + if !slices.Contains(*r.publicServers.Load(), "[2001:db8::53]:53") { + t.Fatalf("global IPv6 lost: %v", effective) + } +} + +func TestScutilNameserversLocalExclusion(t *testing.T) { + for _, tt := range []struct { + name, local, server string + excluded bool + }{ + {"unzoned link-local does not exclude scoped router", "fe80::1", "fe80::1%en0", false}, + {"loopback link-local does not exclude router", "fe80::1%lo0", "fe80::1%en0", false}, + {"other link does not exclude router", "fe80::1%en1", "fe80::1%en0", false}, + {"same link excludes local address", "fe80::1%en0", "fe80::1%en0", true}, + {"unzoned link-local matches unzoned local", "fe80::1", "fe80::1", true}, + {"scoped local does not exclude unzoned candidate", "fe80::1%en0", "fe80::1", false}, + {"global ignores zones", "2001:db8::1%en0", "2001:db8::1%en1", true}, + {"loopback ignores zones", "::1%lo0", "::1", true}, + {"IPv4 mapped local", "::ffff:192.0.2.1", "192.0.2.1", true}, + {"IPv4 mapped candidate", "192.0.2.1", "::ffff:192.0.2.1", true}, + {"IPv4 link-local mapped candidate", "169.254.1.1", "::ffff:169.254.1.1", true}, + {"other global is retained", "2001:db8::1", "2001:db8::53", false}, + } { + t.Run(tt.name, func(t *testing.T) { + got, err := parseScutilNameservers([]byte("nameserver[0] : "+tt.server+"\n"), []netip.Addr{netip.MustParseAddr(tt.local)}) + if err != nil { + t.Fatal(err) + } + var want []string + if !tt.excluded { + want = []string{tt.server} + } + if !slices.Equal(got, want) { + t.Fatalf("nameservers=%v, want %v", got, want) + } + }) + } +} + +func TestScutilNameserversRejectsPartialScan(t *testing.T) { + output := []byte("nameserver[0] : 192.0.2.53\n" + strings.Repeat("x", 128*1024)) + got, err := parseScutilNameservers(output, nil) + if err == nil || got != nil { + t.Fatalf("partial scan accepted: %v, %v", got, err) + } +} diff --git a/resolver.go b/resolver.go index 959d57a9..38e48e7f 100644 --- a/resolver.go +++ b/resolver.go @@ -522,8 +522,13 @@ func defaultLocalIPForServer(server string) net.IP { if err != nil { return nil } - ip := net.ParseIP(host) - if ip != nil && ip.To4() == nil { + addr, err := netip.ParseAddr(host) + if err == nil && addr.Zone() != "" { + // Scoped resolvers select their interface via the destination zone. + // A cached default-interface source can have the wrong family or scope. + return nil + } + if err == nil && addr.Is6() && !addr.Is4In6() { return GetDefaultLocalIPv6() } return GetDefaultLocalIPv4() From db228a86ae16a88309024e18801ee8715292d06f Mon Sep 17 00:00:00 2001 From: Anthony Wong Date: Thu, 1 Oct 2026 13:16:24 +0000 Subject: [PATCH 26/29] fix(cli): fix three log journal defects and test Windows with the race detector --- cmd/cli/dns_proxy_sampling_test.go | 56 +++++++++++ cmd/cli/log_budget_apply_test.go | 142 +++++++++++++++++++++++++++ cmd/cli/log_file.go | 36 +++++-- cmd/cli/log_file_test.go | 69 +++++++++++++ cmd/cli/log_header.go | 13 ++- cmd/cli/log_reader_test.go | 75 ++++++++++++++ cmd/cli/log_writer.go | 26 ++++- cmd/cli/network_events_test.go | 52 ++++++---- cmd/cli/network_snapshot.go | 23 ++++- cmd/cli/network_snapshot_test.go | 37 +++++++ cmd/cli/prog.go | 1 + docs/network-recovery-diagnostics.md | 2 + docs/runtime-internal-logging.md | 4 +- 13 files changed, 505 insertions(+), 31 deletions(-) create mode 100644 cmd/cli/log_budget_apply_test.go diff --git a/cmd/cli/dns_proxy_sampling_test.go b/cmd/cli/dns_proxy_sampling_test.go index e6d27bbf..9b27b637 100644 --- a/cmd/cli/dns_proxy_sampling_test.go +++ b/cmd/cli/dns_proxy_sampling_test.go @@ -170,3 +170,59 @@ func Test_allEndpointsFailedBoundsTheUpstreamName(t *testing.T) { t.Fatalf("the all-endpoints line holds the operator key: %q", reported) } } + +// Test_upstreamOutageOnTheQueryPathLogsOneStatePair drives 633 failed queries +// through the real query path, the count of the customer upload that sized the +// journal. The outage must give one down event, and the recovery one up event +// with the length of the outage. +func Test_upstreamOutageOnTheQueryPathLogsOneStatePair(t *testing.T) { + const failedQueries = 633 + logs := captureDebugMainLog(t) + cfg := &ctrld.Config{} + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) + p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg)} + // A new monitor holds back the failure counts for one second. The test + // clears the flag instead of waiting for that timer. + p.um.clearRecovered(upstreamOS) + p.um.after = func(time.Duration, func()) {} + t.Cleanup(func() { p.querySampler.closeExpired(time.Now().Add(querySampleWindow)) }) + + origOSUpstream := osUpstreamConfig + t.Cleanup(func() { osUpstreamConfig = origOSUpstream }) + // Windows reports no port-unreachable error on a UDP socket, so each query + // waits for the full timeout there. A short timeout keeps 633 queries fast. + osUpstreamConfig = &ctrld.UpstreamConfig{ + Name: "dead resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 20, + } + + ctx := context.Background() + for i := 0; i < failedQueries; i++ { + if rcode := askThroughProxy(t, p, ctx).Rcode; rcode != dns.RcodeServerFailure { + t.Fatalf("query %d: rcode %s, want SERVFAIL", i+1, dns.RcodeToString[rcode]) + } + } + events := jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 1 { + t.Fatalf("state events after %d failed queries: got %d, want 1", failedQueries, len(events)) + } + wantField(t, events[0], "upstream", upstreamOS) + wantField(t, events[0], "state", "down") + + startOSResolverStub(t) + if rcode := askThroughProxy(t, p, ctx).Rcode; rcode != dns.RcodeSuccess { + t.Fatalf("the answered query returned %s, want NOERROR", dns.RcodeToString[rcode]) + } + + events = jsonLogEvents(t, logs, upstreamStateMessage) + if len(events) != 2 { + t.Fatalf("state events after the recovery: got %d, want 2", len(events)) + } + wantField(t, events[1], "upstream", upstreamOS) + wantField(t, events[1], "state", "up") + if _, carried := events[1]["down_for_ms"].(float64); !carried { + t.Fatalf("the up event carries no down_for_ms: %v", events[1]) + } +} diff --git a/cmd/cli/log_budget_apply_test.go b/cmd/cli/log_budget_apply_test.go new file mode 100644 index 00000000..abc27745 --- /dev/null +++ b/cmd/cli/log_budget_apply_test.go @@ -0,0 +1,142 @@ +package cli + +import ( + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/Control-D-Inc/ctrld" +) + +// setupDebugBudgetTest seeds a debug file with four backups, which is the +// default count, and puts back the two limit keys after the test. +func setupDebugBudgetTest(t *testing.T) string { + t.Helper() + dir := setupLogSetupTest(t) + origSize, origBackups := cfg.Service.LogMaxSizeMB, cfg.Service.LogMaxBackups + t.Cleanup(func() { + cfg.Service.LogMaxSizeMB, cfg.Service.LogMaxBackups = origSize, origBackups + }) + cfg.Service.LogMaxSizeMB, cfg.Service.LogMaxBackups = 0, nil + debugPath := filepath.Join(dir, logFileName) + for i := 1; i <= 4; i++ { + if err := os.WriteFile(debugPath+"."+strconv.Itoa(i), []byte("old run\n"), 0o600); err != nil { + t.Fatal(err) + } + } + return debugPath +} + +// assertDebugBudget checks the limits of the open debug file and the backups +// that stay on disk. +func assertDebugBudget(t *testing.T, p *prog, debugPath string, want logBudget) { + t.Helper() + debugWriter, _ := p.internalWriters() + rf := debugWriter.rotating() + if rf == nil { + t.Fatal("the debug file is not open") + } + rf.mu.Lock() + got := rf.budget + rf.mu.Unlock() + if got != want { + t.Fatalf("debug budget = %+v, want %+v", got, want) + } + for i := 1; i <= 4; i++ { + _, err := os.Stat(debugPath + "." + strconv.Itoa(i)) + if i <= want.backups && err != nil { + t.Fatalf("backup %d is gone: %v", i, err) + } + if i > want.backups && !os.IsNotExist(err) { + t.Fatalf("backup %d stays although the limit is %d backups (stat err = %v)", i, want.backups, err) + } + } +} + +// Test_initLogging_appliesTheLimitsOfTheFetchedConfig drives a first start: +// the debug file opens with the local config, and the API preflight then +// writes the limit keys. The second logging setup must apply them at once, so +// no second restart is necessary. +func Test_initLogging_appliesTheLimitsOfTheFetchedConfig(t *testing.T) { + debugPath := setupDebugBudgetTest(t) + cdUID = "test-uid" + p := &prog{cfg: &cfg} + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + defaultBudget, _ := logBudgets(false) + assertDebugBudget(t, p, debugPath, defaultBudget) + + oneBackup := 1 + cfg.Service.LogMaxSizeMB, cfg.Service.LogMaxBackups = 20, &oneBackup + p.initLogging(false) + + assertDebugBudget(t, p, debugPath, logBudget{maxSize: 20 << 20, backups: 1}) +} + +// Test_applyDebugLogBudget_takesTheLimitsOfARefreshedConfig drives a managed +// config refresh, which applies a new config with no restart. +func Test_applyDebugLogBudget_takesTheLimitsOfARefreshedConfig(t *testing.T) { + debugPath := setupDebugBudgetTest(t) + cdUID = "test-uid" + p := &prog{cfg: &cfg} + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + + noBackups := 0 + refreshed := ctrld.ServiceConfig{LogMaxSizeMB: 5, LogMaxBackups: &noBackups} + p.applyDebugLogBudget(debugLogBudget(&refreshed, false)) + + assertDebugBudget(t, p, debugPath, logBudget{maxSize: 5 << 20, backups: 0}) +} + +// Test_applyDebugLogBudget_takesTheLimitsForLogPath checks the log_path file, +// which follows the same two keys. ctrld does not prune its backups, because +// log_path can name a directory that holds the files of other programs. +func Test_applyDebugLogBudget_takesTheLimitsForLogPath(t *testing.T) { + setupDebugBudgetTest(t) + logPath := filepath.Join(t.TempDir(), "operator.log") + cfg.Service.LogPath = logPath + p := &prog{cfg: &cfg} + p.initLogging(true) + + want := logBudget{maxSize: 3 << 20, backups: 2} + p.applyDebugLogBudget(want) + + rf := logPathFile.Load() + rf.mu.Lock() + got := rf.budget + rf.mu.Unlock() + if got != want { + t.Fatalf("log_path budget = %+v, want %+v", got, want) + } +} + +// Test_initLogging_reportsTheRotationOfTheStartHeader drives a restart that +// finds the journal close to its limit. The header of the new run rotates the +// file, and the rotation event must reach the files, because support reads it +// to learn where the history of the last run went. +func Test_initLogging_reportsTheRotationOfTheStartHeader(t *testing.T) { + dir := setupLogSetupTest(t) + cdUID = "test-uid" + _, journalBudget := logBudgets(false) + journalPath := filepath.Join(dir, journalLogFileName) + oldRun := strings.Repeat("o", int(journalBudget.maxSize)-10) + "\n" + if err := os.WriteFile(journalPath, []byte(oldRun), 0o600); err != nil { + t.Fatal(err) + } + p := &prog{cfg: &cfg} + + p.initLogging(true) + t.Cleanup(p.closeInternalLogs) + + if backup := rotatingFileTestContent(t, journalPath+".1"); backup != oldRun { + t.Fatalf("the journal backup holds %d bytes, want the %d bytes of the old run", len(backup), len(oldRun)) + } + for _, path := range []string{journalPath, filepath.Join(dir, logFileName)} { + if content := rotatingFileTestContent(t, path); !strings.Contains(content, "Log rotated") { + t.Fatalf("%s holds no rotation event: %s", filepath.Base(path), content) + } + } +} diff --git a/cmd/cli/log_file.go b/cmd/cli/log_file.go index dee26e74..3dda04ee 100644 --- a/cmd/cli/log_file.go +++ b/cmd/cli/log_file.go @@ -134,8 +134,14 @@ func (rf *rotatingFile) writeReporting(p []byte) (int, *logRotation, error) { // after the lock is free, so the rotation event can go back into this file. func (rf *rotatingFile) Write(p []byte) (int, error) { n, rotated, err := rf.writeReporting(p) + rf.reportRotation(rotated) + return n, err +} + +// reportRotation calls onRotate for a rotation. It must run with rf.mu free. +func (rf *rotatingFile) reportRotation(rotated *logRotation) { if rotated == nil || !rf.rotatedInFlight.CompareAndSwap(false, true) { - return n, err + return } defer rf.rotatedInFlight.Store(false) rf.mu.Lock() @@ -144,7 +150,6 @@ func (rf *rotatingFile) Write(p []byte) (int, error) { if onRotate != nil { onRotate(*rotated) } - return n, err } // setHeader stores a copy of the header bytes for the next file. @@ -158,14 +163,33 @@ func (rf *rotatingFile) setHeader(b []byte) { // writeHeader appends the stored header to the current file, once for each // file that the writer opens. A restart uses it to mark the point where the -// new run continues an old file. +// new run continues an old file. A header that makes the file cross its +// budget goes to a new file, which the rotation starts with the header. func (rf *rotatingFile) writeHeader() error { + rotated, err := rf.writeHeaderReporting() + rf.reportRotation(rotated) + return err +} + +// writeHeaderReporting writes the header and reports a rotation that the header +// caused. It never calls onRotate, because the caller must log the rotation +// outside of the lock, and at start the logger does not reach the files yet. +func (rf *rotatingFile) writeHeaderReporting() (*logRotation, error) { rf.mu.Lock() defer rf.mu.Unlock() - if rf.headerWritten { - return nil + if rf.headerWritten || len(rf.header) == 0 { + return nil, nil } - return rf.writeHeaderLocked() + if !rf.needsRotation(len(rf.header)) { + return nil, rf.writeHeaderLocked() + } + rotated, err := rf.rotateLocked() + if err != nil || rf.headerWritten { + return rotated, err + } + // The file could not move, so the run continues in the old file, and the + // header must still mark the point where the run starts. + return rotated, rf.writeHeaderLocked() } // setBudget takes the disk space of a later config. The file stays open, so a diff --git a/cmd/cli/log_file_test.go b/cmd/cli/log_file_test.go index 428be98e..c5133a49 100644 --- a/cmd/cli/log_file_test.go +++ b/cmd/cli/log_file_test.go @@ -551,3 +551,72 @@ func Test_zeroBackupRotationKeepsTheOpenReader(t *testing.T) { t.Fatalf("the moved file is still on disk: %v", err) } } + +// Test_rotatingFileHeaderStaysInTheBudget drives a restart that finds a file +// close to its limit. The header of the new run must go to a new file, so no +// file grows over its budget. +func Test_rotatingFileHeaderStaysInTheBudget(t *testing.T) { + budget := logBudget{maxSize: 4096, backups: 1} + path := filepath.Join(t.TempDir(), "ctrld-journal.log") + oldRun := strings.Repeat("o", int(budget.maxSize)-10) + "\n" + if err := os.WriteFile(path, []byte(oldRun), 0o600); err != nil { + t.Fatal(err) + } + rf, err := newRotatingFile(path, budget, nil) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + var rotations []logRotation + rf.onRotate = func(record logRotation) { rotations = append(rotations, record) } + header := []byte(`{"level":"info","journal":true,"message":"Log header"}` + "\n") + rf.setHeader(header) + + if err := rf.writeHeader(); err != nil { + t.Fatalf("writeHeader: %v", err) + } + + if backup := rotatingFileTestContent(t, path+".1"); backup != oldRun { + t.Fatalf("the backup holds %d bytes, want the %d bytes of the old run only", len(backup), len(oldRun)) + } + if current := rotatingFileTestContent(t, path); current != string(header) { + t.Fatalf("the current file holds %q, want the header only", current) + } + if len(rotations) != 1 || rotations[0].bytesWritten != int64(len(oldRun)) { + t.Fatalf("rotations = %+v, want one event for the old file", rotations) + } +} + +// Test_rotatingFileHeaderAfterAFailedRotation covers a restart whose rotation +// cannot move the old file. The run continues in that file, and the header +// must still mark the point where the run starts. +func Test_rotatingFileHeaderAfterAFailedRotation(t *testing.T) { + renameErr := errors.New("rename refused") + budget := logBudget{maxSize: 4096, backups: 1} + path := filepath.Join(t.TempDir(), "ctrld-journal.log") + oldRun := strings.Repeat("o", int(budget.maxSize)-10) + "\n" + if err := os.WriteFile(path, []byte(oldRun), 0o600); err != nil { + t.Fatal(err) + } + rf, err := newRotatingFile(path, budget, nil) + if err != nil { + t.Fatalf("newRotatingFile: %v", err) + } + t.Cleanup(func() { _ = rf.close() }) + rf.rename = func(string, string) error { return renameErr } + var rotations []logRotation + rf.onRotate = func(record logRotation) { rotations = append(rotations, record) } + header := []byte(`{"level":"info","journal":true,"message":"Log header"}` + "\n") + rf.setHeader(header) + + if err := rf.writeHeader(); err != nil { + t.Fatalf("writeHeader: %v", err) + } + + if current := rotatingFileTestContent(t, path); current != oldRun+string(header) { + t.Fatalf("the current file holds %d bytes, want the old run and one header", len(current)) + } + if len(rotations) != 1 || !errors.Is(rotations[0].err, renameErr) { + t.Fatalf("rotations = %+v, want one failed rotation", rotations) + } +} diff --git a/cmd/cli/log_header.go b/cmd/cli/log_header.go index 317da3d1..015e5612 100644 --- a/cmd/cli/log_header.go +++ b/cmd/cli/log_header.go @@ -159,13 +159,20 @@ func (p *prog) refreshLogHeader() { // writeLogHeaders puts the header at the append point of each open file. A // header that cannot land leaves the file usable, so the error only earns a -// warning. -func (p *prog) writeLogHeaders() { +// warning. It returns the rotations that the headers caused, because the +// logger does not reach the files yet and cannot report them. +func (p *prog) writeLogHeaders() []logRotation { + var rotations []logRotation for _, rf := range p.openLogFiles() { - if err := rf.writeHeader(); err != nil { + rotated, err := rf.writeHeaderReporting() + if rotated != nil { + rotations = append(rotations, *rotated) + } + if err != nil { mainLog.Load().Warn().Err(err).Msg("could not write log header") } } + return rotations } func prefixStrings(prefixes []netip.Prefix) []string { diff --git a/cmd/cli/log_reader_test.go b/cmd/cli/log_reader_test.go index 369401a4..db49e237 100644 --- a/cmd/cli/log_reader_test.go +++ b/cmd/cli/log_reader_test.go @@ -570,3 +570,78 @@ func Test_boundedLogFilesFixTheSizeAtSelection(t *testing.T) { t.Fatalf("upload = %q (%d bytes), want %q within the budget of 4", got, upload.size, "one\n") } } + +// seedFullLogChain writes a full chain of files at the desktop budget: the +// current file and every backup, each file at its size limit. Every line names +// its file, so a reader can tell which file a part comes from. +func seedFullLogChain(t *testing.T, name string, budget logBudget) { + t.Helper() + path := absHomeDir(name) + for index := budget.backups; index >= 0; index-- { + filePath := path + if index > 0 { + filePath = fmt.Sprintf("%s.%d", path, index) + } + line := fmt.Sprintf("%s file=%d ", name, index) + line += strings.Repeat("x", rotatingFileTestLineSize-len(line)-1) + "\n" + content := bytes.Repeat([]byte(line), int(budget.maxSize)/len(line)) + if err := os.WriteFile(filePath, content, 0o600); err != nil { + t.Fatal(err) + } + } +} + +// Test_logReader_desktopBudgetUpload uses the desktop budgets, not the small +// test budgets: 50 MB of debug files and a full journal chain. The normal +// upload must stay under 17 MB, and the full upload must hold every file. +func Test_logReader_desktopBudgetUpload(t *testing.T) { + if testing.Short() { + t.Skip("writes 56 MB of log files") + } + origSilent, origCdUID, origHomedir := silent, cdUID, homedir + t.Cleanup(func() { silent, cdUID, homedir = origSilent, origCdUID, origHomedir }) + captureMainLog(t) + setLogHeaderTestGlobalLevel(t, zerolog.DebugLevel) + stubLogHeaderNetworkRead(t, "en9") + silent, cdUID, homedir = false, "test-uid", t.TempDir() + debugBudget, journalBudget := logBudgets(false) + seedFullLogChain(t, logFileName, debugBudget) + seedFullLogChain(t, journalLogFileName, journalBudget) + + p := &prog{cfg: &ctrld.Config{}} + p.internalLogWriter = newLogWriterWithSize(logWriterSize) + p.internalJournalWriter = newLogWriterWithSize(logWriterSmallSize) + for _, stream := range []struct { + lw *logWriter + name string + budget logBudget + }{{p.internalLogWriter, logFileName, debugBudget}, {p.internalJournalWriter, journalLogFileName, journalBudget}} { + if err := stream.lw.setLogFile(absHomeDir(stream.name), stream.budget); err != nil { + t.Fatalf("setLogFile %s: %v", stream.name, err) + } + t.Cleanup(stream.lw.closeLogFile) + } + wholeDebug := logFilesContent(t, p.internalLogWriter.rotating().paths()) + wholeJournal := logFilesContent(t, p.internalJournalWriter.rotating().paths()) + if len(wholeDebug) < 50_000_000 { + t.Fatalf("the debug files hold %d bytes, want at least 50 MB", len(wholeDebug)) + } + + start := time.Now() + upload := readLogReader(t, p, false) + if len(upload) >= 17_000_000 { + t.Fatalf("normal upload = %d bytes, want less than 17 MB", len(upload)) + } + debugPart, journalPart := splitUpload(t, upload, start) + if int64(len(debugPart)) > logSendDebugBudget || !bytes.HasSuffix(wholeDebug, debugPart) { + t.Fatalf("debug part = %d bytes, want the newest bytes within %d", len(debugPart), logSendDebugBudget) + } + if !bytes.Equal(journalPart, wholeJournal) { + t.Fatalf("journal part = %d bytes, want the %d bytes of the journal chain", len(journalPart), len(wholeJournal)) + } + + fullDebug, fullJournal := splitUpload(t, readLogReader(t, p, true), start) + if !bytes.Equal(fullDebug, wholeDebug) || !bytes.Equal(fullJournal, wholeJournal) { + t.Fatalf("full upload = %d debug and %d journal bytes, want %d and %d", len(fullDebug), len(fullJournal), len(wholeDebug), len(wholeJournal)) + } +} diff --git a/cmd/cli/log_writer.go b/cmd/cli/log_writer.go index 0c66fda6..68a9bddb 100644 --- a/cmd/cli/log_writer.go +++ b/cmd/cli/log_writer.go @@ -347,6 +347,7 @@ func (p *prog) initInternalLogging(writers []io.Writer) { return } headersWritten := false + var headerRotations []logRotation p.initInternalLogWriterOnce.Do(func() { mainLog.Load().Notice().Msg("internal logging enabled") p.internalLogWriter = newLogWriter() @@ -356,7 +357,7 @@ func (p *prog) initInternalLogging(writers []io.Writer) { // A restart appends to the file it finds, so the header marks the // point where this run starts. p.refreshLogHeader() - p.writeLogHeaders() + headerRotations = p.writeLogHeaders() headersWritten = true }) lw, jlw := p.internalWriters() @@ -372,14 +373,37 @@ func (p *prog) initInternalLogging(writers []io.Writer) { l := mainLog.Load().Output(multi).With().Logger() mainLog.Store(&l) ctrld.ProxyLogger.Store(&l) + // A header that crossed the budget rotated its file before the logger + // reached the files, so its event waits for this logger. + for _, rotated := range headerRotations { + emitLogRotated(rotated) + } if headersWritten { return } // A later call reaches a changed config, so the stored bytes need the new // values. The files keep the header they already hold. + p.applyDebugLogBudget(debugLogBudget(&p.cfg.Service, router.Name() != "")) p.refreshLogHeader() } +// applyDebugLogBudget gives the open debug files the limits of the config that +// runs now. The files open before the API config arrives, and a managed config +// refresh can change the limits with no restart. Only the internal file gets a +// prune: ctrld owns its name, while log_path can name a file in a directory +// that holds the files of other programs. +func (p *prog) applyDebugLogBudget(budget logBudget) { + if debugWriter, _ := p.internalWriters(); debugWriter != nil { + if rf := debugWriter.rotating(); rf != nil { + rf.setBudget(budget) + pruneNumberedBackups(rf.currentPath(), budget.backups) + } + } + if rf := logPathFile.Load(); rf != nil { + rf.setBudget(budget) + } +} + // wrapConsoleWritersAtNotice holds the console at notice level while the files // take every level. A run without -v needs the detail for troubleshooting, and // the same detail on the console is noise. diff --git a/cmd/cli/network_events_test.go b/cmd/cli/network_events_test.go index c8f47c1c..b490280c 100644 --- a/cmd/cli/network_events_test.go +++ b/cmd/cli/network_events_test.go @@ -5,6 +5,7 @@ import ( "net" "net/netip" "slices" + "strings" "testing" "time" @@ -260,11 +261,21 @@ func wantNoField(t *testing.T, event map[string]any, field string) { // noiseDeltaStates returns a delta that moves the AirDrop link-local address. func noiseDeltaStates(t *testing.T, step int) (before, after *netmon.State) { t.Helper() - airdrop := func(address int) deltaTestInterface { - return deltaTestInterface{name: "awdl0", up: true, mtu: 1484, ips: []string{fmt.Sprintf("fe80::%d/64", address)}} + return noiseDeltaStatesFor(t, step, "awdl0") +} + +// noiseDeltaStatesFor returns a delta that moves the link-local address of +// each named noise interface. +func noiseDeltaStatesFor(t *testing.T, step int, names ...string) (before, after *netmon.State) { + t.Helper() + noise := func(address int) []deltaTestInterface { + interfaces := []deltaTestInterface{networkEventsHardwareIface} + for _, name := range names { + interfaces = append(interfaces, deltaTestInterface{name: name, up: true, mtu: 1484, ips: []string{fmt.Sprintf("fe80::%d/64", address)}}) + } + return interfaces } - return deltaTestState(t, "en0", networkEventsHardwareIface, airdrop(step)), - deltaTestState(t, "en0", networkEventsHardwareIface, airdrop(step+1)) + return deltaTestState(t, "en0", noise(step)...), deltaTestState(t, "en0", noise(step+1)...) } func wantStringsField(t *testing.T, event map[string]any, field string, want ...string) { @@ -286,22 +297,29 @@ func wantStringsField(t *testing.T, event map[string]any, field string, want ... } } +// Test_networkEventsNoiseDeltaSkipsTheHandler covers the AirDrop link, the +// low-latency Wi-Fi link, and both in one callback, as an AirDrop transfer +// gives them. func Test_networkEventsNoiseDeltaSkipsTheHandler(t *testing.T) { - h := newNetworkEventsHarness(t, "en0") + for _, names := range [][]string{{"awdl0"}, {"llw0"}, {"awdl0", "llw0"}} { + t.Run(strings.Join(names, "+"), func(t *testing.T) { + h := newNetworkEventsHarness(t, "en0") - before, after := noiseDeltaStates(t, 1) - h.handle(before, after, false) + before, after := noiseDeltaStatesFor(t, 1, names...) + h.handle(before, after, false) - if h.reconciled != 0 || h.ignored != 0 { - t.Fatalf("noise delta ran the handler: reconciled=%d ignored=%d, want 0/0", h.reconciled, h.ignored) - } - debugLines := h.eventsAtLevel(t, noiseDeltaMessage, "debug") - if len(debugLines) != 1 { - t.Fatalf("got %d debug %q lines, want 1", len(debugLines), noiseDeltaMessage) - } - wantStringsField(t, debugLines[0], "interfaces", "awdl0") - if events := jsonLogEvents(t, h.logs, networkTransitionMessage); len(events) != 0 { - t.Fatalf("got %d %q events for a noise delta, want 0", len(events), networkTransitionMessage) + if h.reconciled != 0 || h.ignored != 0 { + t.Fatalf("noise delta ran the handler: reconciled=%d ignored=%d, want 0/0", h.reconciled, h.ignored) + } + debugLines := h.eventsAtLevel(t, noiseDeltaMessage, "debug") + if len(debugLines) != 1 { + t.Fatalf("got %d debug %q lines, want 1", len(debugLines), noiseDeltaMessage) + } + wantStringsField(t, debugLines[0], "interfaces", names...) + if events := jsonLogEvents(t, h.logs, networkTransitionMessage); len(events) != 0 { + t.Fatalf("got %d %q events for a noise delta, want 0", len(events), networkTransitionMessage) + } + }) } } diff --git a/cmd/cli/network_snapshot.go b/cmd/cli/network_snapshot.go index 9af1c022..e52d2ab5 100644 --- a/cmd/cli/network_snapshot.go +++ b/cmd/cli/network_snapshot.go @@ -116,19 +116,36 @@ func buildNetworkSnapshot(in snapshotInputs) networkSnapshot { var routeAddresses []netip.Prefix if in.State != nil { meta := interfaceMetaWithClass(in.Meta) - routeMeta = meta(in.State.DefaultRouteInterface) - routeAddresses = in.State.InterfaceIPs[in.State.DefaultRouteInterface] snapshot.HaveV4, snapshot.HaveV6 = in.State.HaveV4, in.State.HaveV6 snapshot.Interfaces, snapshot.InterfacesOmitted = snapshotInterfaces(in.State, meta) - if snapshot.DefaultRouteV4 == "" { + // Only a platform that reads no route table takes the route of the + // network monitor. On macOS an empty v4 route is a fact, and the + // monitor can still hold the interface of the last network. + if in.RouteV4.Interface == "" && in.RouteV6.Interface == "" { snapshot.DefaultRouteV4 = defaultRouteV4Interface(in.State) } + linkInterface := snapshotLinkInterface(in) + routeMeta = meta(linkInterface) + routeAddresses = in.State.InterfaceIPs[linkInterface] } snapshot.LinkType = linkTypeFor(routeMeta.HardwarePort, routeMeta.Class) snapshot.Tethered = tetheredNetwork(in.RouteV4.Gateway, routeMeta.HardwarePort, routeAddresses) return snapshot } +// snapshotLinkInterface names the interface that the link fields describe. +// The route table wins over the network monitor: during a switch the monitor +// can still hold the old interface, and one record must not mix the routes of +// one moment with the link type of another. It must run with a state. +func snapshotLinkInterface(in snapshotInputs) string { + for _, name := range []string{in.RouteV4.Interface, in.RouteV6.Interface} { + if name != "" { + return name + } + } + return in.State.DefaultRouteInterface +} + // interfaceMetaWithClass fills the class that the platform metadata leaves // empty. A host that names no port still tells a tunnel from a hardware port // by the interface name. diff --git a/cmd/cli/network_snapshot_test.go b/cmd/cli/network_snapshot_test.go index f1276ddd..ed2b7c51 100644 --- a/cmd/cli/network_snapshot_test.go +++ b/cmd/cli/network_snapshot_test.go @@ -558,3 +558,40 @@ func sortedKeys(object map[string]any) []string { slices.Sort(keys) return keys } + +// Test_buildNetworkSnapshot_linkFieldsFollowTheRouteTable covers a network +// switch. The network monitor still holds the old Ethernet interface, while the +// route table already selects Wi-Fi. The link fields must describe the +// interface of the routes in the same record. +func Test_buildNetworkSnapshot_linkFieldsFollowTheRouteTable(t *testing.T) { + meta := snapshotTestMeta(map[string]interfaceMeta{ + "en0": {Class: "hardware", HardwarePort: "Ethernet"}, + "en1": {Class: "hardware", HardwarePort: "Wi-Fi"}, + }) + for _, tt := range []struct { + name string + routeV4 defaultRoute + routeV6 defaultRoute + }{ + {name: "dual_stack", routeV4: defaultRoute{Interface: "en1"}, routeV6: defaultRoute{Interface: "en1"}}, + {name: "v6_only", routeV6: defaultRoute{Interface: "en1"}}, + } { + t.Run(tt.name, func(t *testing.T) { + state := snapshotTestState("en0", + snapshotTestInterface{"en0", true, []string{"192.0.2.10/24"}}, + snapshotTestInterface{"en1", true, []string{"172.20.10.2/28"}}) + + snapshot := buildNetworkSnapshot(snapshotInputs{State: state, RouteV4: tt.routeV4, RouteV6: tt.routeV6, Meta: meta}) + + if snapshot.LinkType != "wifi" { + t.Errorf("link_type = %q, want %q for the interface of the route table", snapshot.LinkType, "wifi") + } + if !snapshot.Tethered { + t.Error("tethered = false, want true for the hotspot address of the route interface") + } + if snapshot.DefaultRouteV4 != tt.routeV4.Interface { + t.Errorf("default_route_v4 = %q, want %q from the route table", snapshot.DefaultRouteV4, tt.routeV4.Interface) + } + }) + } +} diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 7957fa21..8bb1f77e 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -573,6 +573,7 @@ func (p *prog) runWait() { p.mu.Unlock() closeReplacedUpstreams(oldUpstreams, newCfg.Upstream) + p.applyDebugLogBudget(debugLogBudget(&newCfg.Service, router.Name() != "")) // The header names the mode, the listeners, and the upstreams, so the // open files take the values of the config that now runs. p.refreshLogHeader() diff --git a/docs/network-recovery-diagnostics.md b/docs/network-recovery-diagnostics.md index 224fa939..906f5159 100644 --- a/docs/network-recovery-diagnostics.md +++ b/docs/network-recovery-diagnostics.md @@ -67,8 +67,10 @@ The `Network snapshot` event and the header line of every log file render the sa The field `default_route_interface` is gone. Read `default_route_v4` and `default_route_v6` in its place. The two routes and the two gateways come from the route table of macOS, one route per address family. A tunnel that owns the default route has no next hop, so its route names the tunnel and its gateway stays empty. Windows and Linux read no route table in this release. They fill `default_route_v4` from the network monitor and leave `default_route_v6`, `gateway_v4`, and `gateway_v6` empty. +On macOS, `default_route_v4` comes from the network monitor only when the route table gives no route for the two address families. When the route table gives an IPv6 route only, `default_route_v4` stays empty. `link_type` is one of `wifi`, `ethernet`, `usb_tether`, `tunnel`, and `unknown`. +`link_type` and `tethered` describe the interface of the IPv4 default route. When no IPv4 route is present, they describe the interface of the IPv6 default route. When the route table gives no route, they describe the default interface of the network monitor. Thus the link fields and the route fields of one record describe the same interface. macOS reads it from the name of the hardware port. Linux reads `wifi`, `ethernet`, `usb_tether`, and `tunnel` from the kernel. Windows reads the description of the adapter in place of a port name. The words Ethernet, LAN, and Thunderbolt win over the tunnel class on every platform, so a VPN adapter with a description such as `SSL VPN Virtual Ethernet Adapter` reads as `ethernet`. Windows gives `tunnel` to the other VPN adapters. The other links are `unknown`. diff --git a/docs/runtime-internal-logging.md b/docs/runtime-internal-logging.md index 26fad074..bf906b6a 100644 --- a/docs/runtime-internal-logging.md +++ b/docs/runtime-internal-logging.md @@ -32,7 +32,7 @@ Rotation is by size. A write that crosses the size limit rotates the file first. Each stream also keeps a memory buffer: 5 MB for the debug stream and 1 MB for the journal. These buffers serve `ctrld log tail` and the fallback path when a file cannot open. The files hold the history. -A restart deletes the memory buffers and keeps the files. The new process appends a header line, which marks the restart point inside each file. +A restart deletes the memory buffers and keeps the files. The new process appends a header line, which marks the restart point inside each file. When the header would make a file larger than its size limit, ctrld rotates the file first, and the header starts the new file. If the file cannot move, the header goes to the end of the current file. After an open error, ctrld opens the file again and appends the header with no rotation, so that file can pass its limit by one header. ## Header and rotation events @@ -83,6 +83,8 @@ Two keys in the `[service]` section size the debug stream: Both keys also apply to the `log_path` file, which rotates the same way. The journal budget is fixed per platform. +A change to the keys takes effect with no restart. This includes the first start, when the keys come from the Control D API, and a managed configuration refresh. When the backup count decreases, ctrld deletes the extra backups of `ctrld.log`. ctrld does not delete backups of the `log_path` file, because that directory can hold files of other programs. + ## Uploads `ctrld log send` uploads the log to Control D. The body holds these parts, in this order: From 9064f0e75834b157d40721b308e5c6ccc984255f Mon Sep 17 00:00:00 2001 From: Alex Paguis Date: Fri, 2 Oct 2026 11:20:27 +0000 Subject: [PATCH 27/29] docs: known issue, --config start stalls DNS ~100 s behind HasIPv6's netmon --- docs/known-issues.md | 49 ++++++++++++++++++++++++++++++++++++++++++++ net.go | 8 ++++++++ 2 files changed, 57 insertions(+) diff --git a/docs/known-issues.md b/docs/known-issues.md index 77e09ce8..9bd4c2ce 100644 --- a/docs/known-issues.md +++ b/docs/known-issues.md @@ -73,6 +73,55 @@ This affects any VPN client that implements `block-outside-dns` via WFP, includi --- +### DNS Stalls for Up to ~100 Seconds After Start, and the Install Rolls Back + +**Issue**: On Windows in `dns` intercept mode, a freshly started ctrld can leave every query unanswered for a minute or more. The start self-check fails in the meantime and rolls the install back (`SERVICE_SELFCHECK_FAILED`, exit 53). This happens with `ctrld start --config ` and also with `ctrld start --cd `. + +**Status**: Fixed on the `v1.0` branch in `c40cc8d7`, not yet released. v1.5.7 is affected. + +**Description**: The first DoH request sets up the upstream transport, and that setup calls `ctrld.HasIPv6()`. In the affected versions `HasIPv6()` also started a Tailscale `netmon` monitor inside its `sync.Once`. Building the monitor's first interface state calls `getPACWindows`, which asks WinHTTP for proxy auto-discovery (WPAD). WPAD discovery resolves names through the Windows DNS Client. NRPT already sends those lookups to ctrld, and ctrld cannot answer until the same `sync.Once` returns. So every DNS request waits behind proxy discovery, and proxy discovery waits on DNS, until WinHTTP's own timeouts expire. Then all pending queries complete at once. + +It is a race between WinHTTP's proxy discovery and ctrld installing its NRPT rule: + +- If discovery finishes before the rule is in place, nothing happens. +- With `--cd`, `HasIPv6()` is first called during the API fetch, only about 75 to 95 ms before the NRPT rule becomes active. That window is too short wherever discovery is slow, so `--cd` installs are affected too. +- With `--config`, nothing calls `HasIPv6()` before the rule, so the stall is near-certain. +- A network where the proxy-discovery lookups are answered quickly outside ctrld avoids it. On a domain member, for example, the AD domain's names go to the domain controller through the auto-added AD split rule. +- How long the stall lasts depends on how long WinHTTP's discovery takes on that network. + +A goroutine dump from an affected build (`ecf7955e`) taken during the stall shows the DoH transport setup blocked in the syscall, with 111 DNS request goroutines parked in `sync.Once` behind it: + +``` +UpstreamConfig.dohTransport → ensureSetupTransport (sync.Once) + → SetupTransport → ctrld.HasIPv6 (sync.Once) + → netmon.New → netmon.GetState → getPACWindows [Windows syscall] +``` + +While the stall lasts, ctrld opens no connection to its upstream, although the network and the upstream both answer normally. Windows' certificate service lookups (`ctldl.windowsupdate.com`, Sectigo/USERTrust OCSP and CRL hosts) also queue in ctrld, but they are a consequence of the stall, not its cause. Warming Windows' certificate state before the start does not prevent it. + +**Fix**: `c40cc8d7` ("put the network state of the host in the log journal") removed the monitor from `HasIPv6()`. The process's one network monitor now reports IPv6 state through `SetIPv6Available`, and `HasIPv6()` only runs a bounded IPv6 check. On master (2.0) the same change arrived with `c5ad04ab`. + +**Workarounds** (affected versions): +1. **Recommended**: Add `--skip_self_checks` so the service is not rolled back. DNS then recovers by itself once WinHTTP's proxy discovery times out, after about 100 seconds in the QA lab. +2. **Alternative**: Retry the install. Whether it stalls depends on how quickly proxy discovery finishes, so a second attempt can succeed. +3. Take a build with `c40cc8d7` once one is released. + +**Reproduction** (ctrld-qa lab, workgroup endpoint on the simulated home network): + +- `config-file-upstream-ready` starts ctrld from a local config and requires Control D resolution within 20 seconds. + - It fails on every build that has the monitor in `HasIPv6()`: 10 of 10 runs, on the v1.5.7 release and tag, `ecf7955e`, and 2.0 commits before `c5ad04ab`. + - It passes on every build without the monitor: 9 of 9 runs, on `c40cc8d7`, the `v1.0` tip `cc849871`, and master from `6dcdddc2`. +- `smoke-lifecycle` is a normal `--cd` install. + - The v1.5.7 release failed its start self-check in 4 of 4 runs, each with the same signature: `HasIPv6()` 75 to 95 ms before the NRPT rule, then no upstream reply until the rollback. + - The `v1.0` tip `cc849871` passed. +- On the domain-joined endpoint, `--cd` installs pass. + +**Affected Versions**: ctrld ≤ v1.5.7 on Windows, `dns` intercept mode. Near-certain with `--config`; with `--cd`, it depends on how quickly WinHTTP proxy discovery finishes on the network. + +**Last Updated**: 09/30/2026 + +--- + ## Contributing to Known Issues If you encounter an issue not listed here, please: diff --git a/net.go b/net.go index 81a75579..22aaba6f 100644 --- a/net.go +++ b/net.go @@ -15,6 +15,14 @@ var ( ) // HasIPv6 reports whether the current network stack has IPv6 available. +// +// Keep the first call cheap and bounded. It runs on the DNS path (upstream +// transport setup) behind a sync.Once, so every query waits for it, and under +// Windows NRPT it can run after DNS already points at ctrld. v1.5.7 started a +// netmon monitor here; its first interface state runs WinHTTP proxy discovery, +// which needs DNS through this same ctrld, and a --config start stalled for +// about 100 s (docs/known-issues.md). Network monitoring belongs to the one +// monitor that reports through SetIPv6Available. func HasIPv6() bool { hasIPv6Once.Do(func() { ProxyLogger.Load().Debug().Msg("checking for IPv6 availability once") From 5978ddfe3597dfdd24c5ada3992344824ed7dd14 Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 5 Oct 2026 17:27:52 +0700 Subject: [PATCH 28/29] cli: add network fallback and explicit-only Internal Domain modes Internal Domains now support three modes. "os" (network default) is unchanged. "resolvers" becomes explicit resolver with network fallback, the default explicit selection: the configured resolvers are tried first, and a timeout, unreachable resolver, SERVFAIL, NXDOMAIN, REFUSED or NOTIMP hands the query to the network resolvers - matching VPN DNS servers, then domain-less VPN DNS servers, then the OS resolver's LAN nameservers. A valid empty answer stays final. "resolvers_only" keeps the previous strict behavior. The OS step uses a new LanOnlyQueryCtx: the OS resolver then asks only its LAN-classified nameservers (private, loopback, link-local, CGNAT) and sends nothing when it has none, so the private name never reaches a public nameserver, whether DHCP supplied it or it is ctrld's own public fallback. LanQueryCtx only dropped the latter. LAN-only queries get their own singleflight and hot-cache key, so they are never answered with an ordinary query's public answer. In fallback mode a configured resolver that the upstream monitor has marked down is skipped instead of waited on, so an endpoint off the organization network reaches the fallback without paying the resolver's 2s timeout on every query. Nothing else marks a generated resolver up again, so skipping one re-checks it in the background, at most once every 30s; an answer marks it up. Strict mode never skips. An absent mode with resolvers gets the fallback mode. An unrecognized mode with resolvers fails closed to explicit resolver only, so a wire value this build does not know never sends the domain to the Control D upstream; without resolvers it is dropped. Switching between the explicit modes is detected on refresh. Generated upstreams are now identified by an UpstreamConfig.InternalDomain marker that records their mode and that no configuration file can set, instead of by the internal_ key prefix. An upstream a local ctrld.toml or custom config defines as internal_foo, or as a copy of a generated one, is an ordinary upstream again: it keeps the OS-resolver catch-all, recovery and query-health grading. The "resolvers_only" wire value is provisional until the API agrees on it; the dashboard does not offer strict mode yet. --- cmd/cli/cli.go | 1 + cmd/cli/dns_proxy.go | 87 +++- cmd/cli/internal_domains.go | 437 +++++++++++++--- cmd/cli/internal_domains_fallback_test.go | 582 ++++++++++++++++++++++ cmd/cli/internal_domains_review_test.go | 13 +- cmd/cli/internal_domains_test.go | 28 +- cmd/cli/prog.go | 5 +- cmd/cli/query_health_prog_test.go | 11 +- cmd/cli/recovery_scope_test.go | 7 +- cmd/cli/upstream_monitor_test.go | 7 +- config.go | 9 +- docs/internal-domains.md | 110 +++- internal/controld/config.go | 20 +- internal/controld/split_dns_test.go | 10 +- resolver.go | 33 ++ resolver_test.go | 74 +++ 16 files changed, 1292 insertions(+), 142 deletions(-) create mode 100644 cmd/cli/internal_domains_fallback_test.go diff --git a/cmd/cli/cli.go b/cmd/cli/cli.go index 1f5374fd..4183518a 100644 --- a/cmd/cli/cli.go +++ b/cmd/cli/cli.go @@ -1058,6 +1058,7 @@ func logInternalDomainsSummary(logger zerolog.Logger, summary internalDomainsSum event.Int("domains", summary.domains). Int("os_resolver", summary.osMode). Int("explicit_resolver", summary.explicit). + Int("explicit_resolver_only", summary.only). Int("resolvers", summary.resolvers). Int("skipped", summary.skipped). Int("preempted", summary.preempted). diff --git a/cmd/cli/dns_proxy.go b/cmd/cli/dns_proxy.go index 87e0b1b8..50a1dd69 100644 --- a/cmd/cli/dns_proxy.go +++ b/cmd/cli/dns_proxy.go @@ -577,11 +577,13 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { // // An Internal Domain with explicit resolvers does not take part. The bypass // exists because general DNS is broken, which says nothing about the - // administrator's own resolvers: they may well be reachable, and sending a - // private name to the network's DNS is the disclosure the explicit - // selection exists to prevent. Such a query continues to the normal flow, - // where it reaches only its configured resolvers and fails if none answer. - if dnsIntercept && p.recoveryBypass.Load() && !internalDomainExplicitUpstreams(req.ufr.upstreams) { + // administrator's own resolvers: they may well be reachable, and they are + // to be tried first. Such a query continues to the normal flow, where it + // reaches its configured resolvers first. In "explicit resolver only" mode + // it fails if none answer; otherwise it falls back to the VPN resolvers and + // the network's LAN nameservers, never to the public nameservers the bypass + // may use. + if dnsIntercept && p.recoveryBypass.Load() && !p.internalDomainExplicitUpstreams(req.ufr.upstreams) { ctrld.Log(ctx, mainLog.Load().Debug(), "Recovery bypass active: forwarding to OS resolver") resolver, err := ctrld.NewResolver(osUpstreamConfig) if err == nil { @@ -713,13 +715,15 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { // // An Internal Domain with explicit resolvers is skipped here: the // administrator named the resolvers for that suffix, and VPN suffixes - // are auto-detected, so handing the query to a VPN DNS server would - // override an explicit selection with a discovered one. + // are auto-detected, so handing the query to a VPN DNS server first would + // override an explicit selection with a discovered one. In fallback mode + // the VPN DNS servers are asked later, once the configured resolvers have + // not resolved the name. // // Internal Domains in OS-resolver mode are not skipped, they ask for the // endpoint's default resolution, which under intercept mode includes // the VPN's own resolver. - if dnsIntercept && p.vpnDNS != nil && len(req.msg.Question) > 0 && !internalDomainExplicitUpstreams(upstreams) { + if dnsIntercept && p.vpnDNS != nil && len(req.msg.Question) > 0 && !p.internalDomainExplicitUpstreams(upstreams) { domain := req.msg.Question[0].Name if vpnServers := p.vpnDNS.UpstreamForDomain(domain); len(vpnServers) > 0 { ctrld.Log(ctx, mainLog.Load().Debug(), "VPN DNS route matched for domain %s, using servers: %v", domain, vpnServers) @@ -871,7 +875,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { // generated Internal Domain upstream is the organization's private // resolver address. Report the classification through the sampler and // keep the address-bearing error at debug. - if isInternalDomainUpstream(upstream) { + if isGeneratedInternalDomainUpstream(upstreamConfig) { ctrld.Log(ctx, mainLog.Load().Debug().Err(err), "failed to resolve query") ctrld.Log(ctx, p.querySampler.event(sampleClassInternalDomain, upstream).Str("failure", internalDomainFailureReason(err)), "failed to resolve query using an Internal Domain resolver") @@ -897,6 +901,12 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { return nil } + // An Internal Domain in explicit-with-network-fallback mode moves on from a + // configured resolver that answers SERVFAIL or NXDOMAIN, as it does from + // one that does not answer. internalNegative keeps the best of those + // answers, so a name nobody resolves gets a real negative answer. + internalFallback := p.internalDomainFallbackUpstreams(upstreams) + var internalNegative *dns.Msg for n, upstreamConfig := range upstreamConfigs { if upstreamConfig == nil { continue @@ -911,6 +921,16 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { ctrld.Log(ctx, logger, "DNS loop detected") continue } + // A fallback-mode Internal Domain resolver that the monitor reports as + // down is not waited on: off the organization network every query + // would otherwise pay its timeout before the network fallback runs. + // Nothing else marks a generated resolver up again, so skipping it + // starts a background re-check that does. + if internalFallback && p.um.isDown(upstreams[n]) { + ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain resolver %s is down, skipping it", upstreams[n]) + p.internalDomainProbes.probe(p.um, upstreams[n], upstreamConfig, req.msg) + continue + } answer := resolve(upstreams[n], upstreamConfig, req.msg) if answer == nil { if serveStaleCache && staleAnswer != nil { @@ -945,6 +965,12 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { ctrld.Log(ctx, mainLog.Load().Debug(), "failover rcode matched, process to next upstream") continue } + if internalFallback && internalDomainFallbackRcode(answer.Rcode) { + ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain resolver %s answered %s, trying next resolver", + upstreams[n], dns.RcodeToString[answer.Rcode]) + internalNegative = betterInternalDomainNegative(internalNegative, answer) + continue + } // set compression, as it is not set by default when unpacking answer.Compress = true @@ -989,15 +1015,40 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { res.upstream = upstreamConfig.Endpoint return res } + if internalFallback { + ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain resolvers did not resolve the query; trying network resolvers") + answer, negative := p.resolveInternalDomainOnNetwork(ctx, req.msg) + if answer != nil { + answer.Compress = true + if cacheable { + p.addCachedResponse(dnscache.NewKey(req.msg, upstreams[0]), answer) + ctrld.Log(ctx, mainLog.Load().Debug(), "add cached response") + } + hostname := "" + if req.ci != nil { + hostname = req.ci.Hostname + } + ctrld.Log(ctx, mainLog.Load().Info(), "REPLY: internal domain network fallback -> %s (%s): %s", req.ufr.srcAddr, hostname, dns.RcodeToString[answer.Rcode]) + res.answer = answer + return res + } + if negative = betterInternalDomainNegative(internalNegative, negative); negative != nil { + ctrld.Log(ctx, mainLog.Load().Debug(), "no resolver resolved the internal domain; returning %s", dns.RcodeToString[negative.Rcode]) + negative.Compress = true + res.answer = negative + return res + } + } ctrld.Log(ctx, p.querySampler.event(sampleClassAllEndpointsFailed, ""), "all %v endpoints failed", journalUpstreamNames(upstreams)) - // An Internal Domain with explicit resolvers is served only by the - // configured resolvers. When all of them are unreachable the query fails - // here: the OS-resolver catch-all below would send a private name to a resolver - // that was not selected for it, and the recovery flow exists for the loss of - // general DNS, not for one unavailable internal server. - if internalDomainExplicitUpstreams(upstreams) { - ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain resolvers unreachable; not falling back") + // An Internal Domain with explicit resolvers never takes the OS-resolver + // catch-all below: it can reach public nameservers, ctrld's own fallback + // included, which no mode selects for a private name. Fallback mode has + // already tried the VPN resolvers and the network's LAN nameservers above. + // The recovery flow exists for the loss of general DNS, not for one + // unavailable internal server, so it is not started either. + if p.internalDomainExplicitUpstreams(upstreams) { + ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain resolvers unreachable; not using the OS resolver catch all") } else if p.leakOnUpstreamFailure() { if p.um.countHealthy(upstreams) == 0 { p.recoveryCancelMu.Lock() @@ -1046,7 +1097,7 @@ func (p *prog) proxy(ctx context.Context, req *proxyRequest) *proxyResponse { // endpoint away from the organization network never reaches them, and that is // not an outage of the query path. func (p *prog) countFailedClientQuery(upstreams []string) { - if internalDomainExplicitUpstreams(upstreams) { + if p.internalDomainExplicitUpstreams(upstreams) { return } p.health.countFailedQuery() @@ -2705,7 +2756,7 @@ func (p *prog) buildRecoveryUpstreams(reason RecoveryReason) map[string]*ctrld.U case RecoveryReasonOSFailure: for k, uc := range p.cfg.Upstream { name := upstreamPrefix + k - if uc != nil && uc.Type != ctrld.ResolverTypeOS && !isInternalDomainUpstream(name) { + if uc != nil && uc.Type != ctrld.ResolverTypeOS && !isGeneratedInternalDomainUpstream(uc) { upstreams[name] = uc } } diff --git a/cmd/cli/internal_domains.go b/cmd/cli/internal_domains.go index 836a7dcf..f53f9827 100644 --- a/cmd/cli/internal_domains.go +++ b/cmd/cli/internal_domains.go @@ -9,8 +9,11 @@ import ( "sort" "strconv" "strings" + "sync" "syscall" + "time" + "github.com/miekg/dns" "github.com/rs/zerolog" "github.com/Control-D-Inc/ctrld" @@ -19,15 +22,25 @@ import ( ) // internalDomainUpstreamPrefix names the upstreams generated for organization -// Internal Domains that select explicit resolvers. The prefix is what tells -// proxy() that a rule is served only by configured resolvers, so it -// must not collide with the numeric keys used for the Control D upstream. +// Internal Domains that select explicit resolvers. It must not collide with +// the numeric key used for the Control D upstream. The key only names the +// upstream: what identifies it as generated, and in which mode, is +// UpstreamConfig.InternalDomain, which no configuration file can set. const internalDomainUpstreamPrefix = "internal_" +// Values of UpstreamConfig.InternalDomain on a generated upstream. +const ( + // internalDomainUpstreamFallback serves an Internal Domain in explicit + // resolver with network fallback mode. + internalDomainUpstreamFallback = "fallback" + // internalDomainUpstreamOnly serves an Internal Domain in explicit + // resolver only mode: its answer or failure is final. + internalDomainUpstreamOnly = "only" +) + // internalDomainUpstreamName is the Name given to every generated Internal -// Domain upstream. Together with the key prefix and the legacy type it is what -// distinguishes an upstream ctrld generated from one an endpoint custom -// configuration happens to name the same way. +// Domain upstream. It is only a display label: what identifies an upstream as +// generated is UpstreamConfig.InternalDomain, which no configuration can set. const internalDomainUpstreamName = "Internal Domain resolver" // internalDomainsSummary counts the outcome of applying Internal Domains. It @@ -37,6 +50,7 @@ type internalDomainsSummary struct { domains int // domains that produced rules osMode int // domains routed to the OS/default resolver explicit int // domains routed to configured resolvers + only int // explicit domains with no network fallback resolvers int // explicit resolver upstreams generated skipped int // entries dropped: unusable domain, no usable resolver, or duplicate preempted int // domains already covered by a higher precedence rule @@ -129,43 +143,83 @@ func internalDomainEndpoint(resolver string) (string, bool) { return net.JoinHostPort(addr.String(), port), true } -// internalDomainMode reports whether entry selects explicit resolvers, and -// whether ctrld understands its mode at all. +// internalDomainResolution is how an Internal Domain is resolved. +type internalDomainResolution int + +const ( + // internalDomainViaOS is the "network default" mode: the endpoint's + // OS/default resolver. + internalDomainViaOS internalDomainResolution = iota + // internalDomainViaResolvers is the configured resolvers first, then the + // active network and VPN resolvers. The default explicit selection. + internalDomainViaResolvers + // internalDomainViaResolversOnly is the configured resolvers and nothing + // else. + internalDomainViaResolversOnly +) + +// internalDomainMode reports how entry is resolved, and whether ctrld +// understands its mode at all. // // Mode is authoritative when the API sends it. "os" ignores any resolver // addresses the entry still carries, so switching a domain back to the OS // resolver cannot be undone by addresses a previous selection left behind; -// "resolvers" stays explicit even when the list it points at turns out to be -// unusable, rather than silently becoming OS resolution. An empty mode is a -// deployment that predates the field, where the list is the only signal. +// the explicit modes stay explicit even when the list they point at turns out +// to be unusable, rather than silently becoming OS resolution. An empty mode +// is a deployment that predates the field, where the list is the only signal, +// and an explicit list gets the default explicit mode. // // An unrecognized mode is not guessed at: routing a private domain by guess is -// the disclosure this feature exists to prevent. -func internalDomainMode(entry controld.SplitDNS) (explicit, known bool) { +// the disclosure this feature exists to prevent. It fails closed instead. With +// resolvers it is resolved as explicit resolver only, through the resolvers the +// administrator chose and nothing else, so a wire value this build does not +// know, such as a renamed strict mode, cannot send the domain to the Control D +// upstream or the network. Without resolvers there is nothing to fail closed +// to, so the entry is reported as unknown and dropped. +func internalDomainMode(entry controld.SplitDNS) (resolution internalDomainResolution, known bool) { switch strings.ToLower(strings.TrimSpace(entry.Mode)) { case controld.SplitDNSModeOS: - return false, true + return internalDomainViaOS, true case controld.SplitDNSModeResolvers: - return true, true + return internalDomainViaResolvers, true + case controld.SplitDNSModeResolversOnly: + return internalDomainViaResolversOnly, true case "": - return len(entry.Resolvers) > 0, true + if len(entry.Resolvers) > 0 { + return internalDomainViaResolvers, true + } + return internalDomainViaOS, true default: - return false, false + if len(entry.Resolvers) > 0 { + return internalDomainViaResolversOnly, true + } + return internalDomainViaOS, false } } -// internalDomainRouting returns the endpoints an entry routes to, or a reason -// it cannot be used. No endpoints and no reason means OS resolution. +// internalDomainModeRecognized reports whether mode is one this build knows, +// as opposed to one internalDomainMode resolves by failing closed. +func internalDomainModeRecognized(mode string) bool { + switch strings.ToLower(strings.TrimSpace(mode)) { + case "", controld.SplitDNSModeOS, controld.SplitDNSModeResolvers, controld.SplitDNSModeResolversOnly: + return true + } + return false +} + +// internalDomainRouting returns how an entry is resolved and the endpoints it +// routes to, or a reason it cannot be used. No endpoints and no reason means +// OS resolution. // // Generation and refresh comparison both go through here, so the two can never // disagree about what an entry means. -func internalDomainRouting(entry controld.SplitDNS) (endpoints []string, reason string) { - explicit, known := internalDomainMode(entry) +func internalDomainRouting(entry controld.SplitDNS) (resolution internalDomainResolution, endpoints []string, reason string) { + resolution, known := internalDomainMode(entry) if !known { - return nil, "unknown_mode" + return resolution, nil, "unknown_mode" } - if !explicit { - return nil, "" + if resolution == internalDomainViaOS { + return resolution, nil, "" } endpoints = make([]string, 0, len(entry.Resolvers)) for _, resolver := range entry.Resolvers { @@ -178,9 +232,9 @@ func internalDomainRouting(entry controld.SplitDNS) (endpoints []string, reason // different one, so the entry is dropped and the domain keeps its normal // path instead. if len(endpoints) == 0 { - return nil, "no_usable_resolver" + return resolution, nil, "no_usable_resolver" } - return endpoints, "" + return resolution, endpoints, "" } // internalDomainRuleExists reports whether the listener policy already routes @@ -241,7 +295,8 @@ func orderInternalDomainsBySpecificity(entries []controld.SplitDNS) []controld.S // resolvers routes to the OS/default resolver by way of an empty upstream list, // the same representation Magic Folder excludes use. An entry with resolvers // gets one generated upstream per address, listed in configured order so the -// later ones act as failover for the first. +// later ones act as failover for the first. Those upstreams carry the mode in +// UpstreamConfig.InternalDomain. // // cfg is rebuilt from the API response on every fetch, so this function is the // only writer of internal_* upstreams: a removed domain leaves nothing behind. @@ -271,12 +326,15 @@ func applyInternalDomains(cfg *ctrld.Config, entries []controld.SplitDNS) intern // written: an entry that cannot be used, or a domain already covered by // a higher precedence rule, must not leave a generated upstream behind // with no rule referring to it. - endpoints, reason := internalDomainRouting(entry) + resolution, endpoints, reason := internalDomainRouting(entry) if reason != "" { mainLog.Load().Debug().Msgf("internal domains: skipping %q: %s", domain, reason) summary.skipped++ continue } + if !internalDomainModeRecognized(entry.Mode) { + mainLog.Load().Debug().Msgf("internal domains: %q has unrecognized mode %q; using its resolvers only", domain, entry.Mode) + } seen[domain] = struct{}{} sources := []string{domain, "*." + domain} @@ -297,14 +355,19 @@ func applyInternalDomains(cfg *ctrld.Config, entries []controld.SplitDNS) intern continue } + marker := internalDomainUpstreamFallback + if resolution == internalDomainViaResolversOnly { + marker = internalDomainUpstreamOnly + } targets := make([]string, 0, len(endpoints)) for _, endpoint := range endpoints { key := internalDomainUpstreamPrefix + strconv.Itoa(summary.resolvers) cfg.Upstream[key] = &ctrld.UpstreamConfig{ - Name: internalDomainUpstreamName, - Type: ctrld.ResolverTypeLegacy, - Endpoint: endpoint, - Timeout: 5000, + Name: internalDomainUpstreamName, + Type: ctrld.ResolverTypeLegacy, + Endpoint: endpoint, + Timeout: 5000, + InternalDomain: marker, } targets = append(targets, upstreamPrefix+key) summary.resolvers++ @@ -323,63 +386,118 @@ func applyInternalDomains(cfg *ctrld.Config, entries []controld.SplitDNS) intern mainLog.Load().Debug().Msgf("internal domains: %q routed to the OS resolver", domain) } else { summary.explicit++ + if resolution == internalDomainViaResolversOnly { + summary.only++ + } mainLog.Load().Debug().Msgf("internal domains: %q routed to %v", domain, targets) } } return summary } -// isGeneratedInternalDomainUpstream reports whether a cfg.Upstream entry is one +// isGeneratedInternalDomainUpstream reports whether uc is an upstream // applyInternalDomains created. // // Generated Internal Domain resolvers are part of the managed Control D // configuration, not upstreams the endpoint operator chose, so eligibility // checks that ask "is this a plain single-upstream Control D install" must not -// count them. The key prefix alone is not proof of that, because an endpoint -// custom configuration replaces the generated config outright and could name an -// upstream anything; the shape this function checks is only produced here. -func isGeneratedInternalDomainUpstream(name string, uc *ctrld.UpstreamConfig) bool { - return uc != nil && - strings.HasPrefix(name, internalDomainUpstreamPrefix) && - uc.Type == ctrld.ResolverTypeLegacy && - uc.Name == internalDomainUpstreamName +// count them. Neither the key nor any other configurable field is proof of +// that: a local config or an endpoint custom configuration can name an upstream +// anything and give it any type and name. Only the InternalDomain marker is, +// because ctrld sets it and no configuration file can. +func isGeneratedInternalDomainUpstream(uc *ctrld.UpstreamConfig) bool { + return uc != nil && uc.InternalDomain != "" } -// isInternalDomainUpstream reports whether an upstream reference was generated -// for an Internal Domain with explicit resolvers. -func isInternalDomainUpstream(upstream string) bool { - return strings.HasPrefix(upstream, upstreamPrefix+internalDomainUpstreamPrefix) +// internalDomainUpstream returns the generated Internal Domain upstream an +// "upstream." reference names, or nil if it names any other upstream. +func (p *prog) internalDomainUpstream(upstream string) *ctrld.UpstreamConfig { + if p.cfg == nil { + return nil + } + uc := p.cfg.Upstream[strings.TrimPrefix(upstream, upstreamPrefix)] + if !isGeneratedInternalDomainUpstream(uc) { + return nil + } + return uc +} + +// isInternalDomainUpstream reports whether an upstream reference names a +// generated Internal Domain resolver, in either explicit mode. +func (p *prog) isInternalDomainUpstream(upstream string) bool { + return p.internalDomainUpstream(upstream) != nil } // internalDomainExplicitUpstreams reports whether every upstream serving a -// query is an configured Internal Domain resolver. +// query is a configured Internal Domain resolver, in either explicit mode. // -// proxy() uses this to keep such a query off the OS-resolver catch-all when the -// configured resolvers are unreachable: the administrator selected those -// resolvers and no other, so a private name must fail rather than be sent -// somewhere it was not meant to go. It also keeps an unreachable internal -// resolver from triggering the endpoint-wide recovery flow, which exists for -// the loss of general DNS, not for one unavailable internal server. -func internalDomainExplicitUpstreams(upstreams []string) bool { +// proxy() uses this to try the configured resolvers before anything else, and +// to keep such a query off the OS-resolver catch-all when they fail: that +// catch-all may reach public nameservers, and a private name must not be sent +// somewhere it was not meant to go. In fallback mode proxy() tries the +// network resolvers instead (see internalDomainFallbackUpstreams). It also +// keeps an unreachable internal resolver from triggering the endpoint-wide +// recovery flow, which exists for the loss of general DNS, not for one +// unavailable internal server. +func (p *prog) internalDomainExplicitUpstreams(upstreams []string) bool { + if len(upstreams) == 0 { + return false + } + for _, upstream := range upstreams { + if !p.isInternalDomainUpstream(upstream) { + return false + } + } + return true +} + +// internalDomainFallbackUpstreams reports whether upstreams are the configured +// resolvers of an Internal Domain in explicit-with-network-fallback mode, whose +// failure, SERVFAIL or NXDOMAIN hands the query to the active network and VPN +// resolvers. +func (p *prog) internalDomainFallbackUpstreams(upstreams []string) bool { if len(upstreams) == 0 { return false } for _, upstream := range upstreams { - if !isInternalDomainUpstream(upstream) { + uc := p.internalDomainUpstream(upstream) + if uc == nil || uc.InternalDomain != internalDomainUpstreamFallback { return false } } return true } +// internalDomainFallbackRcode reports whether a configured resolver's answer +// sends a fallback-mode query on to the next resolver. SERVFAIL means the +// resolver could not answer, and NXDOMAIN may only mean it does not know a +// name the local network's resolver does. REFUSED and NOTIMP mean it will not +// serve this endpoint, as an organization resolver with an access list answers +// an endpoint off the organization network, which is closer to unreachable +// than to an answer. Any other answer is final, including an empty NOERROR: +// the name exists, it just has no record of that type. +func internalDomainFallbackRcode(rcode int) bool { + switch rcode { + case dns.RcodeServerFailure, dns.RcodeNameError, dns.RcodeRefused, dns.RcodeNotImplemented: + return true + } + return false +} + // internalDomainsEqual reports whether two Internal Domains lists would produce // the same routing. Domains are compared in canonical form and independently of // the order the API returned them, so an unchanged list does not trigger a // reload on every refresh. Resolver order is significant: it is the order the -// administrator chose, and it decides which resolver is tried first. +// administrator chose, and it decides which resolver is tried first. The mode +// is significant too, including a switch between the two explicit modes. func internalDomainsEqual(a, b []controld.SplitDNS) bool { - canonical := func(entries []controld.SplitDNS) []controld.SplitDNS { - out := make([]controld.SplitDNS, 0, len(entries)) + type route struct { + domain string + resolution internalDomainResolution + endpoints []string + } + canonical := func(entries []controld.SplitDNS) []route { + out := make([]route, 0, len(entries)) for _, entry := range entries { domain := normalizeInternalDomain(entry.Domain) if domain == "" { @@ -387,13 +505,13 @@ func internalDomainsEqual(a, b []controld.SplitDNS) bool { } // Entries that cannot be used route nothing, so they cannot make // an otherwise unchanged list look changed. - endpoints, reason := internalDomainRouting(entry) + resolution, endpoints, reason := internalDomainRouting(entry) if reason != "" { continue } - out = append(out, controld.SplitDNS{Domain: domain, Resolvers: endpoints}) + out = append(out, route{domain: domain, resolution: resolution, endpoints: endpoints}) } - sort.Slice(out, func(i, j int) bool { return out[i].Domain < out[j].Domain }) + sort.Slice(out, func(i, j int) bool { return out[i].domain < out[j].domain }) return out } x, y := canonical(a), canonical(b) @@ -401,7 +519,7 @@ func internalDomainsEqual(a, b []controld.SplitDNS) bool { return false } for i := range x { - if x[i].Domain != y[i].Domain || !slices.Equal(x[i].Resolvers, y[i].Resolvers) { + if x[i].domain != y[i].domain || x[i].resolution != y[i].resolution || !slices.Equal(x[i].endpoints, y[i].endpoints) { return false } } @@ -429,12 +547,6 @@ func internalDomainFailureReason(err error) string { return "error" } -// isGeneratedInternalDomainUpstreamRef is isGeneratedInternalDomainUpstream for -// call sites that hold an "upstream." reference rather than the bare key. -func isGeneratedInternalDomainUpstreamRef(upstream string, uc *ctrld.UpstreamConfig) bool { - return isGeneratedInternalDomainUpstream(strings.TrimPrefix(upstream, upstreamPrefix), uc) -} - // logUpstreamProbeFailure reports a background probe failure against the // upstream that the reference upstream names. // @@ -450,10 +562,199 @@ func logUpstreamProbeFailure(upstream string, uc *ctrld.UpstreamConfig, err erro detail := append([]any{}, args...) mainLog.Load().Debug().Err(err).Msgf(format+" for upstream: %q, endpoint: %q", append(detail, uc.Name, uc.Endpoint)...) event := level().Str("upstream", journalUpstreamName(upstream)) - if !isGeneratedInternalDomainUpstreamRef(upstream, uc) { + if !isGeneratedInternalDomainUpstream(uc) { event.Err(err).Msgf(format, detail...) return } event.Str("failure", internalDomainFailureReason(err)). Msgf(format+" for an Internal Domain resolver", detail...) } + +// betterInternalDomainNegative returns whichever of two negative answers a +// fallback-mode query should end with: NXDOMAIN over SERVFAIL, since a +// resolver that says the name does not exist has answered, and otherwise the +// first one seen. +func betterInternalDomainNegative(current, candidate *dns.Msg) *dns.Msg { + switch { + case candidate == nil: + return current + case current == nil: + return candidate + case candidate.Rcode == dns.RcodeNameError && current.Rcode != dns.RcodeNameError: + return candidate + } + return current +} + +// internalDomainVPNUpstream returns the upstream the network fallback uses for +// a VPN DNS server. A seam: VPN DNS servers are dialed on port 53, which a test +// fixture cannot listen on. +var internalDomainVPNUpstream = func(m *vpnDNSManager, server string) *ctrld.UpstreamConfig { + return m.upstreamConfigFor(server) +} + +// internalDomainOSResolve resolves msg through the OS resolver for the network +// fallback. A seam, so a test sees the context the fallback passes: that +// context is what keeps the private name off public nameservers. +var internalDomainOSResolve = func(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + resolver, err := ctrld.NewResolver(osUpstreamConfig) + if err != nil { + return nil, err + } + resolveCtx, cancel := osUpstreamConfig.Context(ctx) + defer cancel() + return resolver.Resolve(resolveCtx, msg) +} + +// resolveInternalDomainOnNetwork is the network fallback of an Internal Domain +// in explicit-with-network-fallback mode. proxy() runs it once the configured +// resolvers have failed, or answered SERVFAIL or NXDOMAIN. +// +// It asks only resolvers that the endpoint's active network or VPN provides, +// in this order: VPN DNS servers whose domains match the query, then VPN DNS +// servers with no domains (both only in dns-intercept mode, where ctrld tracks +// them), then the OS resolver's LAN nameservers. The OS resolver is asked as a +// LAN-only query, so the private name never reaches a public nameserver, not +// even one DHCP supplied, nor ctrld's own public fallback. With no LAN +// nameserver the OS step sends nothing. +// +// The first NOERROR answer is returned as answer, including an empty one, +// which is final. Otherwise negative is the best SERVFAIL or NXDOMAIN answer +// seen. While Windows is serving retained VPN DNS state, a VPN transport +// failure stops the fallback before the OS resolver, as it does for VPN split +// routing. +func (p *prog) resolveInternalDomainOnNetwork(ctx context.Context, msg *dns.Msg) (answer, negative *dns.Msg) { + domain := msg.Question[0].Name + resolve := func(ctx context.Context, uc *ctrld.UpstreamConfig) (*dns.Msg, error) { + resolver, err := ctrld.NewResolver(uc) + if err != nil { + return nil, err + } + resolveCtx, cancel := uc.Context(ctx) + defer cancel() + return resolver.Resolve(resolveCtx, msg) + } + // accept reports whether a network answer is final, and keeps a negative + // one for the end. + accept := func(source string, m *dns.Msg) bool { + if !sameQuestion(msg, m) { + ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain fallback: discarding answer from %s: question mismatch", source) + return false + } + if m.Rcode == dns.RcodeSuccess { + ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain fallback: %s answered %s", source, domain) + return true + } + ctrld.Log(ctx, mainLog.Load().Debug(), "internal domain fallback: %s answered %s for %s", + source, dns.RcodeToString[m.Rcode], domain) + if internalDomainFallbackRcode(m.Rcode) { + negative = betterInternalDomainNegative(negative, m) + } + return false + } + + if dnsIntercept && p.vpnDNS != nil { + var servers []string + for _, server := range slices.Concat(p.vpnDNS.UpstreamForDomain(domain), p.vpnDNS.DomainlessServers()) { + if !slices.Contains(servers, server) { + servers = append(servers, server) + } + } + var gotAnswer, gotTransportFailure bool + for _, server := range servers { + m, err := resolve(ctx, internalDomainVPNUpstream(p.vpnDNS, server)) + if m == nil { + gotTransportFailure = true + ctrld.Log(ctx, mainLog.Load().Debug().Err(err), "internal domain fallback: VPN DNS server %s failed", server) + continue + } + gotAnswer = true + p.vpnDNS.VPNDNSReachable() + if accept("VPN DNS server "+server, m) { + return m, nil + } + } + if !gotAnswer && gotTransportFailure && p.vpnDNS.ShouldFailClosedAfterVPNDNSTransportFailure(domain, servers) { + return nil, negative + } + } + + m, err := internalDomainOSResolve(ctrld.LanOnlyQueryCtx(ctx), msg) + if m == nil { + ctrld.Log(ctx, mainLog.Load().Debug().Err(err), "internal domain fallback: OS resolver LAN nameservers failed") + return nil, negative + } + if accept("OS resolver", m) { + return m, nil + } + return nil, negative +} + +// internalDomainProbeInterval is how often, at most, a down fallback-mode +// Internal Domain resolver is re-checked in the background. +const internalDomainProbeInterval = 30 * time.Second + +// internalDomainProber re-checks down fallback-mode Internal Domain resolvers +// in the background, and marks one up again when it answers. +// +// proxy() skips such a resolver while it is down, so that an endpoint away +// from the organization network goes to the network fallback without waiting +// on it. A generated resolver comes back up only by answering a query: it is +// excluded from recovery, and the loop checker leaves down upstreams alone. So +// without this re-check, a resolver that went down would stay skipped until +// the next reload, even once the endpoint is back on the organization network. +// +// The zero value is ready to use. +type internalDomainProber struct { + mu sync.Mutex + last map[string]time.Time + + // now and start are seams, so a test controls the clock and runs the + // re-check itself. nil means time.Now and a new goroutine. + now func() time.Time + start func(func()) +} + +// probe re-checks upstream in the background, at most once per +// internalDomainProbeInterval, by sending it a copy of msg. The query goes to +// the organization's own resolver, which it was meant for anyway. Any DNS +// answer proves the resolver reachable, the same as on the query path, and +// marks it up in um; a failure leaves it down until the next re-check. +func (pr *internalDomainProber) probe(um *upstreamMonitor, upstream string, uc *ctrld.UpstreamConfig, msg *dns.Msg) { + now := time.Now + if pr.now != nil { + now = pr.now + } + pr.mu.Lock() + if last, ok := pr.last[upstream]; ok && now().Sub(last) < internalDomainProbeInterval { + pr.mu.Unlock() + return + } + if pr.last == nil { + pr.last = make(map[string]time.Time) + } + pr.last[upstream] = now() + pr.mu.Unlock() + + query := msg.Copy() + run := func() { + var answer *dns.Msg + resolver, err := ctrld.NewResolver(uc) + if err == nil { + ctx, cancel := uc.Context(context.Background()) + answer, err = resolver.Resolve(ctx, query) + cancel() + } + if answer == nil { + mainLog.Load().Debug().Err(err).Msgf("internal domains: %s is still down", upstream) + return + } + mainLog.Load().Debug().Msgf("internal domains: %s answered the re-check", upstream) + um.noteSuccess(upstream) + } + if pr.start != nil { + pr.start(run) + return + } + go run() +} diff --git a/cmd/cli/internal_domains_fallback_test.go b/cmd/cli/internal_domains_fallback_test.go new file mode 100644 index 00000000..bcf32b9c --- /dev/null +++ b/cmd/cli/internal_domains_fallback_test.go @@ -0,0 +1,582 @@ +package cli + +import ( + "context" + "net" + "strings" + "testing" + "time" + + "github.com/miekg/dns" + + "github.com/Control-D-Inc/ctrld" + "github.com/Control-D-Inc/ctrld/internal/controld" +) + +// startRcodeDNSFixture is startDNSFixture answering every question with rcode. +// A NOERROR answer carries an A record unless empty is set, which gives the +// name-exists-but-no-record-of-that-type answer. +func startRcodeDNSFixture(t *testing.T, rcode int, empty bool) *dnsFixture { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + f := &dnsFixture{addr: pc.LocalAddr().String()} + started := make(chan struct{}) + server := &dns.Server{ + PacketConn: pc, + NotifyStartedFunc: func() { close(started) }, + Handler: dns.HandlerFunc(func(w dns.ResponseWriter, msg *dns.Msg) { + f.mu.Lock() + if len(msg.Question) > 0 { + f.qs = append(f.qs, msg.Question[0].Name) + } + f.mu.Unlock() + reply := new(dns.Msg) + reply.SetRcode(msg, rcode) + if rcode == dns.RcodeSuccess && !empty && len(msg.Question) > 0 { + rr, _ := dns.NewRR(msg.Question[0].Name + " 60 IN A 10.9.9.9") + reply.Answer = append(reply.Answer, rr) + } + _ = w.WriteMsg(reply) + }), + } + go func() { _ = server.ActivateAndServe() }() + select { + case <-started: + case <-time.After(5 * time.Second): + t.Fatal("DNS fixture did not start") + } + t.Cleanup(func() { _ = server.Shutdown() }) + return f +} + +func deadLoopbackAddr(t *testing.T) string { + t.Helper() + pc, err := net.ListenPacket("udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := pc.LocalAddr().String() + _ = pc.Close() + return addr +} + +// Explicit resolver with network fallback is the default explicit mode: the +// configured resolver is asked first, and a timeout, an unreachable resolver, +// SERVFAIL or NXDOMAIN hands the query to the network's resolver. A valid +// empty answer is final, and "explicit resolver only" never falls back. +func TestInternalDomainsNetworkFallback(t *testing.T) { + for _, tc := range []struct { + name string + mode string + // internal is the configured resolver's rcode; -1 is unreachable. + internal int + internalEmpty bool + network int + wantNetwork bool + wantRcode int + wantAnswer bool + }{ + {name: "nxdomain falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "servfail falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeServerFailure, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "unreachable falls back", mode: controld.SplitDNSModeResolvers, internal: -1, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "absent mode falls back", mode: "", internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "answer is final", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeSuccess, network: dns.RcodeSuccess, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "empty answer is final", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeSuccess, internalEmpty: true, network: dns.RcodeSuccess, wantRcode: dns.RcodeSuccess}, + {name: "refused falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeRefused, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "notimp falls back", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNotImplemented, network: dns.RcodeSuccess, wantNetwork: true, wantRcode: dns.RcodeSuccess, wantAnswer: true}, + {name: "refused and network nxdomain", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeRefused, network: dns.RcodeNameError, wantNetwork: true, wantRcode: dns.RcodeNameError}, + {name: "formerr is final", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeFormatError, network: dns.RcodeSuccess, wantRcode: dns.RcodeFormatError}, + {name: "only mode keeps refused", mode: controld.SplitDNSModeResolversOnly, internal: dns.RcodeRefused, network: dns.RcodeSuccess, wantRcode: dns.RcodeRefused}, + {name: "unknown mode fails closed", mode: "explicit_only", internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantRcode: dns.RcodeNameError}, + {name: "nobody knows the name", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNameError, network: dns.RcodeNameError, wantNetwork: true, wantRcode: dns.RcodeNameError}, + {name: "configured nxdomain beats network servfail", mode: controld.SplitDNSModeResolvers, internal: dns.RcodeNameError, network: dns.RcodeServerFailure, wantNetwork: true, wantRcode: dns.RcodeNameError}, + {name: "unreachable and network nxdomain", mode: controld.SplitDNSModeResolvers, internal: -1, network: dns.RcodeNameError, wantNetwork: true, wantRcode: dns.RcodeNameError}, + {name: "only mode keeps nxdomain", mode: controld.SplitDNSModeResolversOnly, internal: dns.RcodeNameError, network: dns.RcodeSuccess, wantRcode: dns.RcodeNameError}, + {name: "only mode keeps servfail", mode: controld.SplitDNSModeResolversOnly, internal: dns.RcodeServerFailure, network: dns.RcodeSuccess, wantRcode: dns.RcodeServerFailure}, + {name: "only mode fails when unreachable", mode: controld.SplitDNSModeResolversOnly, internal: -1, network: dns.RcodeSuccess, wantRcode: dns.RcodeServerFailure}, + } { + t.Run(tc.name, func(t *testing.T) { + network := startRcodeDNSFixture(t, tc.network, false) + useOSResolverFixture(t, network) + + var internal *dnsFixture + resolver := deadLoopbackAddr(t) + if tc.internal >= 0 { + internal = startRcodeDNSFixture(t, tc.internal, tc.internalEmpty) + resolver = internal.addr + } + + cfg := internalDomainsTestConfig() + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true) + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: tc.mode, Resolvers: []string{resolver}}}) + p := newInternalDomainsProg(t, cfg) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + + if internal != nil { + if got := internal.questions(); len(got) != 1 { + t.Errorf("configured resolver questions = %v, want it asked first", got) + } + } + if got := network.questions(); (len(got) > 0) != tc.wantNetwork { + t.Errorf("network resolver questions = %v, want asked = %v", got, tc.wantNetwork) + } + if res == nil || res.answer == nil { + t.Fatalf("no answer") + } + if res.answer.Rcode != tc.wantRcode { + t.Errorf("rcode = %s, want %s", dns.RcodeToString[res.answer.Rcode], dns.RcodeToString[tc.wantRcode]) + } + if got := len(res.answer.Answer) > 0; got != tc.wantAnswer { + t.Errorf("answer records = %v, want records = %v", res.answer.Answer, tc.wantAnswer) + } + }) + } +} + +// The configured resolvers are all tried, in order, before the network: an +// NXDOMAIN from the first does not skip the second. +func TestInternalDomainsNetworkFallbackTriesEveryConfiguredResolverFirst(t *testing.T) { + network := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + useOSResolverFixture(t, network) + first := startRcodeDNSFixture(t, dns.RcodeNameError, false) + second := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{first.addr, second.addr}}}) + p := newInternalDomainsProg(t, cfg) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + + if len(first.questions()) != 1 || len(second.questions()) != 1 { + t.Errorf("configured resolvers asked %v, %v; want both once", first.questions(), second.questions()) + } + if got := network.questions(); len(got) != 0 { + t.Errorf("network resolver asked %v although a configured resolver answered", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Errorf("answer = %v, want the second resolver's NOERROR", res) + } +} + +// The fallback is no reason to log the private resolver address above debug, +// or the domain: the summary-only REPLY line carries neither. +func TestInternalDomainsNetworkFallbackLogsNoPrivateValues(t *testing.T) { + network := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + useOSResolverFixture(t, network) + internal := startRcodeDNSFixture(t, dns.RcodeNameError, false) + + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}}) + p := newInternalDomainsProg(t, cfg) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + before := len(logOutput.String()) + p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + out := logOutput.String()[before:] + + if !strings.Contains(out, "internal domain network fallback") { + t.Fatalf("expected the fallback REPLY line, got: %s", out) + } + for _, line := range strings.Split(strings.TrimSpace(out), "\n") { + if line == "" || strings.Contains(line, `"level":"debug"`) { + continue + } + if strings.Contains(line, internal.addr) || strings.Contains(line, "corp.example") { + t.Errorf("private value disclosed above debug level: %s", line) + } + } +} + +// internalDomainUpstreamsProg returns a prog whose config has one legacy +// upstream per key, each carrying the given InternalDomain marker. +func internalDomainUpstreamsProg(markers map[string]string) *prog { + cfg := &ctrld.Config{Upstream: make(map[string]*ctrld.UpstreamConfig, len(markers))} + for key, marker := range markers { + cfg.Upstream[key] = &ctrld.UpstreamConfig{ + Name: internalDomainUpstreamName, + Type: ctrld.ResolverTypeLegacy, + Endpoint: "10.0.0.53:53", + InternalDomain: marker, + } + } + return &prog{cfg: cfg} +} + +func TestInternalDomainFallbackUpstreams(t *testing.T) { + p := internalDomainUpstreamsProg(map[string]string{ + "internal_0": internalDomainUpstreamFallback, + "internal_1": internalDomainUpstreamFallback, + "internal_2": internalDomainUpstreamOnly, + "0": "", + }) + for _, tc := range []struct { + in []string + want bool + }{ + {in: nil, want: false}, + {in: []string{"upstream.internal_0", "upstream.internal_1"}, want: true}, + {in: []string{"upstream.internal_2"}, want: false}, + {in: []string{"upstream.internal_0", "upstream.internal_2"}, want: false}, + {in: []string{"upstream.internal_0", "upstream.0"}, want: false}, + {in: []string{upstreamOS}, want: false}, + } { + if got := p.internalDomainFallbackUpstreams(tc.in); got != tc.want { + t.Errorf("internalDomainFallbackUpstreams(%v) = %v, want %v", tc.in, got, tc.want) + } + } + if !p.internalDomainExplicitUpstreams([]string{"upstream.internal_2"}) { + t.Error("an explicit-only upstream must still count as an Internal Domain resolver") + } +} + +// Both explicit modes generate internal_ upstreams; the mode is carried by +// the InternalDomain marker. Switching between the modes is a routing change +// the next refresh must apply. +func TestInternalDomainsExplicitModes(t *testing.T) { + cfg := internalDomainsTestConfig() + summary := applyInternalDomains(cfg, []controld.SplitDNS{ + {Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}, + {Domain: "b.example.com", Mode: " Resolvers_Only ", Resolvers: []string{"10.0.0.54"}}, + }) + if summary.explicit != 2 || summary.only != 1 || summary.resolvers != 2 { + t.Fatalf("summary = %+v", summary) + } + for _, tc := range []struct{ domain, target, marker string }{ + {"a.example.com", "upstream.internal_0", internalDomainUpstreamFallback}, + {"b.example.com", "upstream.internal_1", internalDomainUpstreamOnly}, + } { + if targets, _ := ruleTargets(t, cfg, tc.domain); strings.Join(targets, ",") != tc.target { + t.Errorf("%s targets = %v, want %s", tc.domain, targets, tc.target) + } + uc := cfg.Upstream[strings.TrimPrefix(tc.target, upstreamPrefix)] + if uc == nil || uc.InternalDomain != tc.marker { + t.Errorf("%s upstream = %+v, want marker %q", tc.domain, uc, tc.marker) + } + } + + fallback := []controld.SplitDNS{{Domain: "a.example.com", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{"10.0.0.53"}}} + only := []controld.SplitDNS{{Domain: "a.example.com", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{"10.0.0.53"}}} + legacy := []controld.SplitDNS{{Domain: "a.example.com", Resolvers: []string{"10.0.0.53"}}} + if internalDomainsEqual(fallback, only) { + t.Error("switching to explicit resolver only must be detected as a change") + } + if !internalDomainsEqual(fallback, legacy) { + t.Error("an absent mode with resolvers is the default explicit mode") + } +} + +// An upstream a configuration defines is an ordinary upstream, whatever it is +// named and however closely it copies a generated one: a local ctrld.toml that +// names an upstream internal_, gives it the legacy type and the generated +// name, must still get the OS-resolver catch-all when it fails. +func TestInternalDomainsUserDefinedLookalikeIsOrdinaryUpstream(t *testing.T) { + osFixture := startDNSFixture(t) + useOSResolverFixture(t, osFixture) + + cfg := internalDomainsTestConfig() + cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true) + cfg.Upstream["internal_0"] = &ctrld.UpstreamConfig{ + Name: internalDomainUpstreamName, + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadLoopbackAddr(t), + Timeout: 500, + } + cfg.Listener["0"].Policy.Rules = append(cfg.Listener["0"].Policy.Rules, + ctrld.Rule{"*.corp.example": []string{"upstream.internal_0"}}) + p := newInternalDomainsProg(t, cfg) + if p.isInternalDomainUpstream("upstream.internal_0") || isGeneratedInternalDomainUpstream(cfg.Upstream["internal_0"]) { + t.Fatal("a user-defined upstream was recognized as a generated Internal Domain resolver") + } + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + + if got := osFixture.questions(); len(got) != 1 { + t.Errorf("OS resolver questions = %v, want the ordinary catch all", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Errorf("answer = %v, want the OS resolver's NOERROR", res) + } +} + +// captureInternalDomainProbes makes p's background re-checks run only when the +// test calls them, and returns the ones scheduled so far. +func captureInternalDomainProbes(p *prog) *[]func() { + var probes []func() + p.internalDomainProbes.start = func(run func()) { probes = append(probes, run) } + return &probes +} + +// The off-site case that fallback mode exists for: the organization resolver +// is unreachable and the network answers. Once the monitor reports the +// resolver down, a query must go to the network without waiting on the +// resolver's timeout, and a background re-check must bring it back once it +// answers again. +func TestInternalDomainsDownResolverIsSkippedAndReChecked(t *testing.T) { + network := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + useOSResolverFixture(t, network) + blackhole := blackholeUDPAddr(t) + + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolvers, Resolvers: []string{blackhole}}}) + p := newInternalDomainsProg(t, cfg) + probes := captureInternalDomainProbes(p) + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ask := func() (*proxyResponse, time.Duration) { + t.Helper() + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + start := time.Now() + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + return res, time.Since(start) + } + + const upstream = upstreamPrefix + internalDomainUpstreamPrefix + "0" + p.um.mu.Lock() + p.um.markDown(upstream, maxFailureRequest, "immediate") + p.um.mu.Unlock() + + for i := 0; i < 2; i++ { + res, elapsed := ask() + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Fatalf("query %d: answer = %v, want the network's NOERROR", i+1, res) + } + // The blackholed resolver would cost its 2s read timeout. + if elapsed > time.Second { + t.Fatalf("query %d took %v: it waited on a resolver that is down", i+1, elapsed) + } + } + if got := len(*probes); got != 1 { + t.Fatalf("re-checks scheduled = %d, want one per interval", got) + } + + // The resolver is still unreachable: the re-check leaves it down. + (*probes)[0]() + if !p.um.isDown(upstream) { + t.Fatal("a failed re-check marked the resolver up") + } + + // Back on the organization network: the next re-check, once the interval + // has passed, finds it answering and marks it up, so queries use it again. + internal := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + cfg.Upstream["internal_0"].Endpoint = internal.addr + later := time.Now().Add(internalDomainProbeInterval) + p.internalDomainProbes.now = func() time.Time { return later } + ask() + if got := len(*probes); got != 2 { + t.Fatalf("re-checks scheduled = %d, want a second one after the interval", got) + } + (*probes)[1]() + if p.um.isDown(upstream) { + t.Fatal("the resolver answered the re-check but is still down") + } + before := len(network.questions()) + if res, _ := ask(); res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Fatalf("answer = %v, want the resolver's NOERROR", res) + } + if got := internal.questions(); len(got) != 2 { + t.Errorf("resolver questions = %v, want the re-check and the query", got) + } + if got := len(network.questions()); got != before { + t.Errorf("network asked again although the resolver answered") + } +} + +// Explicit resolver only has no fallback to skip to, so a down resolver is +// still asked: skipping it would only turn its answer into a SERVFAIL. +func TestInternalDomainsOnlyModeDoesNotSkipDownResolver(t *testing.T) { + useOSResolverFixture(t, startDNSFixture(t)) + internal := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{internal.addr}}}) + p := newInternalDomainsProg(t, cfg) + probes := captureInternalDomainProbes(p) + + p.um.mu.Lock() + p.um.markDown(upstreamPrefix+internalDomainUpstreamPrefix+"0", maxFailureRequest, "immediate") + p.um.mu.Unlock() + + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + res := p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) + + if got := internal.questions(); len(got) != 1 { + t.Errorf("resolver questions = %v, want it asked although it is down", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Errorf("answer = %v, want the resolver's NOERROR", res) + } + if len(*probes) != 0 { + t.Errorf("explicit resolver only scheduled %d re-checks", len(*probes)) + } +} + +// fallbackTestQuery sends one query for host.corp.example through proxy(). +func fallbackTestQuery(t *testing.T, p *prog, cfg *ctrld.Config) *proxyResponse { + t.Helper() + addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) + ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.corp.example") + return p.proxy(ctx, &proxyRequest{msg: newDnsMsgWithHostname("host.corp.example.", dns.TypeA), ufr: ufr}) +} + +// The OS step of the fallback is what keeps the private name off public +// nameservers, and it does so through the context it passes. A test that +// swaps the OS resolver for a single-endpoint fixture would pass with any +// context, so this one sees the context itself and runs the step through a +// real OS resolver whose pool holds a LAN and a public nameserver. +func TestInternalDomainsNetworkFallbackOSStepIsLanOnly(t *testing.T) { + lan := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + pool := ctrld.NewResolverWithNameserver([]string{lan.addr, "192.0.2.53:53"}) + var lanOnly []bool + prev := internalDomainOSResolve + internalDomainOSResolve = func(ctx context.Context, msg *dns.Msg) (*dns.Msg, error) { + isLanOnly, _ := ctx.Value(ctrld.LanOnlyQueryCtxKey{}).(bool) + lanOnly = append(lanOnly, isLanOnly) + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + return pool.Resolve(ctx, msg) + } + t.Cleanup(func() { internalDomainOSResolve = prev }) + + internal := startRcodeDNSFixture(t, dns.RcodeNameError, false) + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}}) + p := newInternalDomainsProg(t, cfg) + + res := fallbackTestQuery(t, p, cfg) + if len(lanOnly) != 1 || !lanOnly[0] { + t.Fatalf("OS step contexts LAN-only = %v, want one LAN-only query", lanOnly) + } + if got := lan.questions(); len(got) != 1 { + t.Errorf("LAN nameserver questions = %v, want the fallback query", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Errorf("answer = %v, want the LAN nameserver's NOERROR", res) + } +} + +// useVPNFallbackFixtures routes the fallback's VPN DNS servers to fixtures: +// real VPN DNS servers are dialed on port 53. +func useVPNFallbackFixtures(t *testing.T, servers map[string]string) { + t.Helper() + prev := internalDomainVPNUpstream + internalDomainVPNUpstream = func(_ *vpnDNSManager, server string) *ctrld.UpstreamConfig { + return &ctrld.UpstreamConfig{Name: "VPN DNS", Type: ctrld.ResolverTypeLegacy, Endpoint: servers[server], Timeout: 1000} + } + t.Cleanup(func() { internalDomainVPNUpstream = prev }) +} + +// The fallback asks VPN DNS servers that match the domain, then domain-less +// ones, then the OS resolver, and stops at the first NOERROR. +func TestInternalDomainsNetworkFallbackVPNOrder(t *testing.T) { + for _, tc := range []struct { + name string + matched, domainless int + wantMatched, wantDomless int + wantOS bool + }{ + {name: "matching VPN server answers", matched: dns.RcodeSuccess, domainless: dns.RcodeSuccess, wantMatched: 1}, + {name: "then the domain-less one", matched: dns.RcodeNameError, domainless: dns.RcodeSuccess, wantMatched: 1, wantDomless: 1}, + {name: "then the OS resolver", matched: dns.RcodeNameError, domainless: dns.RcodeRefused, wantMatched: 1, wantDomless: 1, wantOS: true}, + } { + t.Run(tc.name, func(t *testing.T) { + useDNSIntercept(t) + osFixture := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + useOSResolverFixture(t, osFixture) + matched := startRcodeDNSFixture(t, tc.matched, false) + domainless := startRcodeDNSFixture(t, tc.domainless, false) + useVPNFallbackFixtures(t, map[string]string{"10.8.0.1": matched.addr, "10.8.0.2": domainless.addr}) + + internal := startRcodeDNSFixture(t, dns.RcodeNameError, false) + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}}) + p := newInternalDomainsProg(t, cfg) + p.vpnDNS = &vpnDNSManager{ + routes: map[string][]string{"corp.example": {"10.8.0.1"}}, + domainlessServers: []string{"10.8.0.2"}, + } + + res := fallbackTestQuery(t, p, cfg) + if got := len(matched.questions()); got != tc.wantMatched { + t.Errorf("matching VPN server asked %d times, want %d", got, tc.wantMatched) + } + if got := len(domainless.questions()); got != tc.wantDomless { + t.Errorf("domain-less VPN server asked %d times, want %d", got, tc.wantDomless) + } + if got := len(osFixture.questions()) > 0; got != tc.wantOS { + t.Errorf("OS resolver asked = %v, want %v", got, tc.wantOS) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeSuccess { + t.Errorf("answer = %v, want NOERROR", res) + } + }) + } +} + +// While Windows is serving retained VPN DNS state, a VPN transport failure +// stops the fallback before the OS resolver, as it does for VPN split routing. +func TestInternalDomainsNetworkFallbackStopsOnRetainedVPNTransportFailure(t *testing.T) { + useDNSIntercept(t) + prevSettling := vpnDNSSettlingEnabled + vpnDNSSettlingEnabled = true + t.Cleanup(func() { vpnDNSSettlingEnabled = prevSettling }) + osFixture := startRcodeDNSFixture(t, dns.RcodeSuccess, false) + useOSResolverFixture(t, osFixture) + useVPNFallbackFixtures(t, map[string]string{"10.8.0.1": deadLoopbackAddr(t)}) + + internal := startRcodeDNSFixture(t, dns.RcodeNameError, false) + cfg := internalDomainsTestConfig() + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{internal.addr}}}) + p := newInternalDomainsProg(t, cfg) + p.vpnDNS = &vpnDNSManager{ + routes: map[string][]string{"corp.example": {"10.8.0.1"}}, + retainedAfterEmptyDiscovery: true, + } + + res := fallbackTestQuery(t, p, cfg) + if got := osFixture.questions(); len(got) != 0 { + t.Errorf("OS resolver asked %v while retained VPN DNS state is active", got) + } + if res == nil || res.answer == nil || res.answer.Rcode != dns.RcodeNameError { + t.Errorf("answer = %v, want the configured resolver's NXDOMAIN", res) + } +} diff --git a/cmd/cli/internal_domains_review_test.go b/cmd/cli/internal_domains_review_test.go index d2ece04f..8a65c970 100644 --- a/cmd/cli/internal_domains_review_test.go +++ b/cmd/cli/internal_domains_review_test.go @@ -139,9 +139,10 @@ func TestInternalDomainsExplicitResolverSurvivesRecoveryBypass(t *testing.T) { } } -// When the configured resolvers are unreachable the query must fail rather than -// reach the OS resolver, and the failure must be observable on the wire as an -// attempt against the configured address. +// In "explicit resolver only" mode, when the configured resolvers are +// unreachable the query must fail rather than reach the OS resolver, and the +// failure must be observable on the wire as an attempt against the configured +// address. func TestInternalDomainsUnreachableResolverDoesNotReachOSResolver(t *testing.T) { osFixture := startDNSFixture(t) useOSResolverFixture(t, osFixture) @@ -157,7 +158,7 @@ func TestInternalDomainsUnreachableResolverDoesNotReachOSResolver(t *testing.T) cfg := internalDomainsTestConfig() cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true) - applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Resolvers: []string{dead}}}) + applyInternalDomains(cfg, []controld.SplitDNS{{Domain: "corp.example", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{dead}}}) p := newInternalDomainsProg(t, cfg) addr, err := net.ResolveUDPAddr("udp", "192.168.0.1:0") @@ -301,6 +302,10 @@ func TestInternalDomainsSetupKeepsResolverAddressesOutOfInfoLogs(t *testing.T) { // Domain upstream that endpoint is private, so only a classification may appear // at error level. func TestInternalDomainsResolverFailureLogsNoAddress(t *testing.T) { + // The default mode falls back to the network after the failure; keep that + // on loopback. + useOSResolverFixture(t, startDNSFixture(t)) + pc, err := net.ListenPacket("udp", "127.0.0.1:0") if err != nil { t.Fatal(err) diff --git a/cmd/cli/internal_domains_test.go b/cmd/cli/internal_domains_test.go index f96323ec..243ff3e8 100644 --- a/cmd/cli/internal_domains_test.go +++ b/cmd/cli/internal_domains_test.go @@ -403,6 +403,11 @@ func TestInternalDomainsEqual(t *testing.T) { } func TestInternalDomainExplicitUpstreams(t *testing.T) { + p := internalDomainUpstreamsProg(map[string]string{ + "internal_0": internalDomainUpstreamFallback, + "internal_1": internalDomainUpstreamOnly, + "0": "", + }) for _, tc := range []struct { in []string want bool @@ -415,7 +420,7 @@ func TestInternalDomainExplicitUpstreams(t *testing.T) { {nil, false}, {[]string{}, false}, } { - if got := internalDomainExplicitUpstreams(tc.in); got != tc.want { + if got := p.internalDomainExplicitUpstreams(tc.in); got != tc.want { t.Errorf("internalDomainExplicitUpstreams(%v) = %v, want %v", tc.in, got, tc.want) } } @@ -448,13 +453,13 @@ func TestInternalDomainsSummaryLogsCountsOnly(t *testing.T) { // An explicit-resolver Internal Domain must not be handed to VPN DNS split // routing: the administrator named those resolvers, and VPN suffixes are -// auto-detected. When the named resolvers are unreachable the query fails -// instead of reaching the OS resolver. +// auto-detected. In "explicit resolver only" mode, when the named resolvers +// are unreachable the query fails instead of reaching the OS resolver. func TestInternalDomainsBeatVPNSplitRoutingAndDoNotLeak(t *testing.T) { cfg := internalDomainsTestConfig() cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(true) applyInternalDomains(cfg, []controld.SplitDNS{ - {Domain: "aws.example.com", Resolvers: []string{"127.0.0.1:1"}}, + {Domain: "aws.example.com", Mode: controld.SplitDNSModeResolversOnly, Resolvers: []string{"127.0.0.1:1"}}, }) p := &prog{cfg: cfg} @@ -473,7 +478,7 @@ func TestInternalDomainsBeatVPNSplitRoutingAndDoNotLeak(t *testing.T) { } ctx := context.WithValue(context.Background(), ctrld.ReqIdCtxKey{}, requestID()) ufr := p.upstreamFor(ctx, "0", cfg.Listener["0"], addr, "", "host.aws.example.com") - if !internalDomainExplicitUpstreams(ufr.upstreams) { + if !p.internalDomainExplicitUpstreams(ufr.upstreams) { t.Fatalf("upstreams = %v, want only internal resolvers", ufr.upstreams) } @@ -571,9 +576,18 @@ func TestApplyInternalDomainsModeIsAuthoritative(t *testing.T) { wantUpstreams: 1, }, { - // A mode ctrld does not know is not guessed at. - name: "unknown mode is dropped", + // A wire value this build does not know fails closed to the + // resolvers the administrator chose, never to the Control D upstream. + name: "unknown mode with resolvers fails closed", entry: controld.SplitDNS{Domain: "a.example.com", Mode: "forward-someday", Resolvers: []string{"10.0.0.53"}}, + wantTargets: []string{"upstream.internal_0"}, + wantRule: true, + wantUpstreams: 2, + }, + { + // With no resolvers there is nothing to fail closed to. + name: "unknown mode without resolvers is dropped", + entry: controld.SplitDNS{Domain: "a.example.com", Mode: "forward-someday"}, wantRule: false, wantUpstreams: 1, }, diff --git a/cmd/cli/prog.go b/cmd/cli/prog.go index 8bb1f77e..09c38090 100644 --- a/cmd/cli/prog.go +++ b/cmd/cli/prog.go @@ -140,6 +140,7 @@ type prog struct { internalLogWriter *logWriter internalJournalWriter *logWriter querySampler errorSampler + internalDomainProbes internalDomainProber lastNetworkState atomic.Pointer[netmon.State] internalLogSent time.Time runningIface string @@ -859,7 +860,7 @@ func (p *prog) setupUpstream(cfg *ctrld.Config) { // gated on the API confirming the device is gone. managedUpstreams := 0 for n := range cfg.Upstream { - if isGeneratedInternalDomainUpstream(n, cfg.Upstream[n]) { + if isGeneratedInternalDomainUpstream(cfg.Upstream[n]) { continue } managedUpstreams++ @@ -1161,7 +1162,7 @@ func (p *prog) startNetworkJournal() func() { close(stop) }() healthDone := p.health.startLoop(stop, func() (int, bool) { - return p.upstreamMonitorNow().countDownExcept(isInternalDomainUpstream), p.recoveryBypass.Load() + return p.upstreamMonitorNow().countDownExcept(p.isInternalDomainUpstream), p.recoveryBypass.Load() }) var configDone <-chan struct{} if p.dnsConfig != nil { diff --git a/cmd/cli/query_health_prog_test.go b/cmd/cli/query_health_prog_test.go index cd0520db..0281c655 100644 --- a/cmd/cli/query_health_prog_test.go +++ b/cmd/cli/query_health_prog_test.go @@ -107,10 +107,11 @@ func Test_queryHealthKeepsInternalDomainFailuresOutOfTheGrade(t *testing.T) { cfg.Service.LeakOnUpstreamFailure = func(v bool) *bool { return &v }(false) cfg.Upstream = map[string]*ctrld.UpstreamConfig{ internalDomainUpstreamPrefix + "1": { - Name: "dead internal resolver", - Type: ctrld.ResolverTypeLegacy, - Endpoint: deadUpstreamEndpoint, - Timeout: 500, + Name: "dead internal resolver", + Type: ctrld.ResolverTypeLegacy, + Endpoint: deadUpstreamEndpoint, + Timeout: 500, + InternalDomain: internalDomainUpstreamOnly, }, } p := &prog{cfg: cfg, um: newUpstreamMonitor(cfg), health: newQueryHealth()} @@ -140,7 +141,7 @@ func Test_queryHealthKeepsInternalDomainFailuresOutOfTheGrade(t *testing.T) { p.um.markDown(internalUpstream, 3, "immediate") p.um.mu.Unlock() - event, emit := p.health.evaluate(time.Now(), p.um.countDownExcept(isInternalDomainUpstream), false) + event, emit := p.health.evaluate(time.Now(), p.um.countDownExcept(p.isInternalDomainUpstream), false) if !emit { t.Fatal("the first evaluate did not emit") } diff --git a/cmd/cli/recovery_scope_test.go b/cmd/cli/recovery_scope_test.go index fc294269..63c71f3d 100644 --- a/cmd/cli/recovery_scope_test.go +++ b/cmd/cli/recovery_scope_test.go @@ -303,7 +303,9 @@ func TestScopeHarnessRestoresRecoveryWait(t *testing.T) { func TestOSRecoveryCandidates(t *testing.T) { remote := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeDOH} - internal := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy} + internal := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy, InternalDomain: internalDomainUpstreamFallback} + // A user-defined upstream that copies a generated key carries no marker. + lookalike := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeLegacy} osConfig := &ctrld.UpstreamConfig{Type: ctrld.ResolverTypeOS} for _, tc := range []struct { name string @@ -311,9 +313,10 @@ func TestOSRecoveryCandidates(t *testing.T) { want map[string]*ctrld.UpstreamConfig }{ {"configured", map[string]*ctrld.UpstreamConfig{"0": remote, "internal_0": internal, "1": osConfig, "2": nil}, map[string]*ctrld.UpstreamConfig{"upstream.0": remote}}, - {"custom_policy_or_unused", map[string]*ctrld.UpstreamConfig{"0": remote, "unused": internal}, map[string]*ctrld.UpstreamConfig{"upstream.0": remote, "upstream.unused": internal}}, + {"custom_policy_or_unused", map[string]*ctrld.UpstreamConfig{"0": remote, "unused": lookalike}, map[string]*ctrld.UpstreamConfig{"upstream.0": remote, "upstream.unused": lookalike}}, {"only_os", map[string]*ctrld.UpstreamConfig{"0": osConfig}, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}}, {"only_internal", map[string]*ctrld.UpstreamConfig{"internal_0": internal}, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}}, + {"internal_lookalike", map[string]*ctrld.UpstreamConfig{"internal_0": lookalike}, map[string]*ctrld.UpstreamConfig{"upstream.internal_0": lookalike}}, {"empty", nil, map[string]*ctrld.UpstreamConfig{upstreamOS: osUpstreamConfig}}, } { t.Run(tc.name, func(t *testing.T) { diff --git a/cmd/cli/upstream_monitor_test.go b/cmd/cli/upstream_monitor_test.go index 20f6a9b1..3447144c 100644 --- a/cmd/cli/upstream_monitor_test.go +++ b/cmd/cli/upstream_monitor_test.go @@ -293,7 +293,12 @@ func Test_upstreamMonitorCountDownExcept(t *testing.T) { if got := um.countDown(); got != 3 { t.Fatalf("countDown = %d, want 3", got) } - if got := um.countDownExcept(isInternalDomainUpstream); got != 1 { + p := internalDomainUpstreamsProg(map[string]string{ + "0": "", + internalDomainUpstreamPrefix + "1": internalDomainUpstreamFallback, + internalDomainUpstreamPrefix + "2": internalDomainUpstreamOnly, + }) + if got := um.countDownExcept(p.isInternalDomainUpstream); got != 1 { t.Fatalf("countDownExcept(isInternalDomainUpstream) = %d, want 1", got) } if got := um.countDownExcept(nil); got != 3 { diff --git a/config.go b/config.go index 8bcc93e3..4ac52ee2 100644 --- a/config.go +++ b/config.go @@ -269,8 +269,13 @@ type UpstreamConfig struct { Endpoint string `mapstructure:"endpoint" toml:"endpoint,omitempty"` BootstrapIP string `mapstructure:"bootstrap_ip" toml:"bootstrap_ip,omitempty"` Domain string `mapstructure:"-" toml:"-"` - IPStack string `mapstructure:"ip_stack" toml:"ip_stack,omitempty" validate:"ipstack"` - Timeout int `mapstructure:"timeout" toml:"timeout,omitempty" validate:"gte=0"` + // InternalDomain is set only by ctrld, on the upstreams it generates for + // organization Internal Domains, to the mode they serve. It is never read + // from or written to a configuration file, so an upstream a configuration + // defines is never mistaken for a generated one, whatever it is named. + InternalDomain string `mapstructure:"-" toml:"-"` + IPStack string `mapstructure:"ip_stack" toml:"ip_stack,omitempty" validate:"ipstack"` + Timeout int `mapstructure:"timeout" toml:"timeout,omitempty" validate:"gte=0"` // The caller should not access this field directly. // Use UpstreamSendClientInfo instead. SendClientInfo *bool `mapstructure:"send_client_info" toml:"send_client_info,omitempty"` diff --git a/docs/internal-domains.md b/docs/internal-domains.md index 1630b8de..72e70d60 100644 --- a/docs/internal-domains.md +++ b/docs/internal-domains.md @@ -15,7 +15,8 @@ The managed resolver configuration carries the list in `resolver.split_dns`: "resolver": { "split_dns": [ { "domain": "corp.example.com", "mode": "resolvers", "resolvers": ["10.0.0.53", "10.0.0.54"] }, - { "domain": "office.example.com", "mode": "os", "resolvers": [] } + { "domain": "office.example.com", "mode": "os", "resolvers": [] }, + { "domain": "vault.example.com", "mode": "resolvers_only", "resolvers": ["10.0.0.55"] } ] } } @@ -25,13 +26,15 @@ The managed resolver configuration carries the list in `resolver.split_dns`: | `mode` | Behavior | |---|---| -| `os` | The query goes to `upstream.os`, the endpoint's current system resolver, and follows DHCP and VPN resolver changes. Any addresses still present in `resolvers` are **not** read, so switching a domain back to the OS resolver cannot be undone by leftovers from a previous selection. | -| `resolvers` | The query goes only to the addresses in `resolvers`, tried in the order given. An empty or entirely unusable list does not become OS resolution: the entry is dropped instead. | -| absent | A deployment that predates the field. The mode is inferred from `resolvers`, which is what the field encodes. | -| anything else | Dropped. Routing a private domain by guess is the disclosure this feature exists to prevent. | +| `os` | **Network default.** The query goes to `upstream.os`, the endpoint's current system resolver, and follows DHCP and VPN resolver changes. Any addresses still present in `resolvers` are **not** read, so switching a domain back to the OS resolver cannot be undone by leftovers from a previous selection. | +| `resolvers` | **Explicit resolver with network fallback** (the default explicit selection). The query goes to the addresses in `resolvers` first, tried in the order given; if none of them resolves it, it falls back to the network resolvers (see [Network fallback](#network-fallback)). | +| `resolvers_only` | **Explicit resolver only.** The query goes only to the addresses in `resolvers`, and their answer or failure is final. Not offered by the dashboard yet. | +| absent | A deployment that predates the field. Treated as **explicit resolver with network fallback**, the same as `resolvers`. If the entry has no resolvers there is nothing explicit to try, so it uses the network default (`os`). | +| anything else | Fails closed. With resolvers, treated as `resolvers_only`: only the resolvers the administrator chose, so a wire value this build does not know (a renamed strict mode, a future mode) never sends the domain to the Control D upstream or the network. Without resolvers there is nothing to fail closed to, so the entry is dropped. | -A dropped entry leaves its domain on its normal path and is reported in the -`skipped` count. +In both explicit modes an empty or entirely unusable list does not become OS +resolution: the entry is dropped instead. A dropped entry leaves its domain on +its normal path and is reported in the `skipped` count. Addresses may be IPv4 or IPv6, bare or with a port; a bare address uses port 53. An address that is not a valid IP is dropped from the list. @@ -48,8 +51,15 @@ and routing a private domain to the wrong place is worse than not routing it. ## Generated configuration Each accepted domain contributes two policy rules, `` and `*.`, -which is how ctrld matches a suffix and all of its subdomains. Explicit mode -also generates one upstream per resolver address, named `internal_`: +which is how ctrld matches a suffix and all of its subdomains. The explicit +modes also generate one upstream per resolver address, named `internal_`. +Each one carries an internal marker recording its mode, which is what tells +`proxy()` that an upstream is a generated Internal Domain resolver and whether +the network fallback may run. The marker is never read from or written to a +configuration file, so the key name is only a name: an upstream a local +`ctrld.toml` or a custom configuration defines as `internal_foo`, or even as an +exact copy of a generated one, is an ordinary upstream. The generated +configuration looks like this: ```toml [upstream.internal_0] @@ -97,27 +107,75 @@ exists for a source is never overwritten. From highest precedence to lowest: **VPN DNS auto-detection** is evaluated inside `proxy()` in DNS-intercept mode, after policy matching and independently of it, so it needs its own rule: -- An **explicit-resolver** Internal Domain skips VPN routing. The administrator - named the resolvers for that suffix; VPN suffixes are discovered from the OS, - and letting a discovered route override a named one is the leak this feature - exists to prevent. +- An **explicit-resolver** Internal Domain skips VPN routing ahead of its + configured resolvers. The administrator named the resolvers for that suffix; + VPN suffixes are discovered from the OS, and a discovered route must not + override a named one. In `resolvers` mode the VPN DNS servers are asked + afterwards, as part of the [network fallback](#network-fallback). - An **OS-resolver** Internal Domain does not skip it, and behaves exactly like a Magic Folder exclude. It asks for the endpoint's default resolution, which under intercept mode includes the VPN's own resolver. Every domain that is not an Internal Domain keeps its current VPN routing. +## Network fallback + +In `resolvers` mode the configured resolvers are always asked first, even while +the DNS-intercept recovery bypass is active and ahead of VPN DNS split routing. +A configured resolver that times out, is unreachable, or answers SERVFAIL, +NXDOMAIN, REFUSED or NOTIMP hands the query to the next configured resolver, +and once all of them have, to the network resolvers: + +1. VPN DNS servers whose domains match the query, then VPN DNS servers that + advertise no domains (DNS-intercept mode only, where ctrld tracks them). +2. The OS resolver's **LAN** nameservers: private, loopback, link-local and + CGNAT addresses the active network provides. + +A configured resolver that the upstream monitor has marked down (after 10 +seconds of continuous failures, or 50 failures) is skipped rather than waited +on, so an endpoint away from the organization network reaches the fallback +without paying the resolver's timeout, about 2 seconds per resolver, on every +query. Queries in the first seconds off-site, before the resolver is marked +down, still pay it. A generated resolver comes back up only by answering, so +skipping one also re-checks it in the background, at most once every 30 +seconds, with the query that was skipped. Once it answers it is used again. +`resolvers_only` mode never skips: with no fallback to go to, skipping would +only turn the resolver's answer into SERVFAIL. + +The first NOERROR answer wins. NXDOMAIN triggers the fallback because the +configured resolver may be reachable but not know a name that the local +network's resolver does, for example on another subnet. REFUSED and NOTIMP +trigger it because they mean the resolver will not serve this endpoint, which +is how an organization resolver with an access list answers an endpoint off +the organization network. An empty NOERROR (the name exists but has no record +of the requested type) is final, and so is any other rcode. + +The OS step never sends the name to public DNS. It is a LAN-only query: a +public nameserver in the OS pool is not asked at all, whether DHCP supplied it +(a router handing out `192.168.1.1` plus `8.8.8.8`, a Wi-Fi handing out only +public resolvers) or it is ctrld's own public fallback (`76.76.2.0`). With no +LAN nameserver the OS step sends nothing. The general OS-resolver catch-all, +which does reach public nameservers, is never used either. The VPN step uses +the servers the VPN pushes as they are, including a public address a VPN +pushes as its DNS server. + +While Windows is serving retained VPN DNS state, a VPN transport failure stops +the fallback before the OS resolver, as it does for VPN split routing. + +When no resolver resolves the name, the query ends with the best negative +answer it got (NXDOMAIN before any other), or SERVFAIL if it got none. + ## Unreachable explicit resolvers -When every resolver for an explicit-mode domain fails, the query returns -SERVFAIL. It is deliberately not retried against the OS resolver, and it does -not trigger the endpoint recovery flow: +In `resolvers_only` mode, when every configured resolver fails the query returns +SERVFAIL, and an NXDOMAIN or SERVFAIL answer from them is returned as it is. +Either way the query is not retried anywhere else. Administrators who need the +configured resolvers' answer enforced select this mode. -- Sending a private name to a resolver the administrator did not select is the - leak this feature exists to prevent. -- The recovery flow exists for the loss of general DNS. One unavailable internal - server is not that, and letting it start recovery would reset endpoint DNS - settings for every other domain. +In neither explicit mode does a failure trigger the endpoint recovery flow. That +flow exists for the loss of general DNS. One unavailable internal server is not +that, and letting it start recovery would reset endpoint DNS settings for every +other domain. OS-resolver mode has no such restriction: it is the system resolver, so it follows DHCP and VPN changes and shares their failure handling. @@ -133,9 +191,10 @@ second upstream. Eligibility is recomputed on every setup and reload, so adding or removing Internal Domains, or gaining a genuinely custom second upstream, is reflected instead of latching on the first value seen. -Discounting is by upstream shape, not by key name alone: an endpoint custom -configuration replaces the generated config outright and could name an upstream -`internal_0`, which must not buy eligibility. The uninstall itself remains +Discounting is by the internal marker, not by key name or any other +configurable field: an endpoint custom configuration replaces the generated +config outright and could define an upstream `internal_0` that copies a +generated one, which must not buy eligibility. The uninstall itself remains gated on the API confirming the device is gone. ## Refresh @@ -168,6 +227,7 @@ the count. That covers every consumer, not just query handling: of recovery started for another reason. Info and warning logs report counts only — domains applied, OS-resolver and -explicit-resolver totals, generated resolvers, skipped and preempted entries. +explicit-resolver totals (with `explicit_resolver_only` for `resolvers_only`), +generated resolvers, skipped and preempted entries. Domain names and resolver addresses are organization-private and appear only at debug level. diff --git a/internal/controld/config.go b/internal/controld/config.go index adf83a7a..2d8b7c3e 100644 --- a/internal/controld/config.go +++ b/internal/controld/config.go @@ -64,25 +64,31 @@ type ResolverConfig struct { } // Internal Domain resolution modes. Mode is what the administrator selected; -// Resolvers is only meaningful under SplitDNSModeResolvers. +// Resolvers is only meaningful under SplitDNSModeResolvers and +// SplitDNSModeResolversOnly. const ( // SplitDNSModeOS resolves the domain through the endpoint's own OS/default // resolver, so it follows DHCP and VPN resolver changes. SplitDNSModeOS = "os" // SplitDNSModeResolvers resolves the domain through the addresses in - // Resolvers and nothing else. + // Resolvers first. When none of them answers, or they answer SERVFAIL or + // NXDOMAIN, the query falls back to the resolvers of the endpoint's active + // network and VPN. This is the default explicit selection. SplitDNSModeResolvers = "resolvers" + // SplitDNSModeResolversOnly resolves the domain through the addresses in + // Resolvers and nothing else: their answer, or their failure, is final. + SplitDNSModeResolversOnly = "resolvers_only" ) // SplitDNS is one organization Internal Domain: a domain suffix, the resolution // mode the organization administrator selected for it, and the resolver // addresses that mode may need. // -// Mode decides the routing. Resolvers is read only under -// SplitDNSModeResolvers, so addresses left behind by an earlier selection -// cannot resurrect themselves once the administrator switches back to the OS -// resolver. An empty Mode is a deployment that predates the field; the caller -// infers the mode from Resolvers in that case. +// Mode decides the routing. Resolvers is read only under the explicit modes, +// so addresses left behind by an earlier selection cannot resurrect themselves +// once the administrator switches back to the OS resolver. An empty Mode is a +// deployment that predates the field; the caller infers the mode from +// Resolvers in that case. type SplitDNS struct { Domain string `json:"domain"` Mode string `json:"mode"` diff --git a/internal/controld/split_dns_test.go b/internal/controld/split_dns_test.go index 626287a5..7c2e1e74 100644 --- a/internal/controld/split_dns_test.go +++ b/internal/controld/split_dns_test.go @@ -112,6 +112,11 @@ func TestResolverConfigParsesSplitDNSMode(t *testing.T) { "domain": "office.example.com", "mode": "os", "resolvers": [] + }, + { + "domain": "vault.example.com", + "mode": "resolvers_only", + "resolvers": ["10.0.0.55"] } ] } @@ -122,7 +127,7 @@ func TestResolverConfigParsesSplitDNSMode(t *testing.T) { t.Fatal(err) } got := resp.Body.Resolver.SplitDNS - if len(got) != 2 { + if len(got) != 3 { t.Fatalf("split_dns = %+v", got) } if got[0].Domain != "corp.example.com" || got[0].Mode != SplitDNSModeResolvers { @@ -137,6 +142,9 @@ func TestResolverConfigParsesSplitDNSMode(t *testing.T) { if len(got[1].Resolvers) != 0 { t.Errorf("[1].Resolvers = %v, want empty", got[1].Resolvers) } + if got[2].Mode != SplitDNSModeResolversOnly || len(got[2].Resolvers) != 1 { + t.Errorf("[2] = %+v", got[2]) + } } // A response without the field must still decode, so a deployment that predates diff --git a/resolver.go b/resolver.go index 38e48e7f..fb0f9e50 100644 --- a/resolver.go +++ b/resolver.go @@ -139,6 +139,29 @@ func LanQueryCtx(ctx context.Context) context.Context { return context.WithValue(ctx, LanQueryCtxKey{}, true) } +// LanOnlyQueryCtxKey is the context.Context key that limits an OS resolver +// query to its LAN nameservers. +type LanOnlyQueryCtxKey struct{} + +// LanOnlyQueryCtx returns a context.Context with LanOnlyQueryCtxKey set. +// +// An OS resolver query made with it goes only to LAN-classified nameservers: +// private, loopback, link-local and CGNAT addresses. Every public nameserver +// is left out, the ones DHCP or a VPN supplied as well as ctrld's own public +// fallback, and with no LAN nameserver the query fails without being sent. It +// is for names that must not reach public DNS, such as an organization's +// Internal Domains. LanQueryCtx, by contrast, only drops ctrld's own public +// fallback. +func LanOnlyQueryCtx(ctx context.Context) context.Context { + return context.WithValue(ctx, LanOnlyQueryCtxKey{}, true) +} + +// isLanOnlyQuery reports whether ctx was made by LanOnlyQueryCtx. +func isLanOnlyQuery(ctx context.Context) bool { + lanOnly, _ := ctx.Value(LanOnlyQueryCtxKey{}).(bool) + return lanOnly +} + // defaultNameservers is like nameservers with each element formed "ip:53". func defaultNameservers() []string { ns := nameservers() @@ -590,6 +613,12 @@ func (o *osResolver) Resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error // the key so subnet-specific answers are neither coalesced nor hot-cached // across different subnets (RFC 7871 §7.3). key := fmt.Sprintf("%s:%d:%s", domain, qtype, dnscache.CanonicalECS(msg)) + // A LAN-only query must not join, or be answered from the hot cache of, an + // ordinary query for the same name: that answer may have come from a public + // nameserver. + if isLanOnlyQuery(ctx) { + key += ":lan-only" + } // Checking the cache first. if val, ok := o.cache.Load(key); ok { @@ -642,6 +671,10 @@ func (o *osResolver) resolve(ctx context.Context, msg *dns.Msg) (*dns.Msg, error if p := o.lanServers.Load(); p != nil { nss = append(nss, (*p)...) } + // A LAN-only query is never sent to a public nameserver. + if isLanOnlyQuery(ctx) { + publicServers = nil + } numServers := len(nss) + len(publicServers) // If this is a LAN query, skip public DNS. diff --git a/resolver_test.go b/resolver_test.go index 674bef37..32374036 100644 --- a/resolver_test.go +++ b/resolver_test.go @@ -1092,3 +1092,77 @@ func TestInitializeOsResolverReportsTheUnspecifiedReason(t *testing.T) { wantLogField(t, changes[0], "reason", "unspecified") wantLogStrings(t, changes[0], "after", []string{"192.0.2.10"}) } + +// A LAN-only query must reach the LAN nameservers and nothing else. A public +// nameserver in the same pool, as DHCP hands out next to the router, is not +// asked at all, so the name is never sent to it. +func Test_osResolver_LanOnlyQuerySkipsPublicNameservers(t *testing.T) { + const lan, public = "10.0.0.53:53", "192.0.2.53:53" + resolver := newResolverWithNameserver([]string{lan, public, controldPublicDnsWithPort}) + + var mu sync.Mutex + var asked []string + resolver.exchangeDNS = func(_ context.Context, msg *dns.Msg, server string, _ net.IP) (*dns.Msg, time.Duration, error) { + mu.Lock() + asked = append(asked, server) + mu.Unlock() + answer := new(dns.Msg) + answer.SetRcode(msg, dns.RcodeNameError) + return answer, time.Millisecond, nil + } + + msg := new(dns.Msg) + msg.SetQuestion("host.corp.example.", dns.TypeA) + answer, err := resolver.Resolve(LanOnlyQueryCtx(context.Background()), msg) + if err != nil { + t.Fatal(err) + } + if answer.Rcode != dns.RcodeNameError { + t.Errorf("rcode = %s, want the LAN nameserver's NXDOMAIN", dns.RcodeToString[answer.Rcode]) + } + if len(asked) != 1 || asked[0] != lan { + t.Fatalf("nameservers asked = %v, want only %s", asked, lan) + } +} + +// With no LAN nameserver at all, a LAN-only query fails without sending +// anything, rather than falling through to the public ones. +func Test_osResolver_LanOnlyQueryWithoutLanNameserverSendsNothing(t *testing.T) { + resolver := newResolverWithNameserver([]string{"192.0.2.53:53", controldPublicDnsWithPort}) + calls := 0 + resolver.exchangeDNS = func(_ context.Context, msg *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + calls++ + answer := new(dns.Msg) + answer.SetReply(msg) + return answer, time.Millisecond, nil + } + + msg := new(dns.Msg) + msg.SetQuestion("host.corp.example.", dns.TypeA) + if _, err := resolver.Resolve(LanOnlyQueryCtx(context.Background()), msg); err == nil { + t.Fatal("a LAN-only query with no LAN nameserver returned an answer") + } + if calls != 0 { + t.Fatalf("exchange calls = %d, want none", calls) + } +} + +// The hot cache is keyed separately for LAN-only queries: an answer an +// ordinary query got from a public nameserver must not serve a LAN-only one. +func Test_osResolver_LanOnlyQueryDoesNotShareOrdinaryHotCache(t *testing.T) { + resolver := newResolverWithNameserver([]string{"192.0.2.53:53"}) + resolver.exchangeDNS = func(_ context.Context, msg *dns.Msg, _ string, _ net.IP) (*dns.Msg, time.Duration, error) { + answer := new(dns.Msg) + answer.SetReply(msg) + return answer, time.Millisecond, nil + } + + msg := new(dns.Msg) + msg.SetQuestion("host.corp.example.", dns.TypeA) + if _, err := resolver.Resolve(context.Background(), msg); err != nil { + t.Fatalf("ordinary query: %v", err) + } + if _, err := resolver.Resolve(LanOnlyQueryCtx(context.Background()), msg); err == nil { + t.Fatal("a LAN-only query was served the ordinary query's public answer from the hot cache") + } +} From 806683d7271838b0893f3186917ec846699e554b Mon Sep 17 00:00:00 2001 From: Cuong Manh Le Date: Mon, 5 Oct 2026 18:19:36 +0700 Subject: [PATCH 29/29] test(cli): take the upload time bound before reading the upload TestOSRecoverySkipSurvivesDebugTruncation passed time.Now() as the argument after readLogReader, and Go evaluates arguments left to right, so the lower bound was taken after the upload header was rendered. The header check failed whenever a second boundary fell between the two: header time = 11:13:44, want a time between 11:13:45.007 and now Take the bound first, as the other splitUpload callers do. --- cmd/cli/recovery_skip_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/cli/recovery_skip_test.go b/cmd/cli/recovery_skip_test.go index 69cdc5d2..f5a8ad76 100644 --- a/cmd/cli/recovery_skip_test.go +++ b/cmd/cli/recovery_skip_test.go @@ -137,7 +137,11 @@ func TestOSRecoverySkipSurvivesDebugTruncation(t *testing.T) { for range 40 { mainLog.Load().Debug().Msg(strings.Repeat("debug filler", 20)) } - debugPart, journalPart := splitUpload(t, readLogReader(t, p, false), time.Now()) + // Taken before the read renders the upload header: as an argument after + // readLogReader it would be evaluated later, and fail whenever a second + // boundary falls between the two. + start := time.Now() + debugPart, journalPart := splitUpload(t, readLogReader(t, p, false), start) if strings.Contains(string(debugPart), recoverySkipMessage) { t.Fatal("fixture did not truncate the skip from debug") }