From 9b0849393757acab459c44bbdbe61628ececb225 Mon Sep 17 00:00:00 2001 From: Tim Douglas Date: Wed, 29 Jul 2026 22:21:50 -0400 Subject: [PATCH] Bound blackholed upstream connections with TCP_USER_TIMEOUT An upstream that stops acknowledging otherwise holds its CONNECT tunnel for tcp_retries2's ~16 minutes, and clients keep reusing it. Defaults to 30s, overridable with --upstream-tcp-user-timeout. Tested at the 30s default with iptables blackholes over a CONNECT tunnel. An unreachable peer is dropped after ~30s whether data is in flight, the receive window is closed, the SYN goes unanswered, or nothing is in flight at all. A reachable peer acknowledges at the TCP layer well before its application replies, so a response that takes 45s still arrives. A peer whose TCP stack answers while its application is gone stays undetected and needs an app-layer probe, which we're doing with HTTP2 keepalives in the Rust clients. --- cmd/smokescreen.go | 9 +++++++++ go.mod | 2 +- pkg/smokescreen/config.go | 9 +++++++++ pkg/smokescreen/smokescreen.go | 24 +++++++++++++++++++++++- 4 files changed, 42 insertions(+), 2 deletions(-) diff --git a/cmd/smokescreen.go b/cmd/smokescreen.go index be3a75f8..f75b161f 100644 --- a/cmd/smokescreen.go +++ b/cmd/smokescreen.go @@ -61,6 +61,11 @@ func NewConfiguration(args []string, logger *log.Logger) (*smokescreen.Config, e Value: time.Duration(10) * time.Second, Usage: "Time out after `DURATION` when connecting.", }, + cli.DurationFlag{ + Name: "upstream-tcp-user-timeout", + Value: smokescreen.DefaultUpstreamTCPUserTimeout, + Usage: "Close an upstream connection after data sent to it goes unacknowledged for `DURATION`, rather than the kernel's ~16m retransmit budget. Linux only; 0 disables.", + }, cli.BoolFlag{ Name: "proxy-protocol", Usage: "Enable PROXY protocol support.", @@ -198,6 +203,10 @@ func NewConfiguration(args []string, logger *log.Logger) (*smokescreen.Config, e conf.ConnectTimeout = c.Duration("timeout") } + if c.IsSet("upstream-tcp-user-timeout") { + conf.UpstreamTCPUserTimeout = c.Duration("upstream-tcp-user-timeout") + } + if c.IsSet("proxy-protocol") { conf.SupportProxyProtocol = c.Bool("proxy-protocol") } diff --git a/go.mod b/go.mod index 3451d220..82cd2127 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/stretchr/testify v1.8.0 github.com/stripe/goproxy v0.0.0-20241022131412-58117846327a golang.org/x/net v0.17.0 + golang.org/x/sys v0.13.0 gopkg.in/urfave/cli.v1 v1.20.0 gopkg.in/yaml.v2 v2.4.0 ) @@ -30,7 +31,6 @@ require ( github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect golang.org/x/mod v0.8.0 // indirect - golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect golang.org/x/tools v0.6.0 // indirect google.golang.org/protobuf v1.33.0 // indirect diff --git a/pkg/smokescreen/config.go b/pkg/smokescreen/config.go index 9d162023..462a2d64 100644 --- a/pkg/smokescreen/config.go +++ b/pkg/smokescreen/config.go @@ -70,6 +70,12 @@ type Config struct { // A connection is idle if it has been inactive (no bytes in/out) for this many seconds. IdleTimeout time.Duration + // Close an upstream connection after data sent to it goes unacknowledged for + // this long, in place of tcp_retries2's ~16 minutes. Also caps unanswered + // keepalive probes and SYN retries. A slow peer still acknowledges, so this + // does not bound response time. Linux only; 0 disables. + UpstreamTCPUserTimeout time.Duration + // These are *only* used for traditional HTTP proxy requests TransportMaxIdleConns int TransportMaxIdleConnsPerHost int @@ -247,6 +253,8 @@ type authKeyId struct { Id []byte `asn1:"optional,tag:0"` } +const DefaultUpstreamTCPUserTimeout = 30 * time.Second + func NewConfig() *Config { log.SetFormatter(&log.JSONFormatter{ TimestampFormat: time.RFC3339Nano, @@ -263,6 +271,7 @@ func NewConfig() *Config { ShuttingDown: atomic.Value{}, MetricsClient: metrics.NewNoOpMetricsClient(), Network: "ip", + UpstreamTCPUserTimeout: DefaultUpstreamTCPUserTimeout, } } diff --git a/pkg/smokescreen/smokescreen.go b/pkg/smokescreen/smokescreen.go index ce3b49f5..76cfbead 100644 --- a/pkg/smokescreen/smokescreen.go +++ b/pkg/smokescreen/smokescreen.go @@ -24,6 +24,7 @@ import ( "github.com/stripe/smokescreen/pkg/smokescreen/conntrack" "github.com/stripe/smokescreen/pkg/smokescreen/hostport" "github.com/stripe/smokescreen/pkg/smokescreen/metrics" + "golang.org/x/sys/unix" ) const ( @@ -245,6 +246,23 @@ func proxyContext(ctx context.Context) (*goproxy.ProxyCtx, bool) { return pctx, ok } +// tcpUserTimeoutControl returns a net.Dialer.Control hook setting +// TCP_USER_TIMEOUT, or nil when d is zero. net.Dialer exposes no field for the +// option and the frozen syscall package predates it, so the constant comes from +// x/sys/unix, which defines it on Linux only. A setsockopt failure is dropped +// rather than failing the dial. +func tcpUserTimeoutControl(d time.Duration) func(string, string, syscall.RawConn) error { + if d <= 0 { + return nil + } + millis := int(d.Milliseconds()) + return func(_, _ string, c syscall.RawConn) error { + return c.Control(func(fd uintptr) { + syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, millis) + }) + } +} + func dialContext(ctx context.Context, network, addr string) (net.Conn, error) { pctx, ok := proxyContext(ctx) if !ok { @@ -284,7 +302,11 @@ func dialContext(ctx context.Context, network, addr string) (net.Conn, error) { start := time.Now() if sctx.cfg.ProxyDialTimeout == nil { - conn, err = net.DialTimeout(network, d.ResolvedAddr.String(), sctx.cfg.ConnectTimeout) + dialer := net.Dialer{ + Timeout: sctx.cfg.ConnectTimeout, + Control: tcpUserTimeoutControl(sctx.cfg.UpstreamTCPUserTimeout), + } + conn, err = dialer.Dial(network, d.ResolvedAddr.String()) } else { conn, err = sctx.cfg.ProxyDialTimeout(ctx, network, d.ResolvedAddr.String(), sctx.cfg.ConnectTimeout) }