Bound blackholed upstream connections with TCP_USER_TIMEOUT

cmd/smokescreen.go

@@ -61,6 +61,11 @@ func NewConfiguration(args []string, logger *log.Logger) (*smokescreen.Config, e
 			Value: time.Duration(10) * time.Second,
 			Usage: "Time out after `DURATION` when connecting.",
 		},
+		cli.DurationFlag{
+			Name:  "upstream-tcp-user-timeout",
+			Value: smokescreen.DefaultUpstreamTCPUserTimeout,
+			Usage: "Close an upstream connection after data sent to it goes unacknowledged for `DURATION`, rather than the kernel's ~16m retransmit budget.  Linux only; 0 disables.",
+		},
 		cli.BoolFlag{
 			Name:  "proxy-protocol",
 			Usage: "Enable PROXY protocol support.",
@@ -198,6 +203,10 @@ func NewConfiguration(args []string, logger *log.Logger) (*smokescreen.Config, e
 			conf.ConnectTimeout = c.Duration("timeout")
 		}
 
+		if c.IsSet("upstream-tcp-user-timeout") {
+			conf.UpstreamTCPUserTimeout = c.Duration("upstream-tcp-user-timeout")
+		}
+
 		if c.IsSet("proxy-protocol") {
 			conf.SupportProxyProtocol = c.Bool("proxy-protocol")
 		}

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/stretchr/testify v1.8.0
 	github.com/stripe/goproxy v0.0.0-20241022131412-58117846327a
 	golang.org/x/net v0.17.0
+	golang.org/x/sys v0.13.0
 	gopkg.in/urfave/cli.v1 v1.20.0
 	gopkg.in/yaml.v2 v2.4.0
 )
@@ -30,7 +31,6 @@ require (
 	github.com/prometheus/common v0.37.0 // indirect
 	github.com/prometheus/procfs v0.8.0 // indirect
 	golang.org/x/mod v0.8.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
 	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/tools v0.6.0 // indirect
 	google.golang.org/protobuf v1.33.0 // indirect

pkg/smokescreen/config.go

@@ -70,6 +70,12 @@ type Config struct {
 	// A connection is idle if it has been inactive (no bytes in/out) for this many seconds.
 	IdleTimeout time.Duration
 
+	// Close an upstream connection after data sent to it goes unacknowledged for
+	// this long, in place of tcp_retries2's ~16 minutes. Also caps unanswered
+	// keepalive probes and SYN retries. A slow peer still acknowledges, so this
+	// does not bound response time. Linux only; 0 disables.
+	UpstreamTCPUserTimeout time.Duration
+
 	// These are *only* used for traditional HTTP proxy requests
 	TransportMaxIdleConns        int
 	TransportMaxIdleConnsPerHost int
@@ -247,6 +253,8 @@ type authKeyId struct {
 	Id []byte `asn1:"optional,tag:0"`
 }
 
+const DefaultUpstreamTCPUserTimeout = 30 * time.Second
+
 func NewConfig() *Config {
 	log.SetFormatter(&log.JSONFormatter{
 		TimestampFormat: time.RFC3339Nano,
@@ -263,6 +271,7 @@ func NewConfig() *Config {
 		ShuttingDown:            atomic.Value{},
 		MetricsClient:           metrics.NewNoOpMetricsClient(),
 		Network:                 "ip",
+		UpstreamTCPUserTimeout:  DefaultUpstreamTCPUserTimeout,
 	}
 }
 

pkg/smokescreen/smokescreen.go

@@ -24,6 +24,7 @@ import (
 	"github.com/stripe/smokescreen/pkg/smokescreen/conntrack"
 	"github.com/stripe/smokescreen/pkg/smokescreen/hostport"
 	"github.com/stripe/smokescreen/pkg/smokescreen/metrics"
+	"golang.org/x/sys/unix"
 )
 
 const (
@@ -245,6 +246,23 @@ func proxyContext(ctx context.Context) (*goproxy.ProxyCtx, bool) {
 	return pctx, ok
 }
 
+// tcpUserTimeoutControl returns a net.Dialer.Control hook setting
+// TCP_USER_TIMEOUT, or nil when d is zero. net.Dialer exposes no field for the
+// option and the frozen syscall package predates it, so the constant comes from
+// x/sys/unix, which defines it on Linux only. A setsockopt failure is dropped
+// rather than failing the dial.
+func tcpUserTimeoutControl(d time.Duration) func(string, string, syscall.RawConn) error {
+	if d <= 0 {
+		return nil
+	}
+	millis := int(d.Milliseconds())
+	return func(_, _ string, c syscall.RawConn) error {
+		return c.Control(func(fd uintptr) {
+			syscall.SetsockoptInt(int(fd), syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, millis)
+		})
+	}
+}
+
 func dialContext(ctx context.Context, network, addr string) (net.Conn, error) {
 	pctx, ok := proxyContext(ctx)
 	if !ok {
@@ -284,7 +302,11 @@ func dialContext(ctx context.Context, network, addr string) (net.Conn, error) {
 
 	start := time.Now()
 	if sctx.cfg.ProxyDialTimeout == nil {
-		conn, err = net.DialTimeout(network, d.ResolvedAddr.String(), sctx.cfg.ConnectTimeout)
+		dialer := net.Dialer{
+			Timeout: sctx.cfg.ConnectTimeout,
+			Control: tcpUserTimeoutControl(sctx.cfg.UpstreamTCPUserTimeout),
+		}
+		conn, err = dialer.Dial(network, d.ResolvedAddr.String())
 	} else {
 		conn, err = sctx.cfg.ProxyDialTimeout(ctx, network, d.ResolvedAddr.String(), sctx.cfg.ConnectTimeout)
 	}