diff --git a/infra/conf/transport_method.go b/infra/conf/transport_method.go index d36d6e80952c..3eac4404ed70 100644 --- a/infra/conf/transport_method.go +++ b/infra/conf/transport_method.go @@ -292,6 +292,7 @@ type SplitHTTPConfig struct { ScStreamUpServerSecs Int32Range `json:"scStreamUpServerSecs"` ServerMaxHeaderBytes int32 `json:"serverMaxHeaderBytes"` Xmux XmuxConfig `json:"xmux"` + PacketDown *PacketDownConfig `json:"packetDown"` DownloadSettings *StreamConfig `json:"downloadSettings"` Extra json.RawMessage `json:"extra"` } @@ -305,6 +306,12 @@ type XmuxConfig struct { HKeepAlivePeriod int64 `json:"hKeepAlivePeriod"` } +type PacketDownConfig struct { + KeepAlive bool `json:"keepAlive"` + KeepAlivePeriod Int32Range `json:"keepAlivePeriod"` + KeepAliveLength Int32Range `json:"keepAliveLength"` +} + func newRangeConfig(input Int32Range) *splithttp.RangeConfig { return &splithttp.RangeConfig{ From: input.From, @@ -505,6 +512,17 @@ func (c *SplitHTTPConfig) Build() (proto.Message, error) { }, } + if c.PacketDown != nil { + if c.PacketDown.KeepAlivePeriod.From < 0 || c.PacketDown.KeepAliveLength.From < 0 { + return nil, errors.New("invalid negative value in packetDown") + } + config.PacketDown = &splithttp.PacketDownConfig{ + KeepAlive: c.PacketDown.KeepAlive, + KeepAlivePeriod: newRangeConfig(c.PacketDown.KeepAlivePeriod), + KeepAliveLength: newRangeConfig(c.PacketDown.KeepAliveLength), + } + } + if c.DownloadSettings != nil { if c.Mode == "stream-one" { return nil, errors.New(`Can not use "downloadSettings" in "stream-one" mode.`) diff --git a/infra/conf/transport_method_test.go b/infra/conf/transport_method_test.go new file mode 100644 index 000000000000..f4cf4099823a --- /dev/null +++ b/infra/conf/transport_method_test.go @@ -0,0 +1,64 @@ +package conf_test + +import ( + "encoding/json" + "testing" + + "github.com/google/go-cmp/cmp" + . "github.com/xtls/xray-core/infra/conf" + "github.com/xtls/xray-core/transport/internet/splithttp" +) + +func Test_SplitHTTP_PacketDown(t *testing.T) { + rawJSON := `{ + "path": "/sh", + "packetDown": { "keepAlive": true, "keepAlivePeriod": "5-10", "keepAliveLength": 32 } + }` + cfg := new(SplitHTTPConfig) + if err := json.Unmarshal([]byte(rawJSON), cfg); err != nil { + t.Fatal(err) + } + built, err := cfg.Build() + if err != nil { + t.Fatal(err) + } + + pd := built.(*splithttp.Config).GetPacketDown() + if pd == nil { + t.Fatal("packetDown was not built") + } + if !pd.GetKeepAlive() { + t.Error("packetDown.keepAlive should be true") + } + if r := cmp.Diff(int32(5), pd.GetKeepAlivePeriod().GetFrom()); r != "" { + t.Error(r) + } + if r := cmp.Diff(int32(32), pd.GetKeepAliveLength().GetTo()); r != "" { + t.Error(r) + } +} + +func Test_SplitHTTP_PacketDownNegative(t *testing.T) { + rawJSON := `{ "path": "/sh", "packetDown": { "keepAlive": true, "keepAlivePeriod": "-1-10" } }` + cfg := new(SplitHTTPConfig) + parseErr := json.Unmarshal([]byte(rawJSON), cfg) + _, buildErr := cfg.Build() + if parseErr == nil && buildErr == nil { + t.Error("expected negative period to fail at least one stage") + } +} + +func Test_SplitHTTP_PacketDownAbsent(t *testing.T) { + rawJSON := `{ "path": "/sh" }` + cfg := new(SplitHTTPConfig) + if err := json.Unmarshal([]byte(rawJSON), cfg); err != nil { + t.Fatal(err) + } + built, err := cfg.Build() + if err != nil { + t.Fatal(err) + } + if built.(*splithttp.Config).GetPacketDown() != nil { + t.Error("packetDown should stay unset when absent") + } +} diff --git a/transport/internet/splithttp/client.go b/transport/internet/splithttp/client.go index a2b5f75e87a5..c2e7c65ab98b 100644 --- a/transport/internet/splithttp/client.go +++ b/transport/internet/splithttp/client.go @@ -68,6 +68,15 @@ func (c *DefaultDialerClient) OpenStream(ctx context.Context, url string, sessio } c.transportConfig.FillStreamRequest(req, sessionId, "") + // stream-down and stream-one responses carry the downlink; ask the + // server to run packet-down in it. stream-up responses are discarded + // by this client, so they don't need packet-down. + downlinkMaybeFramed := false + if !uploadOnly && c.transportConfig.PacketDown != nil { + req.Header.Set(pdRequestHeader, "1") + downlinkMaybeFramed = true + } + wrc = &WaitReadCloser{wait: done.New()} go func() { resp, err := c.client.Do(req) @@ -91,7 +100,13 @@ func (c *DefaultDialerClient) OpenStream(ctx context.Context, url string, sessio wrc.Close() return } - wrc.(*WaitReadCloser).Set(resp.Body) + if downlinkMaybeFramed && resp.Header.Get(pdResponseHeader) == "1" { + // the body is packet-down frames; strip them before the + // payload reaches the dispatcher + wrc.(*WaitReadCloser).Set(newPacketDownReader(resp.Body)) + } else { + wrc.(*WaitReadCloser).Set(resp.Body) + } }() <-gotConn.Wait() diff --git a/transport/internet/splithttp/config.go b/transport/internet/splithttp/config.go index e06281b1b2d0..8e27a76fe2d4 100644 --- a/transport/internet/splithttp/config.go +++ b/transport/internet/splithttp/config.go @@ -427,6 +427,40 @@ func (c *Config) ExtractMetaFromRequest(req *http.Request, path string) (session return sessionId, seqStr } +func (c *PacketDownConfig) GetNormalizedKeepAlivePeriod() *RangeConfig { + if c.KeepAlivePeriod == nil || c.KeepAlivePeriod.To == 0 { + return &RangeConfig{ + From: 10, + To: 30, + } + } + + return c.KeepAlivePeriod +} + +func (c *PacketDownConfig) GetNormalizedKeepAliveLength() *RangeConfig { + if c.KeepAliveLength == nil || c.KeepAliveLength.To == 0 { + return &RangeConfig{ + From: 24, + To: 64, + } + } + + if c.KeepAliveLength.From > pdMaxFramePayload { + return &RangeConfig{ + From: pdMaxFramePayload, + To: pdMaxFramePayload, + } + } else if c.KeepAliveLength.To > pdMaxFramePayload { + return &RangeConfig{ + From: c.KeepAliveLength.From, + To: pdMaxFramePayload, + } + } + + return c.KeepAliveLength +} + func (m *XmuxConfig) GetNormalizedMaxConcurrency() *RangeConfig { if m.MaxConcurrency == nil { return &RangeConfig{ diff --git a/transport/internet/splithttp/config.pb.go b/transport/internet/splithttp/config.pb.go index c7dfe4ce8ca9..8b53bb8e1a05 100644 --- a/transport/internet/splithttp/config.pb.go +++ b/transport/internet/splithttp/config.pb.go @@ -158,6 +158,66 @@ func (x *XmuxConfig) GetHKeepAlivePeriod() int64 { return 0 } +type PacketDownConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + KeepAlive bool `protobuf:"varint,1,opt,name=keepAlive,proto3" json:"keepAlive,omitempty"` + KeepAlivePeriod *RangeConfig `protobuf:"bytes,2,opt,name=keepAlivePeriod,proto3" json:"keepAlivePeriod,omitempty"` + KeepAliveLength *RangeConfig `protobuf:"bytes,3,opt,name=keepAliveLength,proto3" json:"keepAliveLength,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *PacketDownConfig) Reset() { + *x = PacketDownConfig{} + mi := &file_transport_internet_splithttp_config_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *PacketDownConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PacketDownConfig) ProtoMessage() {} + +func (x *PacketDownConfig) ProtoReflect() protoreflect.Message { + mi := &file_transport_internet_splithttp_config_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PacketDownConfig.ProtoReflect.Descriptor instead. +func (*PacketDownConfig) Descriptor() ([]byte, []int) { + return file_transport_internet_splithttp_config_proto_rawDescGZIP(), []int{2} +} + +func (x *PacketDownConfig) GetKeepAlive() bool { + if x != nil { + return x.KeepAlive + } + return false +} + +func (x *PacketDownConfig) GetKeepAlivePeriod() *RangeConfig { + if x != nil { + return x.KeepAlivePeriod + } + return nil +} + +func (x *PacketDownConfig) GetKeepAliveLength() *RangeConfig { + if x != nil { + return x.KeepAliveLength + } + return nil +} + type Config struct { state protoimpl.MessageState `protogen:"open.v1"` Host string `protobuf:"bytes,1,opt,name=host,proto3" json:"host,omitempty"` @@ -189,13 +249,14 @@ type Config struct { ServerMaxHeaderBytes int32 `protobuf:"varint,27,opt,name=serverMaxHeaderBytes,proto3" json:"serverMaxHeaderBytes,omitempty"` SessionIDTable string `protobuf:"bytes,28,opt,name=sessionIDTable,proto3" json:"sessionIDTable,omitempty"` SessionIDLength *RangeConfig `protobuf:"bytes,29,opt,name=sessionIDLength,proto3" json:"sessionIDLength,omitempty"` + PacketDown *PacketDownConfig `protobuf:"bytes,30,opt,name=packetDown,proto3" json:"packetDown,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } func (x *Config) Reset() { *x = Config{} - mi := &file_transport_internet_splithttp_config_proto_msgTypes[2] + mi := &file_transport_internet_splithttp_config_proto_msgTypes[3] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -207,7 +268,7 @@ func (x *Config) String() string { func (*Config) ProtoMessage() {} func (x *Config) ProtoReflect() protoreflect.Message { - mi := &file_transport_internet_splithttp_config_proto_msgTypes[2] + mi := &file_transport_internet_splithttp_config_proto_msgTypes[3] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -220,7 +281,7 @@ func (x *Config) ProtoReflect() protoreflect.Message { // Deprecated: Use Config.ProtoReflect.Descriptor instead. func (*Config) Descriptor() ([]byte, []int) { - return file_transport_internet_splithttp_config_proto_rawDescGZIP(), []int{2} + return file_transport_internet_splithttp_config_proto_rawDescGZIP(), []int{3} } func (x *Config) GetHost() string { @@ -426,6 +487,13 @@ func (x *Config) GetSessionIDLength() *RangeConfig { return nil } +func (x *Config) GetPacketDown() *PacketDownConfig { + if x != nil { + return x.PacketDown + } + return nil +} + var File_transport_internet_splithttp_config_proto protoreflect.FileDescriptor const file_transport_internet_splithttp_config_proto_rawDesc = "" + @@ -441,7 +509,11 @@ const file_transport_internet_splithttp_config_proto_rawDesc = "" + "\x0ecMaxReuseTimes\x18\x03 \x01(\v2..xray.transport.internet.splithttp.RangeConfigR\x0ecMaxReuseTimes\x12Z\n" + "\x10hMaxRequestTimes\x18\x04 \x01(\v2..xray.transport.internet.splithttp.RangeConfigR\x10hMaxRequestTimes\x12Z\n" + "\x10hMaxReusableSecs\x18\x05 \x01(\v2..xray.transport.internet.splithttp.RangeConfigR\x10hMaxReusableSecs\x12*\n" + - "\x10hKeepAlivePeriod\x18\x06 \x01(\x03R\x10hKeepAlivePeriod\"\xcc\f\n" + + "\x10hKeepAlivePeriod\x18\x06 \x01(\x03R\x10hKeepAlivePeriod\"\xe4\x01\n" + + "\x10PacketDownConfig\x12\x1c\n" + + "\tkeepAlive\x18\x01 \x01(\bR\tkeepAlive\x12X\n" + + "\x0fkeepAlivePeriod\x18\x02 \x01(\v2..xray.transport.internet.splithttp.RangeConfigR\x0fkeepAlivePeriod\x12X\n" + + "\x0fkeepAliveLength\x18\x03 \x01(\v2..xray.transport.internet.splithttp.RangeConfigR\x0fkeepAliveLength\"\xa1\r\n" + "\x06Config\x12\x12\n" + "\x04host\x18\x01 \x01(\tR\x04host\x12\x12\n" + "\x04path\x18\x02 \x01(\tR\x04path\x12\x12\n" + @@ -472,7 +544,10 @@ const file_transport_internet_splithttp_config_proto_rawDesc = "" + "\x0fuplinkChunkSize\x18\x1a \x01(\v2..xray.transport.internet.splithttp.RangeConfigR\x0fuplinkChunkSize\x122\n" + "\x14serverMaxHeaderBytes\x18\x1b \x01(\x05R\x14serverMaxHeaderBytes\x12&\n" + "\x0esessionIDTable\x18\x1c \x01(\tR\x0esessionIDTable\x12X\n" + - "\x0fsessionIDLength\x18\x1d \x01(\v2..xray.transport.internet.splithttp.RangeConfigR\x0fsessionIDLength\x1a:\n" + + "\x0fsessionIDLength\x18\x1d \x01(\v2..xray.transport.internet.splithttp.RangeConfigR\x0fsessionIDLength\x12S\n" + + "\n" + + "packetDown\x18\x1e \x01(\v23.xray.transport.internet.splithttp.PacketDownConfigR\n" + + "packetDown\x1a:\n" + "\fHeadersEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01B\x85\x01\n" + @@ -490,13 +565,14 @@ func file_transport_internet_splithttp_config_proto_rawDescGZIP() []byte { return file_transport_internet_splithttp_config_proto_rawDescData } -var file_transport_internet_splithttp_config_proto_msgTypes = make([]protoimpl.MessageInfo, 4) +var file_transport_internet_splithttp_config_proto_msgTypes = make([]protoimpl.MessageInfo, 5) var file_transport_internet_splithttp_config_proto_goTypes = []any{ (*RangeConfig)(nil), // 0: xray.transport.internet.splithttp.RangeConfig (*XmuxConfig)(nil), // 1: xray.transport.internet.splithttp.XmuxConfig - (*Config)(nil), // 2: xray.transport.internet.splithttp.Config - nil, // 3: xray.transport.internet.splithttp.Config.HeadersEntry - (*internet.StreamConfig)(nil), // 4: xray.transport.internet.StreamConfig + (*PacketDownConfig)(nil), // 2: xray.transport.internet.splithttp.PacketDownConfig + (*Config)(nil), // 3: xray.transport.internet.splithttp.Config + nil, // 4: xray.transport.internet.splithttp.Config.HeadersEntry + (*internet.StreamConfig)(nil), // 5: xray.transport.internet.StreamConfig } var file_transport_internet_splithttp_config_proto_depIdxs = []int32{ 0, // 0: xray.transport.internet.splithttp.XmuxConfig.maxConcurrency:type_name -> xray.transport.internet.splithttp.RangeConfig @@ -504,20 +580,23 @@ var file_transport_internet_splithttp_config_proto_depIdxs = []int32{ 0, // 2: xray.transport.internet.splithttp.XmuxConfig.cMaxReuseTimes:type_name -> xray.transport.internet.splithttp.RangeConfig 0, // 3: xray.transport.internet.splithttp.XmuxConfig.hMaxRequestTimes:type_name -> xray.transport.internet.splithttp.RangeConfig 0, // 4: xray.transport.internet.splithttp.XmuxConfig.hMaxReusableSecs:type_name -> xray.transport.internet.splithttp.RangeConfig - 3, // 5: xray.transport.internet.splithttp.Config.headers:type_name -> xray.transport.internet.splithttp.Config.HeadersEntry - 0, // 6: xray.transport.internet.splithttp.Config.xPaddingBytes:type_name -> xray.transport.internet.splithttp.RangeConfig - 0, // 7: xray.transport.internet.splithttp.Config.scMaxEachPostBytes:type_name -> xray.transport.internet.splithttp.RangeConfig - 0, // 8: xray.transport.internet.splithttp.Config.scMinPostsIntervalMs:type_name -> xray.transport.internet.splithttp.RangeConfig - 0, // 9: xray.transport.internet.splithttp.Config.scStreamUpServerSecs:type_name -> xray.transport.internet.splithttp.RangeConfig - 1, // 10: xray.transport.internet.splithttp.Config.xmux:type_name -> xray.transport.internet.splithttp.XmuxConfig - 4, // 11: xray.transport.internet.splithttp.Config.downloadSettings:type_name -> xray.transport.internet.StreamConfig - 0, // 12: xray.transport.internet.splithttp.Config.uplinkChunkSize:type_name -> xray.transport.internet.splithttp.RangeConfig - 0, // 13: xray.transport.internet.splithttp.Config.sessionIDLength:type_name -> xray.transport.internet.splithttp.RangeConfig - 14, // [14:14] is the sub-list for method output_type - 14, // [14:14] is the sub-list for method input_type - 14, // [14:14] is the sub-list for extension type_name - 14, // [14:14] is the sub-list for extension extendee - 0, // [0:14] is the sub-list for field type_name + 0, // 5: xray.transport.internet.splithttp.PacketDownConfig.keepAlivePeriod:type_name -> xray.transport.internet.splithttp.RangeConfig + 0, // 6: xray.transport.internet.splithttp.PacketDownConfig.keepAliveLength:type_name -> xray.transport.internet.splithttp.RangeConfig + 4, // 7: xray.transport.internet.splithttp.Config.headers:type_name -> xray.transport.internet.splithttp.Config.HeadersEntry + 0, // 8: xray.transport.internet.splithttp.Config.xPaddingBytes:type_name -> xray.transport.internet.splithttp.RangeConfig + 0, // 9: xray.transport.internet.splithttp.Config.scMaxEachPostBytes:type_name -> xray.transport.internet.splithttp.RangeConfig + 0, // 10: xray.transport.internet.splithttp.Config.scMinPostsIntervalMs:type_name -> xray.transport.internet.splithttp.RangeConfig + 0, // 11: xray.transport.internet.splithttp.Config.scStreamUpServerSecs:type_name -> xray.transport.internet.splithttp.RangeConfig + 1, // 12: xray.transport.internet.splithttp.Config.xmux:type_name -> xray.transport.internet.splithttp.XmuxConfig + 5, // 13: xray.transport.internet.splithttp.Config.downloadSettings:type_name -> xray.transport.internet.StreamConfig + 0, // 14: xray.transport.internet.splithttp.Config.uplinkChunkSize:type_name -> xray.transport.internet.splithttp.RangeConfig + 0, // 15: xray.transport.internet.splithttp.Config.sessionIDLength:type_name -> xray.transport.internet.splithttp.RangeConfig + 2, // 16: xray.transport.internet.splithttp.Config.packetDown:type_name -> xray.transport.internet.splithttp.PacketDownConfig + 17, // [17:17] is the sub-list for method output_type + 17, // [17:17] is the sub-list for method input_type + 17, // [17:17] is the sub-list for extension type_name + 17, // [17:17] is the sub-list for extension extendee + 0, // [0:17] is the sub-list for field type_name } func init() { file_transport_internet_splithttp_config_proto_init() } @@ -531,7 +610,7 @@ func file_transport_internet_splithttp_config_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_transport_internet_splithttp_config_proto_rawDesc), len(file_transport_internet_splithttp_config_proto_rawDesc)), NumEnums: 0, - NumMessages: 4, + NumMessages: 5, NumExtensions: 0, NumServices: 0, }, diff --git a/transport/internet/splithttp/config.proto b/transport/internet/splithttp/config.proto index efe6de03f28d..d01d725f1c04 100644 --- a/transport/internet/splithttp/config.proto +++ b/transport/internet/splithttp/config.proto @@ -22,6 +22,12 @@ message XmuxConfig { int64 hKeepAlivePeriod = 6; } +message PacketDownConfig { + bool keepAlive = 1; + RangeConfig keepAlivePeriod = 2; + RangeConfig keepAliveLength = 3; +} + message Config { string host = 1; string path = 2; @@ -52,4 +58,5 @@ message Config { int32 serverMaxHeaderBytes = 27; string sessionIDTable = 28; RangeConfig sessionIDLength = 29; + PacketDownConfig packetDown = 30; } diff --git a/transport/internet/splithttp/hub.go b/transport/internet/splithttp/hub.go index c4573b96130e..a41c0e46c88d 100644 --- a/transport/internet/splithttp/hub.go +++ b/transport/internet/splithttp/hub.go @@ -361,6 +361,14 @@ func (h *requestHandler) ServeHTTP(writer http.ResponseWriter, request *http.Req // teeing the response stream into their cache, causing slowdowns. writer.Header().Set("Cache-Control", "no-store") + // packet-down runs in this response body, if we have the feature on + // and the client asked for it. only round-trip agreement routes the + // body through the framers. + usePacketDown := h.config.PacketDown != nil && request.Header.Get(pdRequestHeader) == "1" + if usePacketDown { + writer.Header().Set(pdResponseHeader, "1") + } + if !h.config.NoSSEHeader { // magic header to make the HTTP middle box consider this as SSE to disable buffer writer.Header().Set("Content-Type", "text/event-stream") @@ -374,12 +382,19 @@ func (h *requestHandler) ServeHTTP(writer http.ResponseWriter, request *http.Req Reader: request.Body, ResponseWriter: writer, } + var downlinkWriter io.WriteCloser = httpSC + var packetDownWriterInstance *packetDownWriter + if usePacketDown { + packetDownWriterInstance = newPacketDownWriter(httpSC, sessionId, h.config.PacketDown) + defer packetDownWriterInstance.Stop() + downlinkWriter = packetDownWriterInstance + } localAddr := h.localAddr if la, ok := request.Context().Value(http.LocalAddrContextKey).(net.Addr); ok && la != nil { localAddr = la } conn := splitConn{ - writer: httpSC, + writer: downlinkWriter, reader: httpSC, remoteAddr: remoteAddr, localAddr: localAddr, diff --git a/transport/internet/splithttp/packet_down.go b/transport/internet/splithttp/packet_down.go new file mode 100644 index 000000000000..a0c5f769ea5b --- /dev/null +++ b/transport/internet/splithttp/packet_down.go @@ -0,0 +1,287 @@ +package splithttp + +import ( + "bufio" + "encoding/binary" + "io" + "sync" + "sync/atomic" + "time" + + "github.com/xtls/xray-core/common/errors" +) + +// packet-down runs in stream-down: the response body is cut into frames, and +// while the downlink is idle the server sends keep-alive padding frames, so a +// middlebox cannot kill the GET for inactivity while the uplink is the only +// side carrying traffic. See #4846 for the problem and the packet-down design +// in #6755: an omittable session id excerpt, an omittable packet seq, a +// mandatory type (data/padding), a mandatory length, then the payload. The +// seq and the excerpt keep the format ready for future multi-path-down and +// retransmission without changing the wire again. +const ( + pdRequestHeader = "X-Packet-Down" // client -> server: run packet-down in this stream-down + pdResponseHeader = "X-Packet-Down" // server -> client: confirmation, the body is framed + + pdTypeData byte = 0x01 + pdTypePadding byte = 0x02 + + pdFlagSessionID byte = 0x01 + pdFlagSeq byte = 0x02 + + pdSessionIDExcerptLen = 8 + pdMaxFramePayload = 0xFFFF +) + +type pdFrame struct { + sessionIDExcerpt []byte + seq uint64 + hasSeq bool + typ byte + payload []byte +} + +func pdWriteFrame(w io.Writer, f pdFrame) error { + if len(f.payload) > pdMaxFramePayload { + return errors.New("packet-down frame payload is too large: ", len(f.payload)) + } + + flags := byte(0) + if len(f.sessionIDExcerpt) > 0 { + flags |= pdFlagSessionID + } + if f.hasSeq { + flags |= pdFlagSeq + } + + header := make([]byte, 4, 4+len(f.sessionIDExcerpt)+8+len(f.payload)) + header[0] = flags + header[1] = f.typ + binary.BigEndian.PutUint16(header[2:4], uint16(len(f.payload))) + if len(f.sessionIDExcerpt) > 0 { + header = append(header, f.sessionIDExcerpt...) + } + if f.hasSeq { + var seq [8]byte + binary.BigEndian.PutUint64(seq[:], f.seq) + header = append(header, seq[:]...) + } + + _, err := w.Write(append(header, f.payload...)) + return err +} + +func pdReadFrame(r *bufio.Reader) (pdFrame, error) { + header := make([]byte, 4) + if _, err := io.ReadFull(r, header); err != nil { + if err == io.EOF { + return pdFrame{}, io.EOF + } + return pdFrame{}, errors.New("truncated packet-down frame header").Base(err) + } + + flags := header[0] + typ := header[1] + length := binary.BigEndian.Uint16(header[2:4]) + + if typ != pdTypeData && typ != pdTypePadding { + return pdFrame{}, errors.New("unknown packet-down frame type: ", typ) + } + + f := pdFrame{ + typ: typ, + } + + if flags&pdFlagSessionID != 0 { + excerpt := make([]byte, pdSessionIDExcerptLen) + if _, err := io.ReadFull(r, excerpt); err != nil { + return pdFrame{}, errors.New("truncated packet-down session id excerpt").Base(err) + } + f.sessionIDExcerpt = excerpt + } + + if flags&pdFlagSeq != 0 { + seq := make([]byte, 8) + if _, err := io.ReadFull(r, seq); err != nil { + return pdFrame{}, errors.New("truncated packet-down seq").Base(err) + } + f.seq = binary.BigEndian.Uint64(seq) + f.hasSeq = true + } + + payload := make([]byte, length) + if _, err := io.ReadFull(r, payload); err != nil { + return pdFrame{}, errors.New("truncated packet-down frame payload").Base(err) + } + f.payload = payload + + return f, nil +} + +func pdSessionIDExcerpt(sessionID string) []byte { + if len(sessionID) < pdSessionIDExcerptLen { + return nil + } + return []byte(sessionID[:pdSessionIDExcerptLen]) +} + +// packetDownWriter cuts everything the dispatcher sends into data frames, and +// a goroutine sends a keep-alive padding frame whenever the downlink has been +// idle for a whole period. the padding is base62, in the spirit of the +// tokenish x-padding, so the extra bytes look like ordinary SSE content. +type packetDownWriter struct { + mu sync.Mutex + writer io.Writer + sessionID string + pdConfig *PacketDownConfig + seq uint64 + lastWrite time.Time + stopped atomic.Bool +} + +func newPacketDownWriter(w io.Writer, sessionId string, pdConfig *PacketDownConfig) *packetDownWriter { + d := &packetDownWriter{ + writer: w, + sessionID: sessionId, + pdConfig: pdConfig, + lastWrite: time.Now(), + } + if pdConfig.GetKeepAlive() { + go d.keepAliveLoop() + } + return d +} + +func (d *packetDownWriter) Write(b []byte) (int, error) { + d.mu.Lock() + defer d.mu.Unlock() + + d.lastWrite = time.Now() + written := 0 + excerpt := pdSessionIDExcerpt(d.sessionID) + seq := d.seq + hasSeq := len(excerpt) > 0 + for len(b) > 0 { + payloadLength := len(b) + if payloadLength > pdMaxFramePayload { + payloadLength = pdMaxFramePayload + } + err := pdWriteFrame(d.writer, pdFrame{ + sessionIDExcerpt: excerpt, + seq: seq, + hasSeq: hasSeq, + typ: pdTypeData, + payload: b[:payloadLength], + }) + if err != nil { + return written, err + } + d.seq++ + seq++ + b = b[payloadLength:] + written += payloadLength + } + return written, nil +} + +func (d *packetDownWriter) keepAliveLoop() { + for { + period := time.Duration(d.pdConfig.GetNormalizedKeepAlivePeriod().rand()) * time.Second + time.Sleep(period) + if d.stopped.Load() { + return + } + + d.mu.Lock() + if time.Since(d.lastWrite) < period { + // the dispatcher was active within this period, there is + // nothing to keep alive + d.mu.Unlock() + continue + } + length := int(d.pdConfig.GetNormalizedKeepAliveLength().rand()) + padding, _ := randStringFromCharset(length, charsetBase62) + if padding == "" { + padding = "X" + } + excerpt := pdSessionIDExcerpt(d.sessionID) + seq := d.seq + hasSeq := len(excerpt) > 0 + d.seq++ + d.lastWrite = time.Now() + err := pdWriteFrame(d.writer, pdFrame{ + sessionIDExcerpt: excerpt, + seq: seq, + hasSeq: hasSeq, + typ: pdTypePadding, + payload: []byte(padding), + }) + d.mu.Unlock() + if err != nil { + return + } + } +} + +// Stop ends the goroutine; the writer keeps working until the underlying +// stream is closed. +func (d *packetDownWriter) Stop() { + d.stopped.Store(true) +} + +// Close passes through to the underlying stream, holding the lock so that a +// final data write cannot interleave with the connection teardown. +func (d *packetDownWriter) Close() error { + d.stopped.Store(true) + + closeFunc, ok := d.writer.(io.Closer) + if !ok { + return nil + } + d.mu.Lock() + defer d.mu.Unlock() + return closeFunc.Close() +} + +// packetDownReader demuxes a framed stream-down body: payload arrives the way +// it did before, padding frames are consumed and dropped. +type packetDownReader struct { + reader io.ReadCloser + buffered *bufio.Reader + pending []byte +} + +func newPacketDownReader(rc io.ReadCloser) *packetDownReader { + return &packetDownReader{ + reader: rc, + buffered: bufio.NewReaderSize(rc, bufSizeForDown), + } +} + +const bufSizeForDown = 16384 + +func (d *packetDownReader) Read(b []byte) (int, error) { + for len(d.pending) == 0 { + frame, err := pdReadFrame(d.buffered) + if err != nil { + return 0, err + } + switch frame.typ { + case pdTypeData: + d.pending = frame.payload + case pdTypePadding: + default: + return 0, errors.New("unknown packet-down frame type: ", frame.typ) + } + } + n := copy(b, d.pending) + d.pending = d.pending[n:] + if len(d.pending) == 0 { + d.pending = nil + } + return n, nil +} + +func (d *packetDownReader) Close() error { + return d.reader.Close() +} diff --git a/transport/internet/splithttp/packet_down_test.go b/transport/internet/splithttp/packet_down_test.go new file mode 100644 index 000000000000..05bf055fdc54 --- /dev/null +++ b/transport/internet/splithttp/packet_down_test.go @@ -0,0 +1,265 @@ +package splithttp + +import ( + "bufio" + "bytes" + "crypto/rand" + "io" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func pdTestConfig(period int32) *PacketDownConfig { + return &PacketDownConfig{ + KeepAlive: true, + KeepAlivePeriod: &RangeConfig{From: period, To: period}, + KeepAliveLength: &RangeConfig{From: 32, To: 32}, + } +} + +// newSlowReader feeds a reader out in fixed-size chunks, like a socket does, +// to make sure the frame reader does not assume whole frames per Read. +type pdChunkReader struct { + r io.Reader + chunk int +} + +func (r *pdChunkReader) Read(b []byte) (int, error) { + if len(b) > r.chunk { + b = b[:r.chunk] + } + return r.r.Read(b) +} + +func newPDClosableReader(r io.Reader) io.ReadCloser { + return io.NopCloser(r) +} + +func newFrameBufReader(b []byte) *bufio.Reader { + return bufio.NewReader(bytes.NewReader(b)) +} + +// a bytes.Buffer behind a mutex: the keep-alive goroutine may still be +// writing while the test reads +type pdLockedBuffer struct { + mu sync.Mutex + b bytes.Buffer +} + +func (t *pdLockedBuffer) Write(p []byte) (int, error) { + t.mu.Lock() + defer t.mu.Unlock() + return t.b.Write(p) +} + +func (t *pdLockedBuffer) Len() int { + t.mu.Lock() + defer t.mu.Unlock() + return t.b.Len() +} + +func (t *pdLockedBuffer) Bytes() []byte { + t.mu.Lock() + defer t.mu.Unlock() + return t.b.Bytes() +} + +func Test_PacketDownFrameRoundTrip(t *testing.T) { + payload := make([]byte, 3000) + rand.Read(payload) + + sessionID := "session_test_SOMETHING_HIGH" + var buf bytes.Buffer + + err := pdWriteFrame(&buf, pdFrame{ + sessionIDExcerpt: pdSessionIDExcerpt(sessionID), + seq: 42, + hasSeq: true, + typ: pdTypeData, + payload: payload, + }) + assert.NoError(t, err) + + frame, err := pdReadFrame(newFrameBufReader(buf.Bytes())) + assert.NoError(t, err) + assert.Equal(t, pdTypeData, frame.typ) + assert.Equal(t, sessionID[:pdSessionIDExcerptLen], string(frame.sessionIDExcerpt)) + assert.Equal(t, uint64(42), frame.seq) + assert.True(t, frame.hasSeq) + assert.Equal(t, payload, frame.payload) +} + +func Test_PacketDownKeepAliveRoundTrip(t *testing.T) { + padding := bytes.Repeat([]byte{'A'}, 32) + + var buf bytes.Buffer + err := pdWriteFrame(&buf, pdFrame{ + typ: pdTypePadding, + payload: padding, + }) + assert.NoError(t, err) + + frame, err := pdReadFrame(newFrameBufReader(buf.Bytes())) + assert.NoError(t, err) + assert.Equal(t, pdTypePadding, frame.typ) + assert.Empty(t, frame.sessionIDExcerpt) + assert.False(t, frame.hasSeq) + assert.Equal(t, padding, frame.payload) +} + +func Test_PacketDownKeepAliveHasNoExcerptWithoutSession(t *testing.T) { + // stream-one has an empty session id; the excerpt stays out of the frame + var buf bytes.Buffer + err := pdWriteFrame(&buf, pdFrame{ + sessionIDExcerpt: pdSessionIDExcerpt(""), + seq: 0, + hasSeq: true, + typ: pdTypeData, + payload: []byte("x"), + }) + assert.NoError(t, err) + + frame, err := pdReadFrame(newFrameBufReader(buf.Bytes())) + assert.NoError(t, err) + assert.Empty(t, frame.sessionIDExcerpt) + assert.True(t, frame.hasSeq) + assert.Equal(t, []byte("x"), frame.payload) +} + +func Test_PacketDownWriterReaderEndToEnd(t *testing.T) { + sessionID := "abcdef0123456789" + fiBuffer := &pdLockedBuffer{} + + d := newPacketDownWriter(fiBuffer, sessionID, pdTestConfig(1000)) + d.Stop() + + payload := make([]byte, 70000) + rand.Read(payload) + _, err := d.Write(payload) + assert.NoError(t, err) + + // feed the framed bytes back through the reader in 1 KiB steps + pr := newPacketDownReader(newPDClosableReader(&pdChunkReader{r: bytes.NewReader(fiBuffer.Bytes()), chunk: 1024})) + got, err := io.ReadAll(pr) + assert.NoError(t, err) + assert.Equal(t, payload, got) +} + +func Test_PacketDownWriterSplitsLargePayload(t *testing.T) { + fiBuffer := &pdLockedBuffer{} + + d := newPacketDownWriter(fiBuffer, "abcdefgh", pdTestConfig(1000)) + d.Stop() + + payload := make([]byte, pdMaxFramePayload+1234) + rand.Read(payload) + _, err := d.Write(payload) + assert.NoError(t, err) + + // 65536 + 1234 payload cut into two frames; walk the wire and check + // the frames line up + r := newFrameBufReader(fiBuffer.Bytes()) + var payloads []byte + seq := uint64(0) + for { + frame, err := pdReadFrame(r) + if err != nil { + break + } + assert.Equal(t, pdTypeData, frame.typ) + assert.Equal(t, seq, frame.seq) + assert.Equal(t, "abcdefgh", string(frame.sessionIDExcerpt)) + payloads = append(payloads, frame.payload...) + seq++ + } + assert.Equal(t, uint64(2), seq) + assert.Equal(t, payload, payloads) +} + +func Test_PacketsDownReaderSkipsPadding(t *testing.T) { + fiBuffer := &bytes.Buffer{} + + assert.NoError(t, pdWriteFrame(fiBuffer, pdFrame{typ: pdTypePadding, payload: []byte("padding-padding-padding")})) + assert.NoError(t, pdWriteFrame(fiBuffer, pdFrame{typ: pdTypeData, hasSeq: true, seq: 0, payload: []byte("hello")})) + assert.NoError(t, pdWriteFrame(fiBuffer, pdFrame{typ: pdTypePadding, payload: []byte("more-padding-more-padding")})) + assert.NoError(t, pdWriteFrame(fiBuffer, pdFrame{typ: pdTypeData, hasSeq: true, seq: 1, payload: []byte("world")})) + + pr := newPacketDownReader(newPDClosableReader(fiBuffer)) + out, err := io.ReadAll(pr) + assert.NoError(t, err) + assert.Equal(t, "helloworld", string(out)) +} + +func Test_PacketDownReaderMalformedFrame(t *testing.T) { + pr := newPacketDownReader(newPDClosableReader(bytes.NewReader([]byte{0x00, 0x09, 0x00}))) + _, err := pr.Read(make([]byte, 16)) + assert.Error(t, err) +} + +func Test_PacketDownEOF(t *testing.T) { + pr := newPacketDownReader(newPDClosableReader(bytes.NewReader(nil))) + _, err := pr.Read(make([]byte, 16)) + assert.Equal(t, io.EOF, err) +} + +func Test_PacketDownKeepAliveWritesWhenIdle(t *testing.T) { + fiBuffer := &pdLockedBuffer{} + + d := newPacketDownWriter(fiBuffer, "wantpaddingxx", pdTestConfig(1)) + defer d.Stop() + + deadline := time.Now().Add(5 * time.Second) + for fiBuffer.Len() == 0 { + if time.Now().After(deadline) { + t.Fatal("no keep-alive frame was written within 5 s") + } + time.Sleep(50 * time.Millisecond) + } + + frame, err := pdReadFrame(newFrameBufReader(fiBuffer.Bytes())) + assert.NoError(t, err) + assert.Equal(t, pdTypePadding, frame.typ) + assert.Equal(t, "wantpadd", string(frame.sessionIDExcerpt)) + assert.True(t, frame.hasSeq) + assert.Equal(t, 32, len(frame.payload)) +} + +func Test_PacketDownKeepAliveSkippedWhenActive(t *testing.T) { + fiBuffer := &pdLockedBuffer{} + + d := newPacketDownWriter(fiBuffer, "active", pdTestConfig(1)) + defer d.Stop() + + deadline := time.Now().Add(3 * time.Second) + for time.Now().Before(deadline) { + _, err := d.Write([]byte("whee")) + assert.NoError(t, err) + time.Sleep(200 * time.Millisecond) + } + + // walk every frame on the wire: all of them must be data, none padding + r := newFrameBufReader(fiBuffer.Bytes()) + for { + frame, err := pdReadFrame(r) + if err != nil { + break + } + assert.Equal(t, pdTypeData, frame.typ, "writer was active, so no padding frame should have been written") + assert.Equal(t, []byte("whee"), frame.payload) + } +} + +func Test_ConfigPDKeepAliveNormalization(t *testing.T) { + pd := &PacketDownConfig{} + assert.Equal(t, int32(10), pd.GetNormalizedKeepAlivePeriod().From) + assert.Equal(t, int32(30), pd.GetNormalizedKeepAlivePeriod().To) + assert.Equal(t, int32(24), pd.GetNormalizedKeepAliveLength().From) + assert.Equal(t, int32(64), pd.GetNormalizedKeepAliveLength().To) + + pd.KeepAliveLength = &RangeConfig{From: 60000, To: 90000} + assert.Equal(t, int32(60000), pd.GetNormalizedKeepAliveLength().From) + assert.Equal(t, int32(pdMaxFramePayload), pd.GetNormalizedKeepAliveLength().To) +} diff --git a/transport/internet/splithttp/splithttp_test.go b/transport/internet/splithttp/splithttp_test.go index 62ca858f3e87..a4754f43f8fe 100644 --- a/transport/internet/splithttp/splithttp_test.go +++ b/transport/internet/splithttp/splithttp_test.go @@ -465,3 +465,121 @@ func Test_maxUpload(t *testing.T) { common.Must(listen.Close()) } + +func Test_ListenXHAndDial_PacketDown(t *testing.T) { + packetDown := &PacketDownConfig{ + KeepAlive: true, + KeepAlivePeriod: &RangeConfig{From: 2, To: 2}, + KeepAliveLength: &RangeConfig{From: 24, To: 64}, + } + + payload := make([]byte, 100*1024) + rand.Read(payload) + + listenPort := tcp.PickPort() + serverStreamSettings := &internet.MemoryStreamConfig{ + ProtocolName: "splithttp", + ProtocolSettings: &Config{ + Path: "/sh", + PacketDown: packetDown, + }, + } + listen, err := ListenXH(context.Background(), net.LocalHostIP, listenPort, serverStreamSettings, func(conn stat.Connection) { + go func(c stat.Connection) { + defer c.Close() + + var b [32]byte + c.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, err := c.Read(b[:]) + if err != nil { + return + } + + written := 0 + for written < len(payload) { + step := 8192 + if written+step > len(payload) { + step = len(payload) - written + } + if _, err := c.Write(payload[written : written+step]); err != nil { + return + } + written += step + } + }(conn) + }) + common.Must(err) + defer listen.Close() + + ctx := context.Background() + clientStreamSettings := &internet.MemoryStreamConfig{ + ProtocolName: "splithttp", + ProtocolSettings: &Config{ + Path: "sh", + PacketDown: packetDown, + }, + } + conn, err := Dial(ctx, net.TCPDestination(net.DomainAddress("localhost"), listenPort), clientStreamSettings) + common.Must(err) + + _, err = conn.Write([]byte("Test connection 1")) + common.Must(err) + + got := make([]byte, len(payload)) + _, err = io.ReadFull(conn, got) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(payload, got) { + t.Error("100 KiB came back different through packet-down") + } + + common.Must(conn.Close()) +} + +func Test_ListenXHAndDial_PacketDownWithoutServer(t *testing.T) { + // client asks, server has no packetDown: the response is still raw + listenPort := tcp.PickPort() + listen, err := ListenXH(context.Background(), net.LocalHostIP, listenPort, &internet.MemoryStreamConfig{ + ProtocolName: "splithttp", + ProtocolSettings: &Config{ + Path: "/sh", + }, + }, func(conn stat.Connection) { + go func(c stat.Connection) { + defer c.Close() + + var b [32]byte + c.SetReadDeadline(time.Now().Add(5 * time.Second)) + _, err := c.Read(b[:]) + if err != nil { + return + } + + common.Must2(c.Write([]byte("Response"))) + }(conn) + }) + common.Must(err) + defer listen.Close() + + ctx := context.Background() + conn, err := Dial(ctx, net.TCPDestination(net.DomainAddress("localhost"), listenPort), &internet.MemoryStreamConfig{ + ProtocolName: "splithttp", + ProtocolSettings: &Config{ + Path: "sh", + PacketDown: &PacketDownConfig{KeepAlive: true}, + }, + }) + common.Must(err) + + _, err = conn.Write([]byte("Test connection 1")) + common.Must(err) + + var b [1024]byte + n, _ := io.ReadFull(conn, b[:]) + if string(b[:n]) != "Response" { + t.Error("response: ", string(b[:n])) + } + + common.Must(conn.Close()) +}