From 7425b380d9c6209f41d8286af43de185c05df9bd Mon Sep 17 00:00:00 2001 From: Luan Nutels - VolkSec Date: Thu, 30 Jul 2026 15:54:34 -0300 Subject: [PATCH] Preserve raw response bytes in ResponseChain Signed-off-by: Luan Nutels - VolkSec --- http/normalization.go | 16 ++++++++++ http/respChain.go | 67 +++++++++++++++++++++++++++++++++++++++++- http/respChain_test.go | 31 +++++++++++++++++++ 3 files changed, 113 insertions(+), 1 deletion(-) diff --git a/http/normalization.go b/http/normalization.go index a9c0238..25f825f 100644 --- a/http/normalization.go +++ b/http/normalization.go @@ -109,6 +109,22 @@ func readNNormalizeRespBody(rc *ResponseChain, body *bytes.Buffer) (err error) { // skip normalization if body is nil return nil } + if rc.rawBody != nil { + rawBuf := &limitedBuffer{buf: rc.rawBody, maxCap: int(rc.maxBodySize)} + _, rawErr := rawBuf.ReadFrom(origBody) + _ = origBody.Close() + if rawErr != nil && !stringsutil.ContainsAnyI(rawErr.Error(), "unexpected EOF", "read: connection reset by peer", "user canceled", "http: request body too large") { + return errors.Wrap(rawErr, "could not read raw response body") + } + if rawErr != nil { + if response.Header == nil { + response.Header = make(http.Header) + } + response.Header.Set("x-nuclei-ignore-error", rawErr.Error()) + } + response.Body = io.NopCloser(bytes.NewReader(rc.rawBody.Bytes())) + origBody = response.Body + } // wrap with decode if applicable wrapped, err := wrapDecodeReader(response) if err != nil { diff --git a/http/respChain.go b/http/respChain.go index ef73bf0..35b25d5 100644 --- a/http/respChain.go +++ b/http/respChain.go @@ -219,6 +219,7 @@ func resetBuffer() { type ResponseChain struct { headers *bytes.Buffer body *bytes.Buffer + rawBody *bytes.Buffer resp *http.Response reloaded bool // if response was reloaded to its previous redirect maxBodySize int64 @@ -229,6 +230,16 @@ type ResponseChain struct { // // If maxBody is less than or equal to zero, it defaults to [DefaultMaxBodySize]. func NewResponseChain(resp *http.Response, maxBody int64) *ResponseChain { + return newResponseChain(resp, maxBody, false) +} + +// NewResponseChainWithRaw creates a response chain that preserves the response +// body bytes before content decoding and charset normalization. +func NewResponseChainWithRaw(resp *http.Response, maxBody int64) *ResponseChain { + return newResponseChain(resp, maxBody, true) +} + +func newResponseChain(resp *http.Response, maxBody int64, preserveRaw bool) *ResponseChain { if maxBody <= 0 { maxBody = int64(DefaultMaxBodySize) } @@ -237,12 +248,16 @@ func NewResponseChain(resp *http.Response, maxBody int64) *ResponseChain { resp.Body = http.MaxBytesReader(nil, resp.Body, maxBody) } - return &ResponseChain{ + chain := &ResponseChain{ headers: getBuffer(), body: getBuffer(), resp: resp, maxBodySize: maxBody, } + if preserveRaw { + chain.rawBody = getBuffer() + } + return chain } // Headers returns the current response headers buffer in the chain. @@ -289,6 +304,27 @@ func (r *ResponseChain) BodyString() string { return r.body.String() } +// RawBodyBytes returns the response body before content decoding and charset +// normalization. It is empty unless the chain was created with +// NewResponseChainWithRaw. +// +// The returned slice is valid only until Close() is called. +func (r *ResponseChain) RawBodyBytes() []byte { + if r.rawBody == nil { + return nil + } + return r.rawBody.Bytes() +} + +// RawBodyString returns a copy of the response body before content decoding +// and charset normalization. +func (r *ResponseChain) RawBodyString() string { + if r.rawBody == nil { + return "" + } + return r.rawBody.String() +} + // FullResponse returns a new buffer containing headers+body. // // Warning: The caller is responsible for managing the returned buffer's @@ -324,6 +360,27 @@ func (r *ResponseChain) FullResponseString() string { return conversion.String(r.FullResponseBytes()) } +// RawFullResponseBytes returns headers followed by the body bytes captured +// before content decoding and charset normalization. +func (r *ResponseChain) RawFullResponseBytes() []byte { + if r.rawBody == nil { + return nil + } + size := r.headers.Len() + r.rawBody.Len() + buf := make([]byte, size) + + copy(buf, r.headers.Bytes()) + copy(buf[r.headers.Len():], r.rawBody.Bytes()) + + return buf +} + +// RawFullResponseString returns a copy of the headers and body bytes captured +// before content decoding and charset normalization. +func (r *ResponseChain) RawFullResponseString() string { + return conversion.String(r.RawFullResponseBytes()) +} + // previous updates response pointer to previous response // if it was redirected and returns true else false func (r *ResponseChain) Previous() bool { @@ -383,6 +440,11 @@ func (r *ResponseChain) Close() { putBuffer(r.body) r.body = nil } + + if r.rawBody != nil { + putBuffer(r.rawBody) + r.rawBody = nil + } } // Has returns true if the response chain has a response @@ -408,4 +470,7 @@ func (r *ResponseChain) Response() *http.Response { func (r *ResponseChain) reset() { r.headers.Reset() r.body.Reset() + if r.rawBody != nil { + r.rawBody.Reset() + } } diff --git a/http/respChain_test.go b/http/respChain_test.go index f7dd325..7b79ed8 100644 --- a/http/respChain_test.go +++ b/http/respChain_test.go @@ -135,6 +135,37 @@ func TestResponseChain_GzipHandling(t *testing.T) { rc.Close() } +func TestResponseChain_PreservesRawGzipBody(t *testing.T) { + originalBody := "compressed response body" + + var compressed bytes.Buffer + gzWriter := gzip.NewWriter(&compressed) + _, err := gzWriter.Write([]byte(originalBody)) + require.NoError(t, err) + require.NoError(t, gzWriter.Close()) + compressedBytes := append([]byte(nil), compressed.Bytes()...) + + resp := &http.Response{ + StatusCode: 200, + Body: io.NopCloser(bytes.NewReader(compressedBytes)), + Header: http.Header{ + "Content-Encoding": []string{"gzip"}, + }, + Proto: "HTTP/1.1", + ProtoMajor: 1, + ProtoMinor: 1, + } + + rc := NewResponseChainWithRaw(resp, -1) + require.NoError(t, rc.Fill()) + t.Cleanup(rc.Close) + + assert.Equal(t, originalBody, rc.BodyString()) + assert.Equal(t, compressedBytes, rc.RawBodyBytes()) + assert.Contains(t, rc.RawFullResponseString(), "Content-Encoding: gzip") + assert.True(t, bytes.HasSuffix(rc.RawFullResponseBytes(), compressedBytes)) +} + // TestResponseChain_EmptyBody tests handling of empty response bodies func TestResponseChain_EmptyBody(t *testing.T) { resp := &http.Response{