Preserve raw response bytes in ResponseChain

http/normalization.go

@@ -109,6 +109,22 @@ func readNNormalizeRespBody(rc *ResponseChain, body *bytes.Buffer) (err error) {
 		// skip normalization if body is nil
 		return nil
 	}
+	if rc.rawBody != nil {
+		rawBuf := &limitedBuffer{buf: rc.rawBody, maxCap: int(rc.maxBodySize)}
+		_, rawErr := rawBuf.ReadFrom(origBody)
+		_ = origBody.Close()
+		if rawErr != nil && !stringsutil.ContainsAnyI(rawErr.Error(), "unexpected EOF", "read: connection reset by peer", "user canceled", "http: request body too large") {
+			return errors.Wrap(rawErr, "could not read raw response body")
+		}
+		if rawErr != nil {
+			if response.Header == nil {
+				response.Header = make(http.Header)
+			}
+			response.Header.Set("x-nuclei-ignore-error", rawErr.Error())
+		}
+		response.Body = io.NopCloser(bytes.NewReader(rc.rawBody.Bytes()))
+		origBody = response.Body
+	}
 	// wrap with decode if applicable
 	wrapped, err := wrapDecodeReader(response)
 	if err != nil {

http/respChain.go

@@ -219,6 +219,7 @@ func resetBuffer() {
 type ResponseChain struct {
 	headers     *bytes.Buffer
 	body        *bytes.Buffer
+	rawBody     *bytes.Buffer
 	resp        *http.Response
 	reloaded    bool // if response was reloaded to its previous redirect
 	maxBodySize int64
@@ -229,6 +230,16 @@ type ResponseChain struct {
 //
 // If maxBody is less than or equal to zero, it defaults to [DefaultMaxBodySize].
 func NewResponseChain(resp *http.Response, maxBody int64) *ResponseChain {
+	return newResponseChain(resp, maxBody, false)
+}
+
+// NewResponseChainWithRaw creates a response chain that preserves the response
+// body bytes before content decoding and charset normalization.
+func NewResponseChainWithRaw(resp *http.Response, maxBody int64) *ResponseChain {
+	return newResponseChain(resp, maxBody, true)
+}
+
+func newResponseChain(resp *http.Response, maxBody int64, preserveRaw bool) *ResponseChain {
 	if maxBody <= 0 {
 		maxBody = int64(DefaultMaxBodySize)
 	}
@@ -237,12 +248,16 @@ func NewResponseChain(resp *http.Response, maxBody int64) *ResponseChain {
 		resp.Body = http.MaxBytesReader(nil, resp.Body, maxBody)
 	}
 
-	return &ResponseChain{
+	chain := &ResponseChain{
 		headers:     getBuffer(),
 		body:        getBuffer(),
 		resp:        resp,
 		maxBodySize: maxBody,
 	}
+	if preserveRaw {
+		chain.rawBody = getBuffer()
+	}
+	return chain
 }
 
 // Headers returns the current response headers buffer in the chain.
@@ -289,6 +304,27 @@ func (r *ResponseChain) BodyString() string {
 	return r.body.String()
 }
 
+// RawBodyBytes returns the response body before content decoding and charset
+// normalization. It is empty unless the chain was created with
+// NewResponseChainWithRaw.
+//
+// The returned slice is valid only until Close() is called.
+func (r *ResponseChain) RawBodyBytes() []byte {
+	if r.rawBody == nil {
+		return nil
+	}
+	return r.rawBody.Bytes()
+}
+
+// RawBodyString returns a copy of the response body before content decoding
+// and charset normalization.
+func (r *ResponseChain) RawBodyString() string {
+	if r.rawBody == nil {
+		return ""
+	}
+	return r.rawBody.String()
+}
+
 // FullResponse returns a new buffer containing headers+body.
 //
 // Warning: The caller is responsible for managing the returned buffer's
@@ -324,6 +360,27 @@ func (r *ResponseChain) FullResponseString() string {
 	return conversion.String(r.FullResponseBytes())
 }
 
+// RawFullResponseBytes returns headers followed by the body bytes captured
+// before content decoding and charset normalization.
+func (r *ResponseChain) RawFullResponseBytes() []byte {
+	if r.rawBody == nil {
+		return nil
+	}
+	size := r.headers.Len() + r.rawBody.Len()
+	buf := make([]byte, size)
+
+	copy(buf, r.headers.Bytes())
+	copy(buf[r.headers.Len():], r.rawBody.Bytes())
+
+	return buf
+}
+
+// RawFullResponseString returns a copy of the headers and body bytes captured
+// before content decoding and charset normalization.
+func (r *ResponseChain) RawFullResponseString() string {
+	return conversion.String(r.RawFullResponseBytes())
+}
+
 // previous updates response pointer to previous response
 // if it was redirected and returns true else false
 func (r *ResponseChain) Previous() bool {
@@ -383,6 +440,11 @@ func (r *ResponseChain) Close() {
 		putBuffer(r.body)
 		r.body = nil
 	}
+
+	if r.rawBody != nil {
+		putBuffer(r.rawBody)
+		r.rawBody = nil
+	}
 }
 
 // Has returns true if the response chain has a response
@@ -408,4 +470,7 @@ func (r *ResponseChain) Response() *http.Response {
 func (r *ResponseChain) reset() {
 	r.headers.Reset()
 	r.body.Reset()
+	if r.rawBody != nil {
+		r.rawBody.Reset()
+	}
 }

http/respChain_test.go

@@ -135,6 +135,37 @@ func TestResponseChain_GzipHandling(t *testing.T) {
 	rc.Close()
 }
 
+func TestResponseChain_PreservesRawGzipBody(t *testing.T) {
+	originalBody := "compressed response body"
+
+	var compressed bytes.Buffer
+	gzWriter := gzip.NewWriter(&compressed)
+	_, err := gzWriter.Write([]byte(originalBody))
+	require.NoError(t, err)
+	require.NoError(t, gzWriter.Close())
+	compressedBytes := append([]byte(nil), compressed.Bytes()...)
+
+	resp := &http.Response{
+		StatusCode: 200,
+		Body:       io.NopCloser(bytes.NewReader(compressedBytes)),
+		Header: http.Header{
+			"Content-Encoding": []string{"gzip"},
+		},
+		Proto:      "HTTP/1.1",
+		ProtoMajor: 1,
+		ProtoMinor: 1,
+	}
+
+	rc := NewResponseChainWithRaw(resp, -1)
+	require.NoError(t, rc.Fill())
+	t.Cleanup(rc.Close)
+
+	assert.Equal(t, originalBody, rc.BodyString())
+	assert.Equal(t, compressedBytes, rc.RawBodyBytes())
+	assert.Contains(t, rc.RawFullResponseString(), "Content-Encoding: gzip")
+	assert.True(t, bytes.HasSuffix(rc.RawFullResponseBytes(), compressedBytes))
+}
+
 // TestResponseChain_EmptyBody tests handling of empty response bodies
 func TestResponseChain_EmptyBody(t *testing.T) {
 	resp := &http.Response{