diff --git a/cmd/aiscan/imports_default_test.go b/cmd/aiscan/imports_default_test.go index 3209663e..56f18f3c 100644 --- a/cmd/aiscan/imports_default_test.go +++ b/cmd/aiscan/imports_default_test.go @@ -10,7 +10,7 @@ import ( ) func TestDefaultCapabilitySet(t *testing.T) { - want := []string{"arsenal", "core", "gogo", "ioa", "neutron", "proton", "proxy", "scan", "search", "spray", "zombie"} + want := []string{"arsenal", "core", "curl", "gogo", "ioa", "neutron", "proton", "proxy", "scan", "search", "spray", "zombie"} if got := capability.IDsSorted(); !slices.Equal(got, want) { t.Fatalf("default capabilities = %#v, want %#v", got, want) } diff --git a/cmd/aiscan/imports_full_test.go b/cmd/aiscan/imports_full_test.go index 38934ba3..a6804080 100644 --- a/cmd/aiscan/imports_full_test.go +++ b/cmd/aiscan/imports_full_test.go @@ -10,7 +10,7 @@ import ( ) func TestFullCapabilitySet(t *testing.T) { - want := []string{"arsenal", "browser", "core", "gogo", "ioa", "katana", "neutron", "passive", "proton", "proxy", "scan", "search", "spray", "zombie"} + want := []string{"arsenal", "browser", "core", "curl", "gogo", "ioa", "katana", "neutron", "passive", "proton", "proxy", "scan", "search", "spray", "zombie"} if got := capability.IDsSorted(); !slices.Equal(got, want) { t.Fatalf("full capabilities = %#v, want %#v", got, want) } diff --git a/skills/aiscan/okf/easm/curl.md b/skills/aiscan/okf/easm/curl.md index a8c7272c..0ebc859e 100644 --- a/skills/aiscan/okf/easm/curl.md +++ b/skills/aiscan/okf/easm/curl.md @@ -21,8 +21,13 @@ Capabilities: - follow redirects (`-L`), with a bounded redirect count (`--max-redirs`) - carry and persist cookies across calls (`-b` / `-c`) - override the naturalized User-Agent and headers when a specific client shape is needed -- include response headers (`-i`), write the body to a file (`-o`), and report - outcome fields (`-w`, e.g. `%{http_code}`, `%{url_effective}`) +- include response headers (`-i`/`-I`), dump them separately (`-D`), write the + body to a file (`-o`), and report outcome fields (`-w`, e.g. `%{http_code}`, + `%{url_effective}`) +- fail on HTTP error responses (`-f`), set a transfer deadline (`-m`), and + select HTTP/1.1 or HTTP/2 (`--http1.1`/`--http2`) +- route a hostname to an explicit address (`--resolve`) when running without a + proxy; preserve URL dot segments with `--path-as-is` Common usage: @@ -31,6 +36,9 @@ curl curl -X POST -d 'a=1&b=2' curl -H 'Authorization: Bearer ...' -i curl -L -b 'sid=abc' -c jar.txt +curl -fsSL -m 10 +curl -D headers.txt -o body.bin +curl --resolve example.test:443:192.0.2.10 https://example.test/ ``` Notes: @@ -39,6 +47,8 @@ Notes: first-class path for evidence-backed HTTP probing. - A browser User-Agent and header set are applied only where you did not set them; `-A` and `-H` always win. +- `--resolve` is rejected when a proxy is active because the proxy owns the + destination connection; it is never silently treated as a no-op. - Unsupported flags are rejected rather than silently ignored, so behavior is never quietly different from what was asked. diff --git a/tools/curl/client.go b/tools/curl/client.go index 7fbaf817..512f1a1f 100644 --- a/tools/curl/client.go +++ b/tools/curl/client.go @@ -6,6 +6,7 @@ import ( "crypto/tls" "crypto/x509" "encoding/base64" + "errors" "fmt" "io" "mime" @@ -13,12 +14,14 @@ import ( "net" "net/http" "net/http/cookiejar" + "net/http/httptrace" "net/textproto" "net/url" "os" "path/filepath" "strconv" "strings" + "sync" "time" toolpb "github.com/chainreactors/aiscan/aop/tool" @@ -53,6 +56,11 @@ var browserDefaults = []Header{ // naturalization defaults, performs the exchange, and writes curl-shaped output. // env and workDir are per-invocation; nothing here mutates the shared Command. func (c *Command) do(ctx context.Context, req *Request, env map[string]string, workDir string, stdout, stderr io.Writer) error { + if req.Version { + _, err := fmt.Fprintln(stdout, compatibilityVersion) + return err + } + proxyURL, caPath := c.egress(env) if req.Proxy != "" { // -x overrides the injected hub egress for this invocation only. @@ -61,21 +69,47 @@ func (c *Command) do(ctx context.Context, req *Request, env map[string]string, w proxyURL = "http://" + proxyURL } } + if len(req.Resolve) > 0 && proxyURL != "" { + // A standard library HTTP proxy owns the destination dial and therefore + // cannot safely honor a local host mapping without also changing CONNECT + // and TLS-SNI behavior. Fail explicitly instead of silently ignoring the + // option (or bypassing the evidence proxy). + return fmt.Errorf("curl: --resolve cannot be used with a proxy") + } client, err := c.buildClient(proxyURL, caPath, req) if err != nil { return err } + trace, err := openASCIITrace(req.TraceASCII, workDir, stdout, stderr) + if err != nil { + return err + } + if trace != nil { + defer func() { _ = trace.Close() }() + } + responses := make([]http.Response, 0, 1) + client.Transport = &capturingTransport{base: client.Transport, responses: &responses, trace: trace} target, err := url.Parse(strings.TrimSpace(req.URL)) if err != nil || target.Scheme == "" || target.Host == "" { return fmt.Errorf("curl: (3) URL rejected: %s", req.URL) } + if !req.PathAsIs { + normalizeCurlURLPath(target) + } body, contentType, err := buildBody(req, target, workDir) if err != nil { return err } + if req.Head { + // --head suppresses the response body even when -X explicitly selects a + // different method (curl uses this combination for header-only probes). + // A HEAD transfer also never carries a request body. + body = nil + contentType = "" + } if req.MaxTime > 0 { var cancel context.CancelFunc @@ -94,6 +128,21 @@ func (c *Command) do(ctx context.Context, req *Request, env map[string]string, w return err } } + if trace != nil { + traceCtx := &httptrace.ClientTrace{ + ConnectStart: func(network, addr string) { + trace.info(" Trying %s...", addr) + }, + ConnectDone: func(network, addr string, connectErr error) { + if connectErr != nil { + trace.info(" Failed to connect to %s: %v", addr, connectErr) + return + } + trace.info(" Connected to %s", addr) + }, + } + httpReq = httpReq.WithContext(httptrace.WithClientTrace(httpReq.Context(), traceCtx)) + } if req.Verbose && !req.Silent { writeVerboseRequest(stderr, httpReq) @@ -101,6 +150,9 @@ func (c *Command) do(ctx context.Context, req *Request, env map[string]string, w resp, err := client.Do(httpReq) if err != nil { + if isTimeoutError(err) { + return fmt.Errorf("curl: (28) %w", err) + } return fmt.Errorf("curl: (7) %w", err) } defer resp.Body.Close() @@ -109,6 +161,16 @@ func (c *Command) do(ctx context.Context, req *Request, env map[string]string, w writeVerboseResponse(stderr, resp) } + if err := dumpResponseHeaders(req, responses, workDir, stdout); err != nil { + return err + } + failed := req.Fail && resp.StatusCode >= http.StatusBadRequest + if failed && !req.Include { + // curl leaves -o untouched (and does not create it) when --fail rejects + // a response, unless headers were explicitly requested with -i. + return c.failResponse(ctx, client, req, resp, workDir, stdout) + } + out, closeOut, err := outputWriter(req, workDir, stdout) if err != nil { return err @@ -116,17 +178,30 @@ func (c *Command) do(ctx context.Context, req *Request, env map[string]string, w defer closeOut() if req.Include { - writeStatusAndHeaders(out, resp) + for i := range responses { + writeStatusAndHeaders(out, &responses[i]) + } + } + if failed { + return c.failResponse(ctx, client, req, resp, workDir, stdout) + } + var written int64 + if req.Head { + // Drain and discard a body some non-conforming servers attach to a + // header-only request, so the connection remains reusable. + _, err = io.Copy(io.Discard, resp.Body) + } else { + written, err = copyResponse(out, resp.Body, req.NoBuffer) } - written, err := io.Copy(out, resp.Body) if err != nil { + if isTimeoutError(err) { + return fmt.Errorf("curl: (28) %w", err) + } return fmt.Errorf("curl: (56) %w", err) } - if req.CookieJar != "" { - if err := writeCookieJar(client.Jar, resp.Request.URL, resolvePath(workDir, req.CookieJar)); err != nil && !req.Silent { - c.Logger.Warnf("curl: write cookie jar: %s", err) - } + if err := persistResponseCookies(c, client, req, resp, workDir); err != nil { + return err } if req.WriteOut != "" { @@ -137,6 +212,25 @@ func (c *Command) do(ctx context.Context, req *Request, env map[string]string, w return nil } +func (c *Command) failResponse(ctx context.Context, client *http.Client, req *Request, resp *http.Response, workDir string, stdout io.Writer) error { + if err := persistResponseCookies(c, client, req, resp, workDir); err != nil { + return err + } + if req.WriteOut != "" { + fmt.Fprint(stdout, expandWriteOut(req.WriteOut, resp, 0)) + } + c.emitArtifact(ctx, resp, 0) + return fmt.Errorf("curl: (22) The requested URL returned error: %s", resp.Status) +} + +func isTimeoutError(err error) bool { + if errors.Is(err, context.DeadlineExceeded) { + return true + } + var netErr net.Error + return errors.As(err, &netErr) && netErr.Timeout() +} + // egress reads the hub proxy and CA path the runner injected into this // execution's environment. The proxy URL already carries the tool-call id as // its username, so captured flows attribute to this call; the CA is present @@ -177,10 +271,41 @@ func (c *Command) buildClient(proxyURL, caPath string, req *Request) (*http.Clie if dialTimeout == 0 { dialTimeout = 30 * time.Second } + dialer := &net.Dialer{Timeout: dialTimeout} + dialContext := dialer.DialContext + if len(req.Resolve) > 0 { + resolve := makeResolveMap(req.Resolve) + dialContext = func(ctx context.Context, network, address string) (net.Conn, error) { + host, port, err := net.SplitHostPort(address) + if err == nil { + entry, ok := lookupResolve(resolve, host, port) + if ok { + var lastErr error + for _, mapped := range entry.Addresses { + conn, dialErr := dialer.DialContext(ctx, network, net.JoinHostPort(mapped, port)) + if dialErr == nil { + return conn, nil + } + lastErr = dialErr + } + if lastErr != nil { + return nil, lastErr + } + } + } + return dialer.DialContext(ctx, network, address) + } + } + forceHTTP2 := req.HTTP2 || !req.HTTP11 transport := &http.Transport{ - TLSClientConfig: tlsConfig, - DialContext: (&net.Dialer{Timeout: dialTimeout}).DialContext, - ForceAttemptHTTP2: true, + TLSClientConfig: tlsConfig, + DialContext: dialContext, + ForceAttemptHTTP2: forceHTTP2, + // Trace the bytes delivered by the transport rather than an implicit + // auto-decompressed gzip stream. An explicit Accept-Encoding header is + // still honored; this only disables Go's automatic compression behavior + // while --trace-ascii is active. + DisableCompression: req.TraceASCII != "", MaxIdleConns: 100, IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: dialTimeout, @@ -426,6 +551,316 @@ func outputWriter(req *Request, workDir string, stdout io.Writer) (io.Writer, fu return file, func() { _ = file.Close() }, nil } +type capturingTransport struct { + base http.RoundTripper + responses *[]http.Response + trace *asciiTrace +} + +func (t *capturingTransport) RoundTrip(req *http.Request) (*http.Response, error) { + if t.trace != nil { + header, body := traceRequestParts(req) + t.trace.block("=>", "Send header", header) + if len(body) > 0 { + t.trace.block("=>", "Send data", body) + } + } + resp, err := t.base.RoundTrip(req) + if err == nil && resp != nil && t.responses != nil { + captured := *resp + captured.Body = nil + captured.Header = resp.Header.Clone() + *t.responses = append(*t.responses, captured) + } + if err == nil && resp != nil && t.trace != nil { + for _, header := range traceResponseHeaderBlocks(resp) { + t.trace.block("<=", "Recv header", header) + } + if resp.Body != nil { + resp.Body = &traceReadCloser{ReadCloser: resp.Body, trace: t.trace} + } + } + return resp, err +} + +// asciiTrace is the local diagnostic sink used by --trace-ascii. It is kept +// separate from the Hub/evidence path: tracing observes the request and +// response but never changes attribution or transport routing. +type asciiTrace struct { + mu sync.Mutex + w io.Writer + closef func() error +} + +func openASCIITrace(path, workDir string, stdout, stderr io.Writer) (*asciiTrace, error) { + if path == "" { + return nil, nil + } + if path == "-" { + return &asciiTrace{w: stdout}, nil + } + if path == "%" { + return &asciiTrace{w: stderr}, nil + } + file, err := os.Create(resolvePath(workDir, path)) + if err != nil { + return nil, fmt.Errorf("curl: (23) Failed to create trace-ascii %q: %w", path, err) + } + return &asciiTrace{w: file, closef: file.Close}, nil +} + +func (t *asciiTrace) Close() error { + if t == nil || t.closef == nil { + return nil + } + return t.closef() +} + +func (t *asciiTrace) info(format string, args ...any) { + t.writef("== Info: "+format+"\n", args...) +} + +func (t *asciiTrace) writef(format string, args ...any) { + if t == nil { + return + } + t.mu.Lock() + defer t.mu.Unlock() + _, _ = fmt.Fprintf(t.w, format, args...) +} + +// block renders the same useful shape as curl's ASCII trace: a direction and +// event line followed by 64-byte offset rows with non-printable bytes shown as +// dots. CRLF pairs become line breaks while offsets continue to count the raw +// bytes, matching libcurl's ASCII dump callback. The wire bytes are +// intentionally kept local and are never emitted as an HTTP artifact. +func (t *asciiTrace) block(direction, event string, data []byte) { + if t == nil { + return + } + const width = 64 + t.mu.Lock() + defer t.mu.Unlock() + fmt.Fprintf(t.w, "%s %s, %d bytes (0x%x)\n", direction, event, len(data), len(data)) + for offset := 0; offset < len(data); { + lineOffset := offset + line := make([]byte, 0, width) + nextOffset := offset + width + for column := 0; column < width && offset+column < len(data); column++ { + if offset+column+1 < len(data) && data[offset+column] == '\r' && data[offset+column+1] == '\n' { + // libcurl removes CRLF from the visible row but advances the + // offset by both raw bytes. + nextOffset = offset + column + 2 + break + } + value := data[offset+column] + if value >= 0x20 && value < 0x80 { + line = append(line, value) + } else { + line = append(line, '.') + } + if offset+column+2 < len(data) && data[offset+column+1] == '\r' && data[offset+column+2] == '\n' { + nextOffset = offset + column + 3 + break + } + } + if len(line) > 0 { + fmt.Fprintf(t.w, "%04x: %s\n", lineOffset, line) + } + offset = nextOffset + } +} + +type traceReadCloser struct { + io.ReadCloser + trace *asciiTrace +} + +func (r *traceReadCloser) Read(p []byte) (int, error) { + n, err := r.ReadCloser.Read(p) + if n > 0 && r.trace != nil { + r.trace.block("<=", "Recv data", p[:n]) + } + return n, err +} + +func traceRequestParts(req *http.Request) (header, body []byte) { + if req == nil { + return nil, nil + } + if req.GetBody != nil { + if bodyReader, err := req.GetBody(); err == nil { + body, _ = io.ReadAll(bodyReader) + _ = bodyReader.Close() + } + } + var buf bytes.Buffer + proto := req.Proto + if proto == "" { + proto = "HTTP/1.1" + } + uri := "/" + if req.URL != nil { + uri = req.URL.RequestURI() + if uri == "" { + uri = "/" + } + } + host := req.Host + if host == "" && req.URL != nil { + host = req.URL.Host + } + fmt.Fprintf(&buf, "%s %s %s\r\n", req.Method, uri, proto) + fmt.Fprintf(&buf, "Host: %s\r\n", host) + _ = req.Header.Write(&buf) + if len(body) > 0 && req.ContentLength >= 0 && req.Header.Get("Content-Length") == "" { + fmt.Fprintf(&buf, "Content-Length: %d\r\n", req.ContentLength) + } + buf.WriteString("\r\n") + return buf.Bytes(), body +} + +func traceResponseHeaders(resp *http.Response) []byte { + if resp == nil { + return nil + } + var buf bytes.Buffer + fmt.Fprintf(&buf, "%s %s\r\n", resp.Proto, resp.Status) + _ = resp.Header.Write(&buf) + buf.WriteString("\r\n") + return buf.Bytes() +} + +// traceResponseHeaderBlocks mirrors libcurl's usual debug callback granularity: +// the status line, each response header, and the terminating CRLF are separate +// HEADER_IN blocks. Keeping the aggregate serializer above is useful for tests +// and makes the wire representation easy to inspect before splitting. +func traceResponseHeaderBlocks(resp *http.Response) [][]byte { + raw := traceResponseHeaders(resp) + if len(raw) == 0 { + return nil + } + blocks := make([][]byte, 0, 1+len(resp.Header)) + for start := 0; start < len(raw); { + relEnd := bytes.Index(raw[start:], []byte("\r\n")) + if relEnd < 0 { + blocks = append(blocks, append([]byte(nil), raw[start:]...)) + break + } + end := start + relEnd + 2 + blocks = append(blocks, append([]byte(nil), raw[start:end]...)) + start = end + } + return blocks +} + +func dumpResponseHeaders(req *Request, responses []http.Response, workDir string, stdout io.Writer) error { + if req.DumpHeader == "" { + return nil + } + if req.DumpHeader == "-" { + for i := range responses { + writeStatusAndHeaders(stdout, &responses[i]) + } + return nil + } + path := resolvePath(workDir, req.DumpHeader) + file, err := os.Create(path) + if err != nil { + return fmt.Errorf("curl: (23) Failed to create dump-header %q: %w", req.DumpHeader, err) + } + for i := range responses { + writeStatusAndHeaders(file, &responses[i]) + } + if err := file.Close(); err != nil { + return fmt.Errorf("curl: (23) Failed to close dump-header %q: %w", req.DumpHeader, err) + } + return nil +} + +func persistResponseCookies(c *Command, client *http.Client, req *Request, resp *http.Response, workDir string) error { + if req.CookieJar == "" { + return nil + } + if resp == nil || resp.Request == nil { + return nil + } + if err := writeCookieJar(client.Jar, resp.Request.URL, resolvePath(workDir, req.CookieJar)); err != nil && !req.Silent { + c.Logger.Warnf("curl: write cookie jar: %s", err) + } + return nil +} + +// copyResponse keeps the normal io.Copy fast path while making -N meaningful +// for writers that expose Flush. net/http response bodies are already streamed; +// this loop only adds an explicit flush after each chunk when requested. +func copyResponse(dst io.Writer, src io.Reader, noBuffer bool) (int64, error) { + if !noBuffer { + return io.Copy(dst, src) + } + buf := make([]byte, 32*1024) + var written int64 + for { + n, readErr := src.Read(buf) + if n > 0 { + m, writeErr := dst.Write(buf[:n]) + written += int64(m) + if writeErr != nil { + return written, writeErr + } + if m != n { + return written, io.ErrShortWrite + } + flushWriter(dst) + } + if readErr != nil { + if readErr == io.EOF { + return written, nil + } + return written, readErr + } + } +} + +type flusher interface{ Flush() } + +func flushWriter(w io.Writer) { + if f, ok := w.(flusher); ok { + f.Flush() + } +} + +func makeResolveMap(entries []ResolveEntry) map[string]ResolveEntry { + resolved := make(map[string]ResolveEntry, len(entries)) + for _, entry := range entries { + key := resolveKey(entry.Host, entry.Port) + // The last --resolve entry for a host/port wins, matching curl's + // resolver table. A leading '+' only changes the DNS-cache lifetime; + // this client resolves per invocation, so Temporary is informational. + resolved[key] = entry + } + return resolved +} + +func lookupResolve(entries map[string]ResolveEntry, host, port string) (ResolveEntry, bool) { + for _, key := range []string{ + resolveKey(host, port), + resolveKey("*", port), + resolveKey(host, "*"), + resolveKey("*", "*"), + } { + if entry, ok := entries[key]; ok { + return entry, true + } + } + return ResolveEntry{}, false +} + +func resolveKey(host, port string) string { + host = strings.TrimSuffix(strings.ToLower(strings.TrimSpace(host)), ".") + return host + ":" + strings.TrimSpace(port) +} + func (c *Command) emitArtifact(ctx context.Context, resp *http.Response, size int64) { if c.Events == nil || resp.Request == nil { return @@ -453,6 +888,84 @@ func resolvePath(workDir, path string) string { return filepath.Join(workDir, path) } +// normalizeCurlURLPath implements the URL dot-segment cleanup performed by +// curl unless --path-as-is is selected. Dot segments are recognized after +// unescaping the segment, so encoded forms such as %2e%2e behave like .. while +// other escapes remain intact in the request target. +func normalizeCurlURLPath(u *url.URL) { + raw := u.EscapedPath() + if raw == "" { + u.Path = "/" + u.RawPath = "" + return + } + trailingSlash := strings.HasSuffix(raw, "/") + parts := strings.Split(raw, "/") + out := make([]string, 0, len(parts)) + for i, part := range parts { + if i == 0 { + // Absolute HTTP URLs always have a leading slash. Keep an empty + // first segment so joining below preserves that invariant. + out = append(out, part) + continue + } + switch dotSegment(part) { + case ".": + if i == len(parts)-1 { + trailingSlash = true + } + case "..": + if i == len(parts)-1 { + trailingSlash = true + } + if len(out) > 1 { + // Pop one segment, including an empty segment. This is why + // /a//../b becomes /a/b rather than /b. + out = out[:len(out)-1] + } + case "": + // curl collapses duplicate slashes at the beginning of an HTTP + // path, while preserving empty segments in the middle. + if len(out) == 1 { + continue + } + out = append(out, part) + default: + out = append(out, part) + } + } + cleaned := strings.Join(out, "/") + if cleaned == "" { + cleaned = "/" + } + if trailingSlash && cleaned != "/" && !strings.HasSuffix(cleaned, "/") { + cleaned += "/" + } + decoded, err := url.PathUnescape(cleaned) + if err != nil { + // Keep the original URL if it contains malformed escaping; the request + // constructor will return curl's normal URL error downstream. + return + } + u.Path = decoded + if decoded == cleaned { + u.RawPath = "" + } else { + u.RawPath = cleaned + } +} + +func dotSegment(raw string) string { + decoded, err := url.PathUnescape(raw) + if err != nil { + return raw + } + if decoded == "." || decoded == ".." { + return decoded + } + return raw +} + func writeStatusAndHeaders(w io.Writer, resp *http.Response) { fmt.Fprintf(w, "%s %s\r\n", resp.Proto, resp.Status) _ = resp.Header.Write(w) diff --git a/tools/curl/client_test.go b/tools/curl/client_test.go index c08c54c5..9876bcaa 100644 --- a/tools/curl/client_test.go +++ b/tools/curl/client_test.go @@ -1,14 +1,17 @@ package curl import ( + "bytes" "context" "io" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" "strings" "testing" + "time" ) // run is a small harness: parse args, execute against a real server, capture @@ -326,3 +329,454 @@ func TestProxyOverride(t *testing.T) { t.Fatalf("body = %q, want via-proxy", out) } } + +func TestDumpHeaders(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Dump-Test", "yes") + _, _ = w.Write([]byte("body")) + })) + defer srv.Close() + + dir := t.TempDir() + out, _, err := run(t, []string{"-D", "headers.txt", srv.URL}, "", dir) + if err != nil { + t.Fatal(err) + } + if out != "body" { + t.Fatalf("body = %q, want body", out) + } + headerData, err := os.ReadFile(filepath.Join(dir, "headers.txt")) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(string(headerData), "X-Dump-Test: yes") || !strings.Contains(string(headerData), "200 OK") { + t.Fatalf("dumped headers missing status/header:\n%s", headerData) + } +} + +func TestDumpHeadersIncludesRedirectResponses(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/start" { + w.Header().Set("X-First", "yes") + http.Redirect(w, r, "/final", http.StatusFound) + return + } + w.Header().Set("X-Second", "yes") + _, _ = w.Write([]byte("final")) + })) + defer srv.Close() + + dir := t.TempDir() + out, _, err := run(t, []string{"-L", "-D", "headers.txt", srv.URL + "/start"}, "", dir) + if err != nil { + t.Fatal(err) + } + if out != "final" { + t.Fatalf("body = %q", out) + } + data, err := os.ReadFile(filepath.Join(dir, "headers.txt")) + if err != nil { + t.Fatal(err) + } + text := string(data) + if !strings.Contains(text, "X-First: yes") || !strings.Contains(text, "X-Second: yes") || !strings.Contains(text, "302 Found") || !strings.Contains(text, "200 OK") { + t.Fatalf("redirect headers = %q", text) + } +} + +func TestTraceASCIIFileContainsWireMarkersAndSanitizedData(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-Trace", "yes") + _, _ = w.Write([]byte("hello\x00world\xff")) + })) + defer srv.Close() + + dir := t.TempDir() + out, _, err := run(t, []string{"--trace-ascii", "trace.log", "-d", "secret=body", srv.URL}, "", dir) + if err != nil { + t.Fatal(err) + } + if out != "hello\x00world\xff" { + t.Fatalf("response body = %q", out) + } + trace, err := os.ReadFile(filepath.Join(dir, "trace.log")) + if err != nil { + t.Fatal(err) + } + text := string(trace) + for _, marker := range []string{"=> Send header", "=> Send data", "<= Recv header", "<= Recv data", "0000:"} { + if !strings.Contains(text, marker) { + t.Fatalf("trace missing %q:\n%s", marker, text) + } + } + if !strings.Contains(text, "secret=body") || !strings.Contains(text, "hello.world.") { + t.Fatalf("trace did not contain expected ASCII/body representation:\n%s", text) + } +} + +func TestTraceASCIIDashAndPercentDestinations(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("body")) + })) + defer srv.Close() + + out, errOut, err := run(t, []string{"--trace-ascii", "-", srv.URL}, "", "") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(out, "=> Send header") || !strings.Contains(out, "body") { + t.Fatalf("trace '-' should share stdout with body: %q", out) + } + if errOut != "" { + t.Fatalf("trace '-' wrote stderr: %q", errOut) + } + + out, errOut, err = run(t, []string{"--trace-ascii", "%", srv.URL}, "", "") + if err != nil { + t.Fatal(err) + } + if out != "body" || !strings.Contains(errOut, "=> Send header") { + t.Fatalf("trace '%%' should use stderr: stdout=%q stderr=%q", out, errOut) + } +} + +func TestTraceASCIIFormatterHandlesCRLFAndRawOffsets(t *testing.T) { + var output bytes.Buffer + trace := &asciiTrace{w: &output} + data := append(bytes.Repeat([]byte{'A'}, 64), '\r', '\n', 'B') + trace.block("<=", "Recv data", data) + text := output.String() + if !strings.Contains(text, "0000: "+strings.Repeat("A", 64)+"\n") { + t.Fatalf("first trace row = %q", text) + } + if strings.Contains(text, "0040:") { + t.Fatalf("trace emitted an empty CRLF row = %q", text) + } + if !strings.Contains(text, "0042: B\n") { + t.Fatalf("CRLF did not advance the raw offset: %q", text) + } +} + +func TestTraceASCIIResponseHeadersUseCurlCallbackBlocks(t *testing.T) { + resp := &http.Response{ + Proto: "HTTP/1.1", + Status: "200 OK", + Header: http.Header{"X-First": {"one"}, "X-Second": {"two"}}, + } + blocks := traceResponseHeaderBlocks(resp) + if len(blocks) != 4 { // status, two fields, terminating CRLF + t.Fatalf("header blocks = %d, want 4: %#v", len(blocks), blocks) + } + if string(blocks[0]) != "HTTP/1.1 200 OK\r\n" || string(blocks[len(blocks)-1]) != "\r\n" { + t.Fatalf("header block boundaries are not curl-shaped: %#v", blocks) + } +} + +func TestTraceASCIIIncludesRedirectTransfers(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/start" { + http.Redirect(w, r, "/final", http.StatusFound) + return + } + _, _ = w.Write([]byte("final")) + })) + defer srv.Close() + + dir := t.TempDir() + if _, _, err := run(t, []string{"-L", "--trace-ascii", "trace.log", srv.URL + "/start"}, "", dir); err != nil { + t.Fatal(err) + } + trace, err := os.ReadFile(filepath.Join(dir, "trace.log")) + if err != nil { + t.Fatal(err) + } + text := string(trace) + if strings.Count(text, "=> Send header") < 2 || strings.Count(text, "<= Recv header") < 2 { + t.Fatalf("redirect trace did not include both transfers:\n%s", text) + } +} + +func TestHeadRequest(t *testing.T) { + var method string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + method = r.Method + w.Header().Set("X-Head-Test", "yes") + _, _ = w.Write([]byte("must-not-be-returned")) + })) + defer srv.Close() + + out, _, err := run(t, []string{"-I", srv.URL}, "", "") + if err != nil { + t.Fatal(err) + } + if method != http.MethodHead { + t.Fatalf("method = %q, want HEAD", method) + } + if !strings.Contains(out, "X-Head-Test: yes") || strings.Contains(out, "must-not-be-returned") { + t.Fatalf("head output = %q", out) + } +} + +func TestHeadFlagKeepsExplicitMethodButSuppressesBody(t *testing.T) { + var method string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + method = r.Method + w.Header().Set("X-Head-Test", "yes") + _, _ = w.Write([]byte("must-not-be-returned")) + })) + defer srv.Close() + + out, _, err := run(t, []string{"-I", "-X", "GET", srv.URL}, "", "") + if err != nil { + t.Fatal(err) + } + if method != http.MethodGet { + t.Fatalf("method = %q, want GET", method) + } + if !strings.Contains(out, "X-Head-Test: yes") || strings.Contains(out, "must-not-be-returned") { + t.Fatalf("header-only explicit-method output = %q", out) + } +} + +func TestDefaultPathNormalizationAndPathAsIs(t *testing.T) { + paths := make(chan string, 4) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths <- r.URL.EscapedPath() + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + if _, _, err := run(t, []string{srv.URL + "/a/../b/./c"}, "", ""); err != nil { + t.Fatal(err) + } + if _, _, err := run(t, []string{"--path-as-is", srv.URL + "/a/../b/./c"}, "", ""); err != nil { + t.Fatal(err) + } + if got := <-paths; got != "/b/c" { + t.Fatalf("default path = %q, want /b", got) + } + if got := <-paths; got != "/a/../b/./c" { + t.Fatalf("--path-as-is path = %q, want /a/../b", got) + } +} + +func TestPathNormalizationHandlesEncodedDots(t *testing.T) { + paths := make(chan string, 2) + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + paths <- r.URL.EscapedPath() + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + if _, _, err := run(t, []string{srv.URL + "/%2e%2e/x"}, "", ""); err != nil { + t.Fatal(err) + } + if _, _, err := run(t, []string{"--path-as-is", srv.URL + "/%2e%2e/x"}, "", ""); err != nil { + t.Fatal(err) + } + if got := <-paths; got != "/x" { + t.Fatalf("default encoded dot path = %q, want /x", got) + } + if got := <-paths; got != "/%2e%2e/x" { + t.Fatalf("--path-as-is encoded dot path = %q", got) + } +} + +func TestHeadGetDataUsesQuery(t *testing.T) { + var method, path string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + method, path = r.Method, r.URL.RequestURI() + w.Header().Set("X-Head-Get", "yes") + w.WriteHeader(http.StatusNoContent) + })) + defer srv.Close() + + out, _, err := run(t, []string{"-I", "-G", "-d", "a=1", srv.URL}, "", "") + if err != nil { + t.Fatal(err) + } + if method != http.MethodHead || path != "/?a=1" { + t.Fatalf("request = %s %s, want HEAD /?a=1", method, path) + } + if !strings.Contains(out, "X-Head-Get: yes") { + t.Fatalf("head output = %q", out) + } +} + +func TestFailSuppressesHTTPErrorBody(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("not-found-body")) + })) + defer srv.Close() + + out, _, err := run(t, []string{"-f", srv.URL}, "", "") + if err == nil || !strings.Contains(err.Error(), "(22)") { + t.Fatalf("error = %v, want curl status 22", err) + } + if out != "" { + t.Fatalf("--fail emitted response body %q", out) + } +} + +func TestFailLeavesOutputFileUntouchedWithoutInclude(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte("not-found-body")) + })) + defer srv.Close() + + dir := t.TempDir() + path := filepath.Join(dir, "existing.txt") + if err := os.WriteFile(path, []byte("old"), 0o644); err != nil { + t.Fatal(err) + } + _, _, err := run(t, []string{"-f", "-o", "existing.txt", srv.URL}, "", dir) + if err == nil || !strings.Contains(err.Error(), "(22)") { + t.Fatalf("error = %v, want curl status 22", err) + } + data, readErr := os.ReadFile(path) + if readErr != nil || string(data) != "old" { + t.Fatalf("--fail changed output file: %q err=%v", data, readErr) + } +} + +func TestShortMaxTimeAliasBoundsTransfer(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + time.Sleep(150 * time.Millisecond) + _, _ = w.Write([]byte("late")) + })) + defer srv.Close() + + started := time.Now() + _, _, err := run(t, []string{"-m", "0.01", srv.URL}, "", "") + if err == nil || !strings.Contains(err.Error(), "(28)") { + t.Fatalf("-m timeout error = %v, want curl status 28", err) + } + if elapsed := time.Since(started); elapsed > time.Second { + t.Fatalf("-m timeout took %s", elapsed) + } +} + +func TestResolveMapsDialWithoutChangingHost(t *testing.T) { + var gotHost string + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotHost = r.Host + _, _ = w.Write([]byte("resolved")) + })) + defer srv.Close() + + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatal(err) + } + port := u.Port() + out, _, err := run(t, []string{ + "--resolve", "example.test:" + port + ":127.0.0.1", + "http://example.test:" + port + "/path", + }, "", "") + if err != nil { + t.Fatal(err) + } + if out != "resolved" { + t.Fatalf("body = %q", out) + } + if gotHost != "example.test:"+port { + t.Fatalf("Host = %q, want example.test:%s", gotHost, port) + } +} + +func TestResolvePreservesTLSServerNameAndHost(t *testing.T) { + var gotHost string + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotHost = r.Host + _, _ = w.Write([]byte("secure-resolved")) + })) + defer srv.Close() + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatal(err) + } + out, _, err := run(t, []string{ + "-k", "--resolve", "example.test:" + u.Port() + ":127.0.0.1", + "https://example.test:" + u.Port() + "/", + }, "", "") + if err != nil { + t.Fatal(err) + } + if out != "secure-resolved" { + t.Fatalf("body = %q", out) + } + if gotHost != "example.test:"+u.Port() { + t.Fatalf("Host = %q, want example.test:%s", gotHost, u.Port()) + } +} + +func TestResolveRejectsActiveProxy(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("proxy")) + })) + defer srv.Close() + if _, _, err := run(t, []string{ + "--resolve", "example.test:80:127.0.0.1", "-x", srv.URL, + "http://example.test/", + }, "", ""); err == nil || !strings.Contains(err.Error(), "--resolve cannot be used with a proxy") { + t.Fatalf("resolve with proxy error = %v", err) + } +} + +func TestResolveWildcardMapsDial(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte("wildcard")) + })) + defer srv.Close() + u, err := url.Parse(srv.URL) + if err != nil { + t.Fatal(err) + } + out, _, err := run(t, []string{ + "--resolve", "*:" + u.Port() + ":127.0.0.1", + "http://any-host.invalid:" + u.Port() + "/", + }, "", "") + if err != nil { + t.Fatal(err) + } + if out != "wildcard" { + t.Fatalf("wildcard resolve body = %q", out) + } +} + +func TestHTTP11DisablesHTTP2Negotiation(t *testing.T) { + var proto string + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + proto = r.Proto + _, _ = w.Write([]byte("ok")) + })) + srv.EnableHTTP2 = true + srv.StartTLS() + defer srv.Close() + + if _, _, err := run(t, []string{"--http1.1", "-k", srv.URL}, "", ""); err != nil { + t.Fatal(err) + } + if proto != "HTTP/1.1" { + t.Fatalf("protocol = %q, want HTTP/1.1", proto) + } +} + +func TestHTTP2Negotiation(t *testing.T) { + var proto string + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + proto = r.Proto + _, _ = w.Write([]byte("ok")) + })) + srv.EnableHTTP2 = true + srv.StartTLS() + defer srv.Close() + + if _, _, err := run(t, []string{"--http2", "-k", srv.URL}, "", ""); err != nil { + t.Fatal(err) + } + if proto != "HTTP/2.0" { + t.Fatalf("protocol = %q, want HTTP/2.0", proto) + } +} diff --git a/tools/curl/cookies.go b/tools/curl/cookies.go index 2d6e9c28..4f164e25 100644 --- a/tools/curl/cookies.go +++ b/tools/curl/cookies.go @@ -36,7 +36,7 @@ func seedCookies(jar http.CookieJar, target *url.URL, spec, workDir string) erro line = strings.TrimPrefix(line, "#HttpOnly_") if fields := strings.Split(line, "\t"); len(fields) == 7 { // domain, flag, path, secure, expiration, name, value - jar.SetCookies(cookieURL(fields[0], fields[2], target), []*http.Cookie{{Name: fields[5], Value: fields[6]}}) + jar.SetCookies(cookieURL(fields[0], fields[2], target), []*http.Cookie{newCookie(fields[5], fields[6])}) continue } inline = append(inline, parseCookieString(line)...) @@ -54,11 +54,17 @@ func parseCookieString(spec string) []*http.Cookie { if !ok || name == "" { continue } - cookies = append(cookies, &http.Cookie{Name: strings.TrimSpace(name), Value: strings.TrimSpace(value)}) + cookies = append(cookies, newCookie(strings.TrimSpace(name), strings.TrimSpace(value))) } return cookies } +func newCookie(name, value string) *http.Cookie { + // Cookie flags are supplied by the caller's cookie source. This tool is a + // client-side jar, so it must not invent Secure/HttpOnly/SameSite policy. + return &http.Cookie{Name: name, Value: value} //nolint:gosec // G124: preserve source cookie attributes +} + func cookieURL(domain, path string, fallback *url.URL) *url.URL { domain = strings.TrimPrefix(domain, ".") if domain == "" { diff --git a/tools/curl/curl.go b/tools/curl/curl.go index 23d1ae04..9286a437 100644 --- a/tools/curl/curl.go +++ b/tools/curl/curl.go @@ -2,6 +2,7 @@ package curl import ( "context" + "fmt" "strings" aop "github.com/chainreactors/aiscan/aop" @@ -21,6 +22,11 @@ type Command struct { toolargs.Base } +// compatibilityVersion is intentionally explicit instead of inheriting the +// host's curl version. Agents can therefore use --version to discover that +// they are talking to the deterministic in-process implementation. +const compatibilityVersion = "curl 8.14.1 (aiscan pure-Go)" + func New() *Command { c := &Command{} c.InitLogger(nil) @@ -63,15 +69,25 @@ Supported options: -e, --referer Referer header -u, --user Basic authentication -o, --output Write body to file instead of stdout + -D, --dump-header Write response headers to a separate file + --trace-ascii Write an ASCII trace (- stdout, % stderr) -i, --include Include response headers in the output + -I, --head Fetch headers only (HEAD request) -s, --silent Silent mode -S, --show-error Show errors even with -s + -f, --fail Fail on HTTP 4xx/5xx responses -w, --write-out After completion, print %{http_code}, %{url_effective}, ... -v, --verbose Log request/response headers + -N, --no-buffer Stream response output without buffering -k, --insecure Do not verify TLS -x, --proxy Use this proxy instead of the runner egress --connect-timeout Connection timeout, seconds - --max-time Overall timeout, seconds + -m, --max-time Overall timeout, seconds + --http2 Prefer HTTP/2 + --http1.1 Force HTTP/1.1 + --resolve host:port:addr Route a host/port to an explicit address + --path-as-is Preserve dot segments in the URL path + --version, -V Print the compatibility version and exit Unlisted flags are rejected rather than silently ignored. Requests are routed through the runner proxy and recorded as HTTP evidence; a browser User-Agent and @@ -84,6 +100,9 @@ func (c *Command) QuickReference() string { curl -X POST -d 'a=1' POST form data curl -H 'Authorization: ...' curl -i -L Include headers, follow redirects + curl -I HEAD request (headers only) + curl -D headers.txt Save response headers separately + curl -fsSL Follow redirects and fail on HTTP errors curl -b 'sid=abc' -c jar.txt Send and persist cookies curl -F 'file=@a.png' Multipart form upload` } @@ -98,6 +117,10 @@ func (c *Command) Run(ctx context.Context, execution *commands.Execution) (_ any if err != nil { return nil, err } + if req.Version { + _, err := fmt.Fprintln(execution.Stdout, compatibilityVersion) + return nil, err + } workDir := execution.Dir if workDir == "" { diff --git a/tools/curl/parse.go b/tools/curl/parse.go index be441664..798b65d5 100644 --- a/tools/curl/parse.go +++ b/tools/curl/parse.go @@ -2,6 +2,7 @@ package curl import ( "fmt" + "math" "strconv" "strings" "time" @@ -40,14 +41,15 @@ type FormPart struct { // Request is the parsed, transport-agnostic shape of one curl invocation. type Request struct { - URL string - Method string - Headers []Header - Data []DataPart - Form []FormPart - Get bool // -G: send data as query string - Follow bool // -L - MaxRedirs int // --max-redirs (default 50 when following) + URL string + Method string + MethodExplicit bool // -X/--request was supplied; prevents -I changing it + Headers []Header + Data []DataPart + Form []FormPart + Get bool // -G: send data as query string + Follow bool // -L + MaxRedirs int // --max-redirs (default 50 when following) UserAgent string // -A Referer string // -e @@ -55,30 +57,51 @@ type Request struct { CookieIn string // -b: cookie string or @file CookieJar string // -c: write jar here after the exchange - Output string // -o - Include bool // -i - Silent bool // -s - ShowError bool // -S - WriteOut string // -w - Verbose bool // -v - Insecure bool // -k - Proxy string // -x: override the egress proxy for this invocation + Output string // -o + DumpHeader string // -D: write response headers to a file (or - for stdout) + TraceASCII string // --trace-ascii: write an ASCII wire trace to a file + Include bool // -i + Head bool // -I/--head: issue a HEAD request and include headers + Silent bool // -s + ShowError bool // -S + Fail bool // -f/--fail: fail on HTTP 4xx/5xx + WriteOut string // -w + Verbose bool // -v + NoBuffer bool // -N/--no-buffer: stream response writes without buffering + Insecure bool // -k + Proxy string // -x: override the egress proxy for this invocation + HTTP2 bool // --http2: prefer/require HTTP/2 where available + HTTP11 bool // --http1.1: disable HTTP/2 negotiation + PathAsIs bool // --path-as-is: preserve dot segments in the URL path + Version bool // --version/-V: print the curl compatibility version + Resolve []ResolveEntry // --resolve host:port:address, repeatable ConnectTimeout time.Duration // --connect-timeout MaxTime time.Duration // --max-time } +// ResolveEntry describes one --resolve mapping. Addresses are tried in order +// when a target is dialed; the URL host (and therefore the HTTP Host and TLS +// SNI name) remains unchanged. +type ResolveEntry struct { + Host string + Port string + Addresses []string + Temporary bool // --resolve +host:... uses curl's temporary DNS lifetime +} + // valueShort maps short flags that consume a value. var valueShort = map[byte]string{ 'X': "request", 'H': "header", 'd': "data", 'b': "cookie", 'c': "cookie-jar", 'A': "user-agent", 'e': "referer", 'u': "user", 'o': "output", 'w': "write-out", - 'F': "form", 'x': "proxy", + 'F': "form", 'x': "proxy", 'D': "dump-header", 'm': "max-time", } // boolShort maps short flags that take no value. var boolShort = map[byte]string{ 'G': "get", 'L': "location", 'i': "include", 's': "silent", - 'S': "show-error", 'v': "verbose", 'k': "insecure", + 'S': "show-error", 'v': "verbose", 'k': "insecure", 'I': "head", 'f': "fail", + 'N': "no-buffer", 'V': "version", } // Parse turns a curl argument vector into a Request, rejecting any flag outside @@ -129,6 +152,14 @@ func Parse(args []string) (*Request, error) { } } + // --version is a local informational query and intentionally does not need + // a URL (matching curl's `curl --version` behavior). + if r.Version { + // The native curl frontend exits after printing version information and + // ignores any URL supplied alongside -V/--version. + return r, nil + } + if r.URL == "" { switch len(urls) { case 0: @@ -143,12 +174,20 @@ func Parse(args []string) (*Request, error) { } if r.Method == "" { - if (len(r.Data) > 0 || len(r.Form) > 0) && !r.Get { + if r.Head { + r.Method = "HEAD" + } else if (len(r.Data) > 0 || len(r.Form) > 0) && !r.Get { r.Method = "POST" } else { r.Method = "GET" } } + if r.Head { + r.Include = true + } + if r.Head && !r.Get && (len(r.Data) > 0 || len(r.Form) > 0) { + return nil, fmt.Errorf("curl: (2) cannot combine -I/--head with -d/--data or -F/--form") + } if len(r.Form) > 0 && len(r.Data) > 0 { return nil, fmt.Errorf("curl: (2) -d/--data and -F/--form cannot be combined") } @@ -202,6 +241,7 @@ func (r *Request) applyLong(name string, need func() (string, error)) error { return err } r.Method = strings.ToUpper(v) + r.MethodExplicit = true case "header": v, err := need() if err != nil { @@ -276,12 +316,35 @@ func (r *Request) applyLong(name string, need func() (string, error)) error { return err } r.Output = v + case "dump-header": + v, err := need() + if err != nil { + return err + } + r.DumpHeader = v + case "trace-ascii": + v, err := need() + if err != nil { + return err + } + if v == "" { + return fmt.Errorf("curl: --trace-ascii requires a non-empty file") + } + // curl's trace and verbose modes are mutually exclusive; the last + // selector wins when command-line options are repeated. + r.TraceASCII = v + r.Verbose = false case "include": r.Include = true + case "head": + r.Head = true + r.Include = true case "silent": r.Silent = true case "show-error": r.ShowError = true + case "fail": + r.Fail = true case "write-out": v, err := need() if err != nil { @@ -290,8 +353,11 @@ func (r *Request) applyLong(name string, need func() (string, error)) error { r.WriteOut = v case "verbose": r.Verbose = true + r.TraceASCII = "" case "insecure": r.Insecure = true + case "no-buffer": + r.NoBuffer = true case "proxy": v, err := need() if err != nil { @@ -318,6 +384,30 @@ func (r *Request) applyLong(name string, need func() (string, error)) error { return fmt.Errorf("curl: --max-time %w", err) } r.MaxTime = d + case "http2": + // curl treats repeated HTTP-version selectors as last-option-wins and + // emits only a diagnostic on the native frontend. The parser has no + // stderr channel, so retain the deterministic state without silently + // ignoring either recognized option. + r.HTTP11 = false + r.HTTP2 = true + case "http1.1": + r.HTTP2 = false + r.HTTP11 = true + case "path-as-is": + r.PathAsIs = true + case "resolve": + v, err := need() + if err != nil { + return err + } + entry, err := parseResolve(v) + if err != nil { + return err + } + r.Resolve = append(r.Resolve, entry) + case "version": + r.Version = true default: return fmt.Errorf("curl: unsupported flag --%s", name) } @@ -386,8 +476,81 @@ func parseHeader(raw string) Header { func parseSeconds(v string) (time.Duration, error) { f, err := strconv.ParseFloat(strings.TrimSpace(v), 64) - if err != nil || f < 0 { + if err != nil || f < 0 || math.IsNaN(f) || math.IsInf(f, 0) { return 0, fmt.Errorf("expects a non-negative number of seconds: %q", v) } - return time.Duration(f * float64(time.Second)), nil + nanos := f * float64(time.Second) + // float64((1<<63)-1) rounds up to 1<<63. Use an inclusive bound so a + // value that would wrap time.Duration into a negative duration is rejected + // instead of silently turning into an effectively unbounded timeout. + if nanos >= float64(1<<63) { + return 0, fmt.Errorf("is too large: %q", v) + } + return time.Duration(nanos), nil +} + +// parseResolve parses curl's host:port:address spelling. The address portion +// may itself contain colons (for example an IPv6 literal), so only the first +// two separators are significant. Multiple comma-separated addresses are +// accepted and tried in order by the dialer. +func parseResolve(raw string) (ResolveEntry, error) { + raw = strings.TrimSpace(raw) + temporaryEntry := strings.HasPrefix(raw, "+") + if temporaryEntry { + raw = strings.TrimSpace(strings.TrimPrefix(raw, "+")) + } + host, rest, ok := splitResolveHost(raw) + if !ok { + return ResolveEntry{}, fmt.Errorf("curl: --resolve expects host:port:address: %q", raw) + } + second := strings.IndexByte(rest, ':') + if second <= 0 || second == len(rest)-1 { + return ResolveEntry{}, fmt.Errorf("curl: --resolve expects host:port:address: %q", raw) + } + port := strings.TrimSpace(rest[:second]) + addressSpec := strings.TrimSpace(rest[second+1:]) + if host == "" || port == "" || addressSpec == "" { + return ResolveEntry{}, fmt.Errorf("curl: --resolve expects host:port:address: %q", raw) + } + if port != "*" { + n, err := strconv.Atoi(port) + if err != nil || n < 1 || n > 65535 { + return ResolveEntry{}, fmt.Errorf("curl: --resolve has invalid port %q", port) + } + } + addresses := make([]string, 0, 1) + for _, value := range strings.Split(addressSpec, ",") { + value = strings.TrimSpace(value) + if value == "" { + continue + } + // Strip optional brackets from IPv6 literals; net.JoinHostPort adds + // them back when constructing the dial target. + value = strings.TrimPrefix(value, "[") + value = strings.TrimSuffix(value, "]") + addresses = append(addresses, value) + } + if len(addresses) == 0 { + return ResolveEntry{}, fmt.Errorf("curl: --resolve has no address: %q", raw) + } + host = strings.Trim(strings.TrimSpace(host), "[]") + if host == "" { + return ResolveEntry{}, fmt.Errorf("curl: --resolve has no host: %q", raw) + } + return ResolveEntry{Host: strings.TrimSuffix(strings.ToLower(host), "."), Port: port, Addresses: addresses, Temporary: temporaryEntry}, nil +} + +func splitResolveHost(raw string) (host, rest string, ok bool) { + if strings.HasPrefix(raw, "[") { + end := strings.IndexByte(raw, ']') + if end < 0 || end+1 >= len(raw) || raw[end+1] != ':' { + return "", "", false + } + return raw[:end+1], raw[end+2:], true + } + idx := strings.IndexByte(raw, ':') + if idx <= 0 || idx == len(raw)-1 { + return "", "", false + } + return raw[:idx], raw[idx+1:], true } diff --git a/tools/curl/parse_test.go b/tools/curl/parse_test.go index 40017e3f..55bcbaf5 100644 --- a/tools/curl/parse_test.go +++ b/tools/curl/parse_test.go @@ -1,6 +1,7 @@ package curl import ( + "net/url" "testing" "time" ) @@ -64,6 +65,16 @@ func TestParseShortBundle(t *testing.T) { } } +func TestParseCommonFailBundle(t *testing.T) { + req, err := Parse([]string{"-fsSL", "https://x"}) + if err != nil { + t.Fatal(err) + } + if !req.Fail || !req.Silent || !req.ShowError || !req.Follow { + t.Fatalf("-fsSL not fully applied: %+v", req) + } +} + func TestParseBundleTrailingValue(t *testing.T) { req, err := Parse([]string{"-so", "out.txt", "https://x"}) if err != nil { @@ -97,6 +108,14 @@ func TestParseTimeouts(t *testing.T) { } } +func TestParseTimeoutOverflowIsRejected(t *testing.T) { + for _, value := range []string{"9223372036.854775808", "1e100", "NaN", "+Inf"} { + if _, err := Parse([]string{"-m", value, "https://x"}); err == nil { + t.Fatalf("Parse accepted overflowing timeout %q", value) + } + } +} + func TestParseUnsupportedFlagErrors(t *testing.T) { if _, err := Parse([]string{"--http2-prior-knowledge", "https://x"}); err == nil { t.Fatal("expected error for unsupported long flag") @@ -192,3 +211,176 @@ func TestParseProxy(t *testing.T) { t.Fatalf("proxy = %q", req.Proxy) } } + +func TestParseCompatibilityFlags(t *testing.T) { + req, err := Parse([]string{ + "-m", "2.5", "-D", "headers.txt", "-I", "-f", "-N", + "--http2", "--resolve", "example.test:8443:127.0.0.1", + "--path-as-is", "https://example.test:8443/a/../b", + }) + if err != nil { + t.Fatal(err) + } + if req.MaxTime != 2500*time.Millisecond || req.DumpHeader != "headers.txt" { + t.Fatalf("short compatibility flags parsed incorrectly: %+v", req) + } + if !req.Head || req.Method != "HEAD" || !req.Include || !req.Fail || !req.NoBuffer { + t.Fatalf("head/fail/no-buffer parsed incorrectly: %+v", req) + } + if !req.HTTP2 || req.HTTP11 || !req.PathAsIs || len(req.Resolve) != 1 { + t.Fatalf("transport flags parsed incorrectly: %+v", req) + } + entry := req.Resolve[0] + if entry.Host != "example.test" || entry.Port != "8443" || len(entry.Addresses) != 1 || entry.Addresses[0] != "127.0.0.1" { + t.Fatalf("resolve parsed incorrectly: %+v", entry) + } +} + +func TestParseTraceASCII(t *testing.T) { + req, err := Parse([]string{"--trace-ascii", "trace.log", "https://x"}) + if err != nil { + t.Fatal(err) + } + if req.TraceASCII != "trace.log" || req.Verbose { + t.Fatalf("trace-ascii parsed incorrectly: %+v", req) + } +} + +func TestParseTraceVerboseLastOptionWins(t *testing.T) { + trace, err := Parse([]string{"--trace-ascii", "trace.log", "-v", "https://x"}) + if err != nil { + t.Fatal(err) + } + if trace.TraceASCII != "" || !trace.Verbose { + t.Fatalf("verbose should disable an earlier trace: %+v", trace) + } + verbose, err := Parse([]string{"-v", "--trace-ascii", "trace.log", "https://x"}) + if err != nil { + t.Fatal(err) + } + if verbose.TraceASCII != "trace.log" || verbose.Verbose { + t.Fatalf("trace should disable an earlier verbose: %+v", verbose) + } +} + +func TestParseHTTPVersionLastOptionWins(t *testing.T) { + first, err := Parse([]string{"--http2", "--http1.1", "https://x"}) + if err != nil { + t.Fatal(err) + } + if first.HTTP2 || !first.HTTP11 { + t.Fatalf("last --http1.1 should win: %+v", first) + } + second, err := Parse([]string{"--http1.1", "--http2", "https://x"}) + if err != nil { + t.Fatal(err) + } + if !second.HTTP2 || second.HTTP11 { + t.Fatalf("last --http2 should win: %+v", second) + } +} + +func TestParseResolveIPv6AndTemporary(t *testing.T) { + req, err := Parse([]string{ + "--resolve", "example.test:443:127.0.0.1", + "--resolve", "+example.test:443:[::1],127.0.0.2", + "--resolve", "*:80:127.0.0.3", + "https://example.test", + }) + if err != nil { + t.Fatal(err) + } + if len(req.Resolve) != 3 || req.Resolve[1].Host != "example.test" || !req.Resolve[1].Temporary || req.Resolve[1].Addresses[0] != "::1" { + t.Fatalf("resolve entries = %+v", req.Resolve) + } +} + +func TestParseHeadDoesNotOverrideExplicitMethod(t *testing.T) { + for _, args := range [][]string{ + {"-X", "GET", "-I", "https://x"}, + {"-I", "-X", "POST", "https://x"}, + } { + req, err := Parse(args) + if err != nil { + t.Fatalf("Parse(%v): %v", args, err) + } + if req.Method == "HEAD" || !req.MethodExplicit || !req.Head || !req.Include { + t.Fatalf("-I should preserve explicit method while enabling header-only mode: %+v", req) + } + } +} + +func TestParseHeadRejectsRequestBody(t *testing.T) { + for _, args := range [][]string{ + {"-I", "-d", "a=1", "https://x"}, + {"-I", "-X", "POST", "-F", "a=b", "https://x"}, + } { + if _, err := Parse(args); err == nil { + t.Fatalf("Parse(%v) accepted a body with --head", args) + } + } +} + +func TestParseHeadGetDataUsesQuery(t *testing.T) { + req, err := Parse([]string{"-I", "-G", "-d", "a=1", "https://x"}) + if err != nil { + t.Fatal(err) + } + if req.Method != "HEAD" || !req.Head || !req.Get || len(req.Data) != 1 { + t.Fatalf("head GET data parsed incorrectly: %+v", req) + } +} + +func TestNormalizeCurlURLPathRFCExamples(t *testing.T) { + cases := map[string]string{ + "/a/b/c/./../../g": "/a/g", + "/a/b/c/./../../g/": "/a/g/", + "/a/b/c/../..": "/a/", + "/a/b/c/../../..": "/", + "/a/b/c/../../../g": "/g", + "/a/b/c/./../../g/.": "/a/g/", + } + for input, want := range cases { + u, err := url.Parse("http://example.test" + input) + if err != nil { + t.Fatal(err) + } + normalizeCurlURLPath(u) + if got := u.EscapedPath(); got != want { + t.Errorf("%s => %s, want %s", input, got, want) + } + } +} + +func TestNormalizeCurlURLPathRepeatedSlashExamples(t *testing.T) { + cases := map[string]string{ + "//a///b": "/a///b", + "/a//../b": "/a/b", + "/a/./b": "/a/b", + "/../x": "/x", + "/../../x": "/x", + "/a//b/../c": "/a//c", + "/a/%2e%2e/b": "/b", + "/a/%2E./b": "/b", + "/a/.%2e/b": "/b", + } + for input, want := range cases { + u, _ := url.Parse("http://example.test" + input) + normalizeCurlURLPath(u) + if got := u.EscapedPath(); got != want { + t.Errorf("%s => %s, want %s", input, got, want) + } + } +} + +func TestParseVersionDoesNotNeedURL(t *testing.T) { + for _, args := range [][]string{{"--version"}, {"-V", "https://ignored.example"}} { + req, err := Parse(args) + if err != nil { + t.Fatalf("Parse(%v): %v", args, err) + } + if !req.Version { + t.Fatalf("Parse(%v) did not set Version: %+v", args, req) + } + } +} diff --git a/tools/proxy/hub_test.go b/tools/proxy/hub_test.go index dc63e9a5..38bd40ec 100644 --- a/tools/proxy/hub_test.go +++ b/tools/proxy/hub_test.go @@ -193,7 +193,7 @@ func TestCaptureFiltersAndVerbs(t *testing.T) { } first := store.Query(QueryOpts{Last: 1}) if len(first) == 1 { - out := runMitm(t, store, hub, "flow", first[0].Exchange.ID) + out := runMitm(t, store, hub, "flow", first[0].ID) if !strings.Contains(out, "Request Headers") { t.Errorf("flow detail missing headers: %q", out) } diff --git a/tools/proxy/hub_traffic.go b/tools/proxy/hub_traffic.go index 4066103f..b332ddfe 100644 --- a/tools/proxy/hub_traffic.go +++ b/tools/proxy/hub_traffic.go @@ -71,7 +71,7 @@ func flowToProto(flow *Flow) *traffic.Flow { if flow == nil { return nil } - message := flow.Exchange.Proto() + message := flow.Proto() message.ToolId = flow.ToolID if !flow.Timestamp.IsZero() { message.Timestamp = timestamppb.New(flow.Timestamp) diff --git a/tools/proxy/mitm.go b/tools/proxy/mitm.go index 26ac1cea..5ea2b229 100644 --- a/tools/proxy/mitm.go +++ b/tools/proxy/mitm.go @@ -216,7 +216,7 @@ func (a *captureAddon) Response(f *mitmproxy.Flow) { flow.Request.Body = snip(f.Request.Body, maxBodySnip) } if f.Response != nil { - flow.Exchange.Response = &traffic.Response{ + flow.Response = &traffic.Response{ StatusCode: f.Response.StatusCode, Headers: pairsFromHTTP(f.Response.Header), } @@ -326,7 +326,7 @@ func (s *FlowStore) Add(f Flow) Flow { s.mu.Lock() defer s.mu.Unlock() s.seq++ - f.Exchange.ID = strconv.Itoa(s.seq) + f.ID = strconv.Itoa(s.seq) if len(s.flows) >= s.cap { copy(s.flows, s.flows[1:]) s.flows[len(s.flows)-1] = f @@ -366,7 +366,7 @@ func (s *FlowStore) Get(id int) *Flow { defer s.mu.RUnlock() want := strconv.Itoa(id) for i := range s.flows { - if s.flows[i].Exchange.ID == want { + if s.flows[i].ID == want { f := s.flows[i] return &f } @@ -434,7 +434,7 @@ func formatFlowList(flows []Flow) string { errMark = " ERR" } sb.WriteString(fmt.Sprintf(" %-6s %-6s %-4d %-50s %-14s %dms%s\n", - f.Exchange.ID, f.Request.Method, statusCodeOf(&f), urlStr, truncate(ct, 14), f.Duration.Milliseconds(), errMark)) + f.ID, f.Request.Method, statusCodeOf(&f), urlStr, truncate(ct, 14), f.Duration.Milliseconds(), errMark)) } return sb.String() } @@ -449,7 +449,7 @@ func statusCodeOf(f *Flow) int { func formatFlowDetail(f *Flow) string { var sb strings.Builder - sb.WriteString(fmt.Sprintf("=== Flow #%s ===\n", f.Exchange.ID)) + sb.WriteString(fmt.Sprintf("=== Flow #%s ===\n", f.ID)) sb.WriteString(fmt.Sprintf("Time: %s Method: %s Status: %d Duration: %dms TLS: %v\n", f.Timestamp.Format(time.RFC3339), f.Request.Method, statusCodeOf(f), f.Duration.Milliseconds(), f.TLS)) sb.WriteString(fmt.Sprintf("URL: %s\n", f.Request.URL)) @@ -498,7 +498,7 @@ func formatFlowAnalysis(flows []Flow) string { sb.WriteString("\n\n") for _, f := range flows { - sb.WriteString(fmt.Sprintf("#%s [%d] %s %s (%dms)\n", f.Exchange.ID, statusCodeOf(&f), f.Request.Method, f.Request.URL, f.Duration.Milliseconds())) + sb.WriteString(fmt.Sprintf("#%s [%d] %s %s (%dms)\n", f.ID, statusCodeOf(&f), f.Request.Method, f.Request.URL, f.Duration.Milliseconds())) if f.Error != "" { sb.WriteString(fmt.Sprintf(" ERROR: %s\n", f.Error)) }