Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 13 additions & 7 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ The format registry contains:

- ZIP, TAR, ar, gzip, bzip2, xz, zstd, PDF, CFBF, PNG, JPEG, and GIF
- ELF, Mach-O (thin and universal), PE/COFF, and WebAssembly
- plain text, HTML, XML, and SVG
- plain text, JSON, HTML, XML, and SVG

Detection uses bytes only. ZIP-based package types such as JAR, wheel, and
NuGet remain `zip`, and compressed payloads are not opened. A `CA FE BA BE`
Expand All @@ -71,27 +71,33 @@ carriage return, and escape are the permitted C0 controls. Other C0 controls
classify the input as binary. Invalid UTF-8 without a NUL is unknown with
`ReasonInvalidText`; callers that need Latin-1 can apply their own fallback.

JSON detection validates the complete input, including arrays and scalar
top-level values. Surrounding JSON whitespace is accepted. A bounded prefix
that contains valid or incomplete JSON syntax reports JSON with
`ReasonNeedMore` because later bytes can complete or invalidate the value.

HTML, XML, and SVG signatures supply format metadata before the shared text
rules run. The metadata remains present if malformed or control-bearing input
is classified as unknown or binary.

## Performance

The detector performs no allocations for the supplied fixtures. On an Apple
M1 Pro with Go 1.26.5, a 4 KiB text input takes about 1.5 microseconds, the
mixed 4 KiB fixture corpus averages about 0.77 microseconds per call, and a
M1 Pro with Go 1.26.6, a 4 KiB text input takes about 1.5 microseconds, the
mixed 4 KiB fixture corpus averages about 2.1 microseconds per call, and a
1 MiB text input takes about 0.35 milliseconds. Importing and calling the
package adds 16,640 bytes to a stripped minimal binary.
package adds about 20 KiB to a stripped minimal binary.

Run the package benchmarks on the target machine:

```bash
go test -run '^$' -bench . -benchmem
```

The implementation scans at most 512 bytes for registered signatures. Text
validation is linear in the supplied byte count and uses fixed auxiliary
memory.
The implementation scans at most 512 bytes for fixed signatures. JSON parsing
and text validation are linear in the supplied byte count. JSON parsing uses
auxiliary memory proportional to nesting depth; text validation uses fixed
auxiliary memory.

## Provenance

Expand Down
9 changes: 9 additions & 0 deletions benchmark_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,13 @@ import (
var benchmarkResult Result

var benchmarkText1MiB = bytes.Repeat([]byte{'x'}, 1<<20)
var benchmarkJSON4KiB = jsonFixture(4096)

var benchmarkCorpus = [][]byte{
padFixture([]byte("package magic\n\nfunc Detect(data []byte) Result { return Result{} }\n")),
padFixture([]byte("\xef\xbb\xbfUnicode text: héllo, 世界\n")),
padFixture([]byte("<?xml version=\"1.0\"?><svg >")),
benchmarkJSON4KiB,
padFixture([]byte("PK\x03\x04")),
padFixture([]byte("\x89PNG\r\n\x1a\n")),
padFixture([]byte{0xff}),
Expand Down Expand Up @@ -72,3 +74,10 @@ func padFixture(prefix []byte) []byte {
}
return append(bytes.Clone(prefix), bytes.Repeat([]byte{'x'}, 4096-len(prefix))...)
}

func jsonFixture(size int) []byte {
data := bytes.Repeat([]byte{'x'}, size)
data[0] = '"'
data[len(data)-1] = '"'
return data
}
56 changes: 54 additions & 2 deletions fuzz_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,12 @@
package magic

import "testing"
import (
"bytes"
"encoding/json"
"fmt"
"testing"
"unicode/utf8"
)

func FuzzDetect(f *testing.F) {
seeds := [][]byte{
Expand All @@ -26,11 +32,18 @@ func FuzzDetect(f *testing.F) {
[]byte("<html>"),
[]byte("<?xml version=\"1.0\"?>"),
[]byte("<svg>"),
[]byte(`{"schemaVersion":2}`),
[]byte(`[1,"two",false,null]`),
[]byte(`"value"`),
[]byte(`1e+2`),
[]byte(`{"truncated"`),
[]byte(`1e+`),
}
for _, seed := range seeds {
f.Add(seed)
}
f.Add(makeTAR(f))
f.Add(makeJSONTARCollision(f))

f.Fuzz(func(t *testing.T, data []byte) {
first := Detect(data)
Expand All @@ -39,10 +52,19 @@ func FuzzDetect(f *testing.F) {
t.Fatalf("Detect is not deterministic: %#v then %#v", first, second)
}
assertResultInvariants(t, first, false, len(data))
binary, _ := binaryFormat(data)
expectJSON := binary == "" && json.Valid(data) && utf8.Valid(data)
if got := first.Format == FormatJSON; got != expectJSON {
t.Fatalf("Detect JSON match = %v, want %v for %x", got, expectJSON, data)
}

prefix := DetectPrefix(data)
if len(data) > 0 {
expectedPrefix := first
if parseJSON(data) == jsonIncomplete {
expectedPrefix.Format = FormatJSON
expectedPrefix.MIME = mimeJSON
}
if prefix.Reason == ReasonNeedMore {
expectedPrefix.Reason = ReasonNeedMore
}
Expand All @@ -51,12 +73,39 @@ func FuzzDetect(f *testing.F) {
}
}

if format, _ := binaryFormat(data); format != "" && prefix != first {
if binary != "" && prefix != first {
t.Fatalf("terminal binary signature changed for prefix: %#v, complete: %#v", prefix, first)
}
})
}

func makeJSONTARCollision(t testing.TB) []byte {
t.Helper()

data := bytes.Repeat([]byte{'a'}, sniffLength)
data[0] = '"'
data[len(data)-1] = '"'
copy(data[tarMagicOffset:tarMagicEnd], "ustar ")

checksum := 0
for index, value := range data {
if index >= tarChecksumFrom && index < tarChecksumTo {
checksum += ' '
} else {
checksum += int(value)
}
}
copy(data[tarChecksumFrom:tarChecksumTo], fmt.Sprintf("%06o ", checksum))

if !json.Valid(data) {
t.Fatal("JSON/TAR fixture is not valid JSON")
}
if format, _ := binaryFormat(data); format != FormatTAR {
t.Fatalf("JSON/TAR fixture format = %q, want %q", format, FormatTAR)
}
return data
}

func FuzzDetectPrefix(f *testing.F) {
seeds := [][]byte{
nil,
Expand All @@ -67,6 +116,9 @@ func FuzzDetectPrefix(f *testing.F) {
[]byte("\x89PNG\r\n"),
[]byte("\x89PNG\r\n\x1a\n"),
[]byte("<svg>"),
[]byte(`{"schemaVersion":2}`),
[]byte(`{"truncated"`),
[]byte(`1e+`),
}
for _, seed := range seeds {
f.Add(seed)
Expand Down
Loading