diff --git a/README.md b/README.md index 4dac7de..207e7b1 100644 --- a/README.md +++ b/README.md @@ -56,7 +56,7 @@ The format registry contains: - ZIP, TAR, ar, gzip, bzip2, xz, zstd, PDF, CFBF, PNG, JPEG, and GIF - ELF, Mach-O (thin and universal), PE/COFF, and WebAssembly -- plain text, HTML, XML, and SVG +- plain text, JSON, HTML, XML, and SVG Detection uses bytes only. ZIP-based package types such as JAR, wheel, and NuGet remain `zip`, and compressed payloads are not opened. A `CA FE BA BE` @@ -71,6 +71,11 @@ carriage return, and escape are the permitted C0 controls. Other C0 controls classify the input as binary. Invalid UTF-8 without a NUL is unknown with `ReasonInvalidText`; callers that need Latin-1 can apply their own fallback. +JSON detection validates the complete input, including arrays and scalar +top-level values. Surrounding JSON whitespace is accepted. A bounded prefix +that contains valid or incomplete JSON syntax reports JSON with +`ReasonNeedMore` because later bytes can complete or invalidate the value. + HTML, XML, and SVG signatures supply format metadata before the shared text rules run. The metadata remains present if malformed or control-bearing input is classified as unknown or binary. @@ -78,10 +83,10 @@ is classified as unknown or binary. ## Performance The detector performs no allocations for the supplied fixtures. On an Apple -M1 Pro with Go 1.26.5, a 4 KiB text input takes about 1.5 microseconds, the -mixed 4 KiB fixture corpus averages about 0.77 microseconds per call, and a +M1 Pro with Go 1.26.6, a 4 KiB text input takes about 1.5 microseconds, the +mixed 4 KiB fixture corpus averages about 2.1 microseconds per call, and a 1 MiB text input takes about 0.35 milliseconds. Importing and calling the -package adds 16,640 bytes to a stripped minimal binary. +package adds about 20 KiB to a stripped minimal binary. Run the package benchmarks on the target machine: @@ -89,9 +94,10 @@ Run the package benchmarks on the target machine: go test -run '^$' -bench . -benchmem ``` -The implementation scans at most 512 bytes for registered signatures. Text -validation is linear in the supplied byte count and uses fixed auxiliary -memory. +The implementation scans at most 512 bytes for fixed signatures. JSON parsing +and text validation are linear in the supplied byte count. JSON parsing uses +auxiliary memory proportional to nesting depth; text validation uses fixed +auxiliary memory. ## Provenance diff --git a/benchmark_test.go b/benchmark_test.go index 3dbf311..91151bd 100644 --- a/benchmark_test.go +++ b/benchmark_test.go @@ -8,11 +8,13 @@ import ( var benchmarkResult Result var benchmarkText1MiB = bytes.Repeat([]byte{'x'}, 1<<20) +var benchmarkJSON4KiB = jsonFixture(4096) var benchmarkCorpus = [][]byte{ padFixture([]byte("package magic\n\nfunc Detect(data []byte) Result { return Result{} }\n")), padFixture([]byte("\xef\xbb\xbfUnicode text: héllo, 世界\n")), padFixture([]byte("")), + benchmarkJSON4KiB, padFixture([]byte("PK\x03\x04")), padFixture([]byte("\x89PNG\r\n\x1a\n")), padFixture([]byte{0xff}), @@ -72,3 +74,10 @@ func padFixture(prefix []byte) []byte { } return append(bytes.Clone(prefix), bytes.Repeat([]byte{'x'}, 4096-len(prefix))...) } + +func jsonFixture(size int) []byte { + data := bytes.Repeat([]byte{'x'}, size) + data[0] = '"' + data[len(data)-1] = '"' + return data +} diff --git a/fuzz_test.go b/fuzz_test.go index db2fefc..7096b28 100644 --- a/fuzz_test.go +++ b/fuzz_test.go @@ -1,6 +1,12 @@ package magic -import "testing" +import ( + "bytes" + "encoding/json" + "fmt" + "testing" + "unicode/utf8" +) func FuzzDetect(f *testing.F) { seeds := [][]byte{ @@ -26,11 +32,18 @@ func FuzzDetect(f *testing.F) { []byte(""), []byte(""), []byte(""), + []byte(`{"schemaVersion":2}`), + []byte(`[1,"two",false,null]`), + []byte(`"value"`), + []byte(`1e+2`), + []byte(`{"truncated"`), + []byte(`1e+`), } for _, seed := range seeds { f.Add(seed) } f.Add(makeTAR(f)) + f.Add(makeJSONTARCollision(f)) f.Fuzz(func(t *testing.T, data []byte) { first := Detect(data) @@ -39,10 +52,19 @@ func FuzzDetect(f *testing.F) { t.Fatalf("Detect is not deterministic: %#v then %#v", first, second) } assertResultInvariants(t, first, false, len(data)) + binary, _ := binaryFormat(data) + expectJSON := binary == "" && json.Valid(data) && utf8.Valid(data) + if got := first.Format == FormatJSON; got != expectJSON { + t.Fatalf("Detect JSON match = %v, want %v for %x", got, expectJSON, data) + } prefix := DetectPrefix(data) if len(data) > 0 { expectedPrefix := first + if parseJSON(data) == jsonIncomplete { + expectedPrefix.Format = FormatJSON + expectedPrefix.MIME = mimeJSON + } if prefix.Reason == ReasonNeedMore { expectedPrefix.Reason = ReasonNeedMore } @@ -51,12 +73,39 @@ func FuzzDetect(f *testing.F) { } } - if format, _ := binaryFormat(data); format != "" && prefix != first { + if binary != "" && prefix != first { t.Fatalf("terminal binary signature changed for prefix: %#v, complete: %#v", prefix, first) } }) } +func makeJSONTARCollision(t testing.TB) []byte { + t.Helper() + + data := bytes.Repeat([]byte{'a'}, sniffLength) + data[0] = '"' + data[len(data)-1] = '"' + copy(data[tarMagicOffset:tarMagicEnd], "ustar ") + + checksum := 0 + for index, value := range data { + if index >= tarChecksumFrom && index < tarChecksumTo { + checksum += ' ' + } else { + checksum += int(value) + } + } + copy(data[tarChecksumFrom:tarChecksumTo], fmt.Sprintf("%06o ", checksum)) + + if !json.Valid(data) { + t.Fatal("JSON/TAR fixture is not valid JSON") + } + if format, _ := binaryFormat(data); format != FormatTAR { + t.Fatalf("JSON/TAR fixture format = %q, want %q", format, FormatTAR) + } + return data +} + func FuzzDetectPrefix(f *testing.F) { seeds := [][]byte{ nil, @@ -67,6 +116,9 @@ func FuzzDetectPrefix(f *testing.F) { []byte("\x89PNG\r\n"), []byte("\x89PNG\r\n\x1a\n"), []byte(""), + []byte(`{"schemaVersion":2}`), + []byte(`{"truncated"`), + []byte(`1e+`), } for _, seed := range seeds { f.Add(seed) diff --git a/json.go b/json.go new file mode 100644 index 0000000..d373dea --- /dev/null +++ b/json.go @@ -0,0 +1,406 @@ +package magic + +import "unicode/utf8" + +type jsonParseResult uint8 + +const ( + jsonInvalid jsonParseResult = iota + jsonIncomplete + jsonComplete + + jsonControlLimit = 0x20 + jsonInlineDepth = 64 + jsonMaximumDepth = 10000 +) + +type jsonContainer uint8 + +const ( + jsonArray jsonContainer = iota + jsonObject +) + +type jsonExpectation uint8 + +const ( + jsonExpectValue jsonExpectation = iota + jsonExpectArrayValueOrEnd + jsonExpectObjectKeyOrEnd + jsonExpectObjectKey + jsonExpectObjectColon + jsonExpectArrayCommaOrEnd + jsonExpectObjectCommaOrEnd + jsonExpectDocumentEnd +) + +type jsonParser struct { + data []byte + offset int + depth int + containers [jsonInlineDepth]jsonContainer + extraContainers []jsonContainer +} + +func isJSON(data []byte, prefix bool) bool { + result := parseJSON(data) + return result == jsonComplete || prefix && result == jsonIncomplete +} + +func parseJSON(data []byte) jsonParseResult { + parser := jsonParser{data: data} + parser.skipWhitespace() + if parser.offset == len(parser.data) { + return jsonInvalid + } + return parser.parse() +} + +func (parser *jsonParser) parse() jsonParseResult { + expectation := jsonExpectValue + for { + parser.skipWhitespace() + if parser.offset == len(parser.data) { + if expectation == jsonExpectDocumentEnd { + return jsonComplete + } + return jsonIncomplete + } + + next, result := parser.parseExpectation(expectation) + if result != jsonComplete { + return result + } + expectation = next + } +} + +func (parser *jsonParser) parseExpectation( + expectation jsonExpectation, +) (jsonExpectation, jsonParseResult) { + switch expectation { + case jsonExpectValue: + return parser.parseExpectedValue() + case jsonExpectArrayValueOrEnd: + return parser.parseExpectedArrayValueOrEnd() + case jsonExpectObjectKeyOrEnd: + return parser.parseExpectedObjectKeyOrEnd() + case jsonExpectObjectKey: + return parser.parseExpectedObjectKey() + case jsonExpectObjectColon: + return parser.parseExpectedObjectColon() + case jsonExpectArrayCommaOrEnd: + return parser.parseExpectedArrayCommaOrEnd() + case jsonExpectObjectCommaOrEnd: + return parser.parseExpectedObjectCommaOrEnd() + default: + return expectation, jsonInvalid + } +} + +func (parser *jsonParser) parseExpectedValue() (jsonExpectation, jsonParseResult) { + switch parser.data[parser.offset] { + case '{': + return jsonExpectObjectKeyOrEnd, parser.openContainer(jsonObject) + case '[': + return jsonExpectArrayValueOrEnd, parser.openContainer(jsonArray) + default: + result := parser.parseScalar() + return parser.expectAfterValue(), result + } +} + +func (parser *jsonParser) parseExpectedArrayValueOrEnd() ( + jsonExpectation, + jsonParseResult, +) { + if parser.data[parser.offset] != ']' { + return jsonExpectValue, jsonComplete + } + parser.closeContainer() + return parser.expectAfterValue(), jsonComplete +} + +func (parser *jsonParser) parseExpectedObjectKeyOrEnd() ( + jsonExpectation, + jsonParseResult, +) { + if parser.data[parser.offset] != '}' { + return jsonExpectObjectKey, jsonComplete + } + parser.closeContainer() + return parser.expectAfterValue(), jsonComplete +} + +func (parser *jsonParser) parseExpectedObjectKey() ( + jsonExpectation, + jsonParseResult, +) { + if parser.data[parser.offset] != '"' { + return jsonExpectObjectColon, jsonInvalid + } + return jsonExpectObjectColon, parser.parseString() +} + +func (parser *jsonParser) parseExpectedObjectColon() ( + jsonExpectation, + jsonParseResult, +) { + if parser.data[parser.offset] != ':' { + return jsonExpectValue, jsonInvalid + } + parser.offset++ + return jsonExpectValue, jsonComplete +} + +func (parser *jsonParser) parseExpectedArrayCommaOrEnd() ( + jsonExpectation, + jsonParseResult, +) { + switch parser.data[parser.offset] { + case ']': + parser.closeContainer() + return parser.expectAfterValue(), jsonComplete + case ',': + parser.offset++ + return jsonExpectValue, jsonComplete + default: + return jsonExpectValue, jsonInvalid + } +} + +func (parser *jsonParser) parseExpectedObjectCommaOrEnd() ( + jsonExpectation, + jsonParseResult, +) { + switch parser.data[parser.offset] { + case '}': + parser.closeContainer() + return parser.expectAfterValue(), jsonComplete + case ',': + parser.offset++ + return jsonExpectObjectKey, jsonComplete + default: + return jsonExpectObjectKey, jsonInvalid + } +} + +func (parser *jsonParser) parseScalar() jsonParseResult { + switch parser.data[parser.offset] { + case '"': + return parser.parseString() + case 't': + return parser.parseLiteral("true") + case 'f': + return parser.parseLiteral("false") + case 'n': + return parser.parseLiteral("null") + case '-': + return parser.parseNumber() + default: + if isDigit(parser.data[parser.offset]) { + return parser.parseNumber() + } + return jsonInvalid + } +} + +func (parser *jsonParser) openContainer(container jsonContainer) jsonParseResult { + if parser.depth == jsonMaximumDepth { + return jsonInvalid + } + if parser.depth < len(parser.containers) { + parser.containers[parser.depth] = container + } else { + parser.extraContainers = append(parser.extraContainers, container) + } + parser.depth++ + parser.offset++ + return jsonComplete +} + +func (parser *jsonParser) closeContainer() { + parser.depth-- + if parser.depth >= len(parser.containers) { + parser.extraContainers = parser.extraContainers[:len(parser.extraContainers)-1] + } + parser.offset++ +} + +func (parser *jsonParser) expectAfterValue() jsonExpectation { + if parser.depth == 0 { + return jsonExpectDocumentEnd + } + if parser.currentContainer() == jsonArray { + return jsonExpectArrayCommaOrEnd + } + return jsonExpectObjectCommaOrEnd +} + +func (parser *jsonParser) currentContainer() jsonContainer { + index := parser.depth - 1 + if index < len(parser.containers) { + return parser.containers[index] + } + return parser.extraContainers[index-len(parser.containers)] +} + +func (parser *jsonParser) parseString() jsonParseResult { + parser.offset++ + for parser.offset < len(parser.data) { + value := parser.data[parser.offset] + switch { + case value == '"': + parser.offset++ + return jsonComplete + case value == '\\': + parser.offset++ + if parser.offset == len(parser.data) { + return jsonIncomplete + } + escape := parser.data[parser.offset] + parser.offset++ + switch escape { + case '"', '\\', '/', 'b', 'f', 'n', 'r', 't': + case 'u': + for range 4 { + if parser.offset == len(parser.data) { + return jsonIncomplete + } + if !isHexadecimal(parser.data[parser.offset]) { + return jsonInvalid + } + parser.offset++ + } + default: + return jsonInvalid + } + case value < jsonControlLimit: + return jsonInvalid + case value < utf8.RuneSelf: + parser.offset++ + default: + remaining := parser.data[parser.offset:] + if !utf8.FullRune(remaining) { + return jsonIncomplete + } + runeValue, size := utf8.DecodeRune(remaining) + if runeValue == utf8.RuneError && size == 1 { + return jsonInvalid + } + parser.offset += size + } + } + return jsonIncomplete +} + +func (parser *jsonParser) parseLiteral(literal string) jsonParseResult { + for index := range len(literal) { + if parser.offset == len(parser.data) { + return jsonIncomplete + } + if parser.data[parser.offset] != literal[index] { + return jsonInvalid + } + parser.offset++ + } + return jsonComplete +} + +func (parser *jsonParser) parseNumber() jsonParseResult { + if parser.data[parser.offset] == '-' { + parser.offset++ + } + + if result := parser.parseInteger(); result != jsonComplete { + return result + } + + if parser.offset < len(parser.data) && parser.data[parser.offset] == '.' { + parser.offset++ + if result := parser.parseDigits(); result != jsonComplete { + return result + } + } + + if parser.offset < len(parser.data) && + (parser.data[parser.offset] == 'e' || parser.data[parser.offset] == 'E') { + parser.offset++ + if parser.offset == len(parser.data) { + return jsonIncomplete + } + if parser.data[parser.offset] == '+' || parser.data[parser.offset] == '-' { + parser.offset++ + } + if result := parser.parseDigits(); result != jsonComplete { + return result + } + } + + return jsonComplete +} + +func (parser *jsonParser) parseInteger() jsonParseResult { + if parser.offset == len(parser.data) { + return jsonIncomplete + } + + value := parser.data[parser.offset] + if value == '0' { + parser.offset++ + if parser.offset < len(parser.data) && isDigit(parser.data[parser.offset]) { + return jsonInvalid + } + return jsonComplete + } + if value < '1' || value > '9' { + return jsonInvalid + } + + parser.offset++ + parser.skipDigits() + return jsonComplete +} + +func (parser *jsonParser) parseDigits() jsonParseResult { + if parser.offset == len(parser.data) { + return jsonIncomplete + } + if !isDigit(parser.data[parser.offset]) { + return jsonInvalid + } + + parser.skipDigits() + return jsonComplete +} + +func (parser *jsonParser) skipDigits() { + for parser.offset < len(parser.data) && isDigit(parser.data[parser.offset]) { + parser.offset++ + } +} + +func (parser *jsonParser) skipWhitespace() { + for parser.offset < len(parser.data) && isJSONWhitespace(parser.data[parser.offset]) { + parser.offset++ + } +} + +func isJSONWhitespace(value byte) bool { + switch value { + case ' ', '\t', '\n', '\r': + return true + default: + return false + } +} + +func isHexadecimal(value byte) bool { + return value >= '0' && value <= '9' || + value >= 'a' && value <= 'f' || + value >= 'A' && value <= 'F' +} + +func isDigit(value byte) bool { + return value >= '0' && value <= '9' +} diff --git a/json_test.go b/json_test.go new file mode 100644 index 0000000..c687ad3 --- /dev/null +++ b/json_test.go @@ -0,0 +1,203 @@ +package magic + +import ( + "strings" + "testing" +) + +func TestJSONDetection(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + }{ + {name: "object", input: `{"schemaVersion": 2}`}, + {name: "array", input: `[1, "two", false, null]`}, + {name: "string", input: `"hello"`}, + {name: "number", input: `-12.5e+2`}, + {name: "true", input: `true`}, + {name: "false", input: `false`}, + {name: "null", input: `null`}, + {name: "surrounding whitespace", input: " \t\r\n{\"key\": \"value\"}\n"}, + {name: "Unicode", input: `{"message":"héllo, 世界","escaped":"\uD834\uDD1E"}`}, + {name: "escaped characters", input: `["\b\f\n\r\t\/\\\""]`}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + assertResult(t, Detect([]byte(test.input)), Result{ + Kind: KindText, + MIME: mimeJSON, + Format: FormatJSON, + Encoding: encodingUTF8, + }) + }) + } +} + +func TestInvalidJSONRetainsExistingClassification(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + }{ + {name: "mismatched delimiters", input: `{]`}, + {name: "whitespace only", input: " \t\r\n"}, + {name: "missing value", input: `{"key":}`}, + {name: "single quoted key", input: `{'key': 1}`}, + {name: "trailing comma", input: `[1,]`}, + {name: "unterminated string", input: `"value`}, + {name: "truncated literal", input: `tru`}, + {name: "leading zero", input: `01`}, + {name: "truncated fraction", input: `1.`}, + {name: "truncated exponent", input: `1e+`}, + {name: "invalid escape", input: `"\x"`}, + {name: "unescaped control", input: "\"value\tvalue\""}, + {name: "leading form feed", input: "\f{}"}, + {name: "trailing form feed", input: "{}\f"}, + {name: "trailing content", input: `{}x`}, + {name: "second value", input: `true false`}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + assertResult(t, Detect([]byte(test.input)), Result{ + Kind: KindText, + MIME: mimeText, + Format: FormatText, + Encoding: encodingUTF8, + }) + }) + } +} + +func TestInvalidUTF8JSONRetainsUnknownClassification(t *testing.T) { + t.Parallel() + + tests := [][]byte{ + {'"', 0xff, '"'}, + {'{', '}', 0xff}, + } + for _, input := range tests { + assertResult(t, Detect(input), Result{ + Kind: KindUnknown, + Reason: ReasonInvalidText, + }) + } +} + +func TestJSONNestingDepth(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + open string + close string + }{ + {name: "arrays", open: "[", close: "]"}, + {name: "objects", open: `{"value":`, close: "}"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + input := strings.Repeat(test.open, jsonMaximumDepth) + "0" + + strings.Repeat(test.close, jsonMaximumDepth) + if got := Detect([]byte(input)); got.Format != FormatJSON { + t.Fatalf("Detect() = %#v, want JSON at maximum nesting depth", got) + } + + input = test.open + input + test.close + if got := Detect([]byte(input)); got.Format == FormatJSON { + t.Fatalf("Detect() = %#v, want nesting depth limit to reject JSON", got) + } + }) + } +} + +func TestJSONInlineNestingDoesNotAllocate(t *testing.T) { + input := []byte(strings.Repeat("[", jsonInlineDepth) + "0" + + strings.Repeat("]", jsonInlineDepth)) + + if allocations := testing.AllocsPerRun(1000, func() { + Detect(input) + }); allocations != 0 { + t.Fatalf("Detect allocated %.2f times for inline JSON nesting", allocations) + } +} + +func TestJSONPrefixDetection(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + }{ + {name: "complete object", input: `{}`}, + {name: "truncated object", input: `{`}, + {name: "truncated object key", input: `{"key`}, + {name: "truncated object value", input: `{"key":`}, + {name: "truncated array", input: `[`}, + {name: "truncated array value", input: `[1,`}, + {name: "truncated string", input: `"value`}, + {name: "truncated escape", input: `"value\`}, + {name: "truncated Unicode escape", input: `"value\u12`}, + {name: "truncated negative number", input: `-`}, + {name: "truncated fraction", input: `1.`}, + {name: "truncated exponent", input: `1e`}, + {name: "truncated signed exponent", input: `1e+`}, + {name: "truncated true", input: `tru`}, + {name: "truncated false", input: `fals`}, + {name: "truncated null", input: `nul`}, + {name: "leading whitespace", input: " \n{"}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + assertResult(t, DetectPrefix([]byte(test.input)), Result{ + Kind: KindText, + MIME: mimeJSON, + Format: FormatJSON, + Encoding: encodingUTF8, + Reason: ReasonNeedMore, + }) + }) + } +} + +func TestInvalidJSONPrefixRetainsExistingClassification(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + }{ + {name: "plain text", input: `package`}, + {name: "whitespace only", input: " \t\r\n"}, + {name: "mismatched delimiters", input: `{]`}, + {name: "trailing comma", input: `[1,]`}, + {name: "leading zero", input: `01`}, + {name: "invalid literal", input: `truex`}, + {name: "invalid escape", input: `"\x"`}, + {name: "trailing content", input: `{}x`}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + assertResult(t, DetectPrefix([]byte(test.input)), Result{ + Kind: KindText, + MIME: mimeText, + Format: FormatText, + Encoding: encodingUTF8, + Reason: ReasonNeedMore, + }) + }) + } +} diff --git a/magic.go b/magic.go index 4b265ae..cbfff0b 100644 --- a/magic.go +++ b/magic.go @@ -46,6 +46,7 @@ const ( FormatHTML = "html" FormatXML = "xml" FormatSVG = "svg" + FormatJSON = "json" FormatZIP = "zip" FormatTAR = "tar" FormatGZIP = "gzip" @@ -69,6 +70,7 @@ const ( mimeHTML = "text/html" mimeXML = "text/xml" mimeSVG = "image/svg+xml" + mimeJSON = "application/json" mimeZIP = "application/zip" mimeTAR = "application/x-tar" mimeGZIP = "application/gzip" @@ -117,6 +119,10 @@ func detect(data []byte, prefix bool) Result { } format, mime := textFormat(data) + if format == "" && isJSON(data, prefix) { + format = FormatJSON + mime = mimeJSON + } result := classifyText(data) if format != "" { result.Format = format diff --git a/magic_test.go b/magic_test.go index 39c7bf0..0b9fc86 100644 --- a/magic_test.go +++ b/magic_test.go @@ -133,6 +133,7 @@ func TestDetectAllocations(t *testing.T) { inputs := [][]byte{ make([]byte, 4096), []byte("package magic\n"), + []byte(`{"schemaVersion":2}`), []byte("\xff\xfeh\x00i\x00"), []byte("\x89PNG\r\n\x1a\n"), }