diff --git a/dataconv/helper.go b/dataconv/helper.go index cf6bad3..75ed890 100644 --- a/dataconv/helper.go +++ b/dataconv/helper.go @@ -71,12 +71,12 @@ func MarshalStarlarkJSON(data starlark.Value, indent int) (string, error) { return strings.TrimSpace(bf.String()), nil } -// UnmarshalStarlarkJSON unmarshals JSON bytes into a starlark.Value. +// UnmarshalStarlarkJSON decodes one complete JSON document into a starlark.Value. // It decodes through json.Decoder.UseNumber to preserve integer precision, // applies TypeConvert (including time-string recognition), then calls Marshal. // DecodeStarlarkJSON instead uses the upstream native-value JSON decoder. -// Known limitation: malformed trailing closing delimiters can be accepted; -// callers requiring strict JSON syntax should check encoding/json.Valid first. +// Only JSON whitespace (space, tab, carriage return, newline) may follow +// the document. func UnmarshalStarlarkJSON(data []byte) (starlark.Value, error) { var m interface{} // decode with UseNumber so integers keep their exact value: a plain @@ -87,9 +87,9 @@ func UnmarshalStarlarkJSON(data []byte) (starlark.Value, error) { if err := dec.Decode(&m); err != nil { return starlark.None, err } - // Check for further input, allowing trailing whitespace. More does not - // reject every malformed suffix, including stray closing delimiters. - if dec.More() { + // Only JSON whitespace may follow the first value. More can miss stray + // closing delimiters; inspecting the suffix avoids decoding another value. + if len(bytes.TrimLeft(data[dec.InputOffset():], " \t\r\n")) != 0 { return starlark.None, fmt.Errorf("unexpected trailing data after JSON value") } diff --git a/dataconv/helper_test.go b/dataconv/helper_test.go index 3d7cba0..56dfb67 100644 --- a/dataconv/helper_test.go +++ b/dataconv/helper_test.go @@ -5,6 +5,7 @@ import ( "fmt" "math/big" "reflect" + "strings" "testing" "time" @@ -380,6 +381,39 @@ func TestUnmarshalStarlarkJSON(t *testing.T) { } } +// JSON document boundaries: closing delimiters are trailing data too, and +// only the four JSON whitespace bytes may follow a complete value. +func TestUnmarshalStarlarkJSONSingleDocument(t *testing.T) { + for _, input := range []string{ + `1]`, `1}`, `null]`, `null }`, `1] false`, " \t1 \r\n]", `{"a":1}]`, `{"a":1}}`, `[1]]`, `[1]}`, + `1 2`, `{} {}`, `1 junk`, `1 [`, + "1\v", "1\f", "1\u00a0", "1\ufeff", + "1 [" + strings.Repeat("0,", 8192) + "0]", + } { + name := input + if len(name) > 80 { + name = "large second document" + } + t.Run(name, func(t *testing.T) { + got, err := UnmarshalStarlarkJSON([]byte(input)) + if err == nil || !strings.Contains(err.Error(), "unexpected trailing data") { + t.Fatalf("UnmarshalStarlarkJSON(%q) = %v, %v; want trailing-data error", input, got, err) + } + if got != starlark.None { + t.Fatalf("on error got %v, want None", got) + } + }) + } + for _, input := range []string{"123456789012345678901234567890", " \t\r\n123456789012345678901234567890 \t\r\n"} { + t.Run(input, func(t *testing.T) { + got, err := UnmarshalStarlarkJSON([]byte(input)) + if err != nil || got.String() != "123456789012345678901234567890" || got.Type() != "int" { + t.Fatalf("UnmarshalStarlarkJSON(%q) = %v, %v; want exact integer", input, got, err) + } + }) + } +} + // TestUnmarshalStarlarkJSONNumberFidelity pins the number handling of // UnmarshalStarlarkJSON: integers survive exactly at any magnitude (decoding // uses UseNumber instead of collapsing through float64, which silently diff --git a/lib/json/README.md b/lib/json/README.md index d01ca69..1a8bb8c 100644 --- a/lib/json/README.md +++ b/lib/json/README.md @@ -173,7 +173,9 @@ print(result, error) `validate(data, schema) -> None` checks a JSON document against a [JSON Schema](https://json-schema.org) (drafts 4, 6, 7, 2019-09, 2020-12 — detected from `$schema`, default 2020-12). It returns `None` when the data conforms; otherwise it **errors** with one line per violation, each prefixed by its [JSON Pointer](https://datatracker.ietf.org/doc/html/rfc6901) location (e.g. `at /age: must be >= 0 but found -3`; long lists are capped with `... and N more`). -**Known limit:** `validate` / `try_validate` do not reliably reject malformed trailing closing delimiters in data or schema text (for example `1]` or `{"type":"integer"}]`). Check each text with `decode(data_text)` / `decode(schema_text)` first, then pass the **original texts** to `validate(data_text, schema_text)`. Passing decoded top-level strings would reinterpret them as JSON text. The [data-tail fix](https://github.com/1set/starlet/pull/185) is separate; schema parsing currently uses [`jsonschema/v5 v5.3.1`](https://github.com/santhosh-tekuri/jsonschema/blob/v5.3.1/resource.go#L269). This limitation does not describe `decode` / `try_decode`. +Data text must contain exactly one complete JSON value; surrounding JSON whitespace (space, tab, carriage return, newline) is allowed. + +**Known schema limit:** `validate` / `try_validate` can still accept malformed trailing closing delimiters in schema text (for example `{"type":"integer"}]`). Check schema text with `decode(schema_text)` first, then pass the **original text** to `validate(data, schema_text)`. Schema parsing currently uses [`jsonschema/v5 v5.3.1`](https://github.com/santhosh-tekuri/jsonschema/blob/v5.3.1/resource.go#L269). This limitation does not describe `decode` / `try_decode`. Schemas must be **self-contained**: an external `$ref` (`file://`, `http://`) is rejected (`not allowed`) — this is what keeps the module pure. A bad schema or malformed data text reports `invalid schema` / `invalid data`. Compiled schemas are cached (bounded), so repeated validation against the same schema text avoids recompilation. diff --git a/lib/json/json_test.go b/lib/json/json_test.go index 472c3c3..88e6d59 100644 --- a/lib/json/json_test.go +++ b/lib/json/json_test.go @@ -1,6 +1,7 @@ package json_test import ( + "strings" "testing" "github.com/1set/starlet/dataconv" @@ -1193,3 +1194,70 @@ func TestJSONValidate(t *testing.T) { }) } } + +// JSON document boundaries are cannot-run failures, not schema violations. +func TestValidateSingleDocument(t *testing.T) { + members, err := itn.NewAssertLoader(json.ModuleName, json.LoadModule)(&starlark.Thread{}, json.ModuleName) + if err != nil { + t.Fatal(err) + } + for _, input := range []string{ + `1]`, `1}`, `null]`, `null }`, `1] false`, " \t1 \r\n]", `{"a":1}]`, `{"a":1}}`, `[1]]`, `[1]}`, + `1 2`, `{} {}`, `1 junk`, `1 [`, + "1\v", "1\f", "1\u00a0", "1\ufeff", + "1 [" + strings.Repeat("0,", 8192) + "0]", + } { + name := input + if len(name) > 80 { + name = "large second document" + } + t.Run(name, func(t *testing.T) { + for _, source := range []starlark.Value{starlark.String(input), starlark.Bytes(input)} { + for _, name := range []string{"validate", "try_validate"} { + t.Run(name+"/"+source.Type(), func(t *testing.T) { + got, err := starlark.Call(&starlark.Thread{}, members[name], starlark.Tuple{source, starlark.String(`{}`)}, nil) + if name == "validate" { + if err == nil || !strings.Contains(err.Error(), "json.validate: invalid data: unexpected trailing content") { + t.Fatalf("validate(%q) = %v, %v; want invalid-data error", input, got, err) + } + return + } + if err != nil { + t.Fatalf("try_validate(%q): %v", input, err) + } + pair, ok := got.(starlark.Tuple) + if !ok || len(pair) != 2 || pair[0] != starlark.None || !strings.Contains(pair[1].String(), "invalid data: unexpected trailing content") { + t.Fatalf("try_validate(%q) = %v; want (None, invalid-data message)", input, got) + } + }) + } + } + }) + } + for _, input := range []string{"123456789012345678901234567890", " \t\r\n123456789012345678901234567890 \t\r\n"} { + t.Run(input, func(t *testing.T) { + for _, source := range []starlark.Value{starlark.String(input), starlark.Bytes(input)} { + for _, name := range []string{"validate", "try_validate"} { + got, err := starlark.Call(&starlark.Thread{}, members[name], starlark.Tuple{ + source, starlark.String(`{"const":123456789012345678901234567890}`), + }, nil) + if err != nil { + t.Fatal(err) + } + if name == "validate" && got != starlark.None { + t.Fatalf("validate(%q) = %v, want None", input, got) + } + if name == "try_validate" { + pair, ok := got.(starlark.Tuple) + if !ok || len(pair) != 2 { + t.Fatalf("try_validate(%q) = %v; want result/error tuple", input, got) + } + if pair[0] != starlark.True || pair[1] != starlark.None { + t.Fatalf("try_validate(%q) = %v, want (True, None)", input, got) + } + } + } + } + }) + } +} diff --git a/lib/json/validate.go b/lib/json/validate.go index 60f58d7..e75a309 100644 --- a/lib/json/validate.go +++ b/lib/json/validate.go @@ -84,8 +84,9 @@ func formatValidationError(ve *jsonschema.ValidationError) string { // pointer-per-violation message otherwise. try_validate distinguishes the // three outcomes: (True, None) valid, (False, details) invalid, and // (None, error) when validation could not run (bad schema, bad arguments, -// malformed JSON text). Known limitation: malformed trailing closing delimiters -// can be accepted in data or schema text; this is not strict syntax validation. +// malformed JSON text). Data text must contain exactly one complete JSON value. +// Known limitation: malformed trailing closing delimiters can still be accepted +// in schema text. func generateValidate(try bool) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { return func(_ *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { var data, schema starlark.Value @@ -137,11 +138,9 @@ func prepareValidation(data, schema starlark.Value) (*jsonschema.Schema, interfa if err := dec.Decode(&doc); err != nil { return nil, nil, fmt.Errorf("invalid data: %w", err) } - // Validate a single JSON document. Trailing content (a second document - // or garbage) means the whole input was never checked, so reject it - // rather than silently validating only the first value. dec.More() - // stays false for trailing whitespace. - if dec.More() { + // Only JSON whitespace may follow the first value. Inspect the suffix + // directly so rejected input does not decode a second document. + if len(bytes.TrimLeft(dataBytes[dec.InputOffset():], " \t\r\n")) != 0 { return nil, nil, fmt.Errorf("invalid data: unexpected trailing content after JSON document") } return compiled, doc, nil diff --git a/lib/serial/README.md b/lib/serial/README.md index 5757102..e264725 100644 --- a/lib/serial/README.md +++ b/lib/serial/README.md @@ -106,7 +106,7 @@ print(val, e1, bad, e2 != None) # Output: [1, 2, 3] None None True ``` -**Known limit:** `loads` / `try_loads` can accept malformed trailing closing delimiters, for example `1]`. They are not strict JSON syntax validators. Validate input syntax in the host first if that guarantee is required; the [tail-check fix](https://github.com/1set/starlet/pull/185) is separate from this documentation update. +`loads` / `try_loads` require one complete JSON value; only JSON whitespace (space, tab, carriage return, newline) may follow it. ## Notes / boundaries diff --git a/lib/serial/serial.go b/lib/serial/serial.go index e00d77c..490d5ea 100644 --- a/lib/serial/serial.go +++ b/lib/serial/serial.go @@ -124,10 +124,9 @@ func generateLoads(try bool) func(*starlark.Thread, *starlark.Builtin, starlark. if err := dec.Decode(&raw); err != nil { return failResult(try, err, fn, true) } - // Check for further input after the decoded value. More does not - // reject every malformed suffix, including stray closing delimiters; - // trailing whitespace/newlines still pass. - if dec.More() { + // Only JSON whitespace may follow the first value. Inspect the suffix + // directly so rejected input does not decode a second document. + if len(strings.TrimLeft(s[dec.InputOffset():], " \t\r\n")) != 0 { return failResult(try, fmt.Errorf("unexpected trailing data after JSON value"), fn, true) } val, err := decode(raw) diff --git a/lib/serial/serial_test.go b/lib/serial/serial_test.go index c866cd3..5dcb54a 100644 --- a/lib/serial/serial_test.go +++ b/lib/serial/serial_test.go @@ -1,6 +1,7 @@ package serial_test import ( + "strings" "testing" "time" @@ -427,3 +428,65 @@ func TestLoadModule_Serial(t *testing.T) { }) } } + +// JSON document boundaries preserve the throwing and try_loads error contracts. +func TestLoadsSingleDocument(t *testing.T) { + members, err := itn.NewAssertLoader(serial.ModuleName, serial.LoadModule)(&starlark.Thread{}, serial.ModuleName) + if err != nil { + t.Fatal(err) + } + for _, input := range []string{ + `1]`, `1}`, `null]`, `null }`, `1] false`, " \t1 \r\n]", `{"a":1}]`, `{"a":1}}`, `[1]]`, `[1]}`, + `1 2`, `{} {}`, `1 junk`, `1 [`, + "1\v", "1\f", "1\u00a0", "1\ufeff", + "1 [" + strings.Repeat("0,", 8192) + "0]", + } { + name := input + if len(name) > 80 { + name = "large second document" + } + t.Run(name, func(t *testing.T) { + for _, name := range []string{"loads", "try_loads"} { + t.Run(name, func(t *testing.T) { + got, err := starlark.Call(&starlark.Thread{}, members[name], starlark.Tuple{starlark.String(input)}, nil) + if name == "loads" { + if err == nil || !strings.Contains(err.Error(), "serial.loads: unexpected trailing data") { + t.Fatalf("loads(%q) = %v, %v; want trailing-data error", input, got, err) + } + return + } + if err != nil { + t.Fatalf("try_loads(%q): %v", input, err) + } + pair, ok := got.(starlark.Tuple) + if !ok || len(pair) != 2 || pair[0] != starlark.None || !strings.Contains(pair[1].String(), "unexpected trailing data") { + t.Fatalf("try_loads(%q) = %v; want (None, trailing-data message)", input, got) + } + }) + } + }) + } + for _, input := range []string{"123456789012345678901234567890", " \t\r\n123456789012345678901234567890 \t\r\n"} { + t.Run(input, func(t *testing.T) { + for _, name := range []string{"loads", "try_loads"} { + got, err := starlark.Call(&starlark.Thread{}, members[name], starlark.Tuple{starlark.String(input)}, nil) + if err != nil { + t.Fatal(err) + } + if name == "try_loads" { + pair, ok := got.(starlark.Tuple) + if !ok || len(pair) != 2 { + t.Fatalf("try_loads(%q) = %v; want result/error tuple", input, got) + } + if pair[1] != starlark.None { + t.Fatalf("try_loads(%q) error = %v", input, pair[1]) + } + got = pair[0] + } + if got.String() != "123456789012345678901234567890" || got.Type() != "int" { + t.Fatalf("%s(%q) = %v; want exact integer", name, input, got) + } + } + }) + } +}