Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions dataconv/helper.go
Original file line number Diff line number Diff line change
Expand Up @@ -71,12 +71,12 @@ func MarshalStarlarkJSON(data starlark.Value, indent int) (string, error) {
return strings.TrimSpace(bf.String()), nil
}

// UnmarshalStarlarkJSON unmarshals JSON bytes into a starlark.Value.
// UnmarshalStarlarkJSON decodes one complete JSON document into a starlark.Value.
// It decodes through json.Decoder.UseNumber to preserve integer precision,
// applies TypeConvert (including time-string recognition), then calls Marshal.
// DecodeStarlarkJSON instead uses the upstream native-value JSON decoder.
// Known limitation: malformed trailing closing delimiters can be accepted;
// callers requiring strict JSON syntax should check encoding/json.Valid first.
// Only JSON whitespace (space, tab, carriage return, newline) may follow
// the document.
func UnmarshalStarlarkJSON(data []byte) (starlark.Value, error) {
var m interface{}
// decode with UseNumber so integers keep their exact value: a plain
Expand All @@ -87,9 +87,9 @@ func UnmarshalStarlarkJSON(data []byte) (starlark.Value, error) {
if err := dec.Decode(&m); err != nil {
return starlark.None, err
}
// Check for further input, allowing trailing whitespace. More does not
// reject every malformed suffix, including stray closing delimiters.
if dec.More() {
// Only JSON whitespace may follow the first value. More can miss stray
// closing delimiters; inspecting the suffix avoids decoding another value.
if len(bytes.TrimLeft(data[dec.InputOffset():], " \t\r\n")) != 0 {
return starlark.None, fmt.Errorf("unexpected trailing data after JSON value")
}

Expand Down
34 changes: 34 additions & 0 deletions dataconv/helper_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"
"math/big"
"reflect"
"strings"
"testing"
"time"

Expand Down Expand Up @@ -380,6 +381,39 @@ func TestUnmarshalStarlarkJSON(t *testing.T) {
}
}

// JSON document boundaries: closing delimiters are trailing data too, and
// only the four JSON whitespace bytes may follow a complete value.
func TestUnmarshalStarlarkJSONSingleDocument(t *testing.T) {
for _, input := range []string{
`1]`, `1}`, `null]`, `null }`, `1] false`, " \t1 \r\n]", `{"a":1}]`, `{"a":1}}`, `[1]]`, `[1]}`,
`1 2`, `{} {}`, `1 junk`, `1 [`,
"1\v", "1\f", "1\u00a0", "1\ufeff",
"1 [" + strings.Repeat("0,", 8192) + "0]",
} {
name := input
if len(name) > 80 {
name = "large second document"
}
t.Run(name, func(t *testing.T) {
got, err := UnmarshalStarlarkJSON([]byte(input))
if err == nil || !strings.Contains(err.Error(), "unexpected trailing data") {
t.Fatalf("UnmarshalStarlarkJSON(%q) = %v, %v; want trailing-data error", input, got, err)
}
if got != starlark.None {
t.Fatalf("on error got %v, want None", got)
}
})
}
for _, input := range []string{"123456789012345678901234567890", " \t\r\n123456789012345678901234567890 \t\r\n"} {
t.Run(input, func(t *testing.T) {
got, err := UnmarshalStarlarkJSON([]byte(input))
if err != nil || got.String() != "123456789012345678901234567890" || got.Type() != "int" {
t.Fatalf("UnmarshalStarlarkJSON(%q) = %v, %v; want exact integer", input, got, err)
}
})
}
}

// TestUnmarshalStarlarkJSONNumberFidelity pins the number handling of
// UnmarshalStarlarkJSON: integers survive exactly at any magnitude (decoding
// uses UseNumber instead of collapsing through float64, which silently
Expand Down
4 changes: 3 additions & 1 deletion lib/json/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -173,7 +173,9 @@ print(result, error)

`validate(data, schema) -> None` checks a JSON document against a [JSON Schema](https://json-schema.org) (drafts 4, 6, 7, 2019-09, 2020-12 — detected from `$schema`, default 2020-12). It returns `None` when the data conforms; otherwise it **errors** with one line per violation, each prefixed by its [JSON Pointer](https://datatracker.ietf.org/doc/html/rfc6901) location (e.g. `at /age: must be >= 0 but found -3`; long lists are capped with `... and N more`).

**Known limit:** `validate` / `try_validate` do not reliably reject malformed trailing closing delimiters in data or schema text (for example `1]` or `{"type":"integer"}]`). Check each text with `decode(data_text)` / `decode(schema_text)` first, then pass the **original texts** to `validate(data_text, schema_text)`. Passing decoded top-level strings would reinterpret them as JSON text. The [data-tail fix](https://github.com/1set/starlet/pull/185) is separate; schema parsing currently uses [`jsonschema/v5 v5.3.1`](https://github.com/santhosh-tekuri/jsonschema/blob/v5.3.1/resource.go#L269). This limitation does not describe `decode` / `try_decode`.
Data text must contain exactly one complete JSON value; surrounding JSON whitespace (space, tab, carriage return, newline) is allowed.

**Known schema limit:** `validate` / `try_validate` can still accept malformed trailing closing delimiters in schema text (for example `{"type":"integer"}]`). Check schema text with `decode(schema_text)` first, then pass the **original text** to `validate(data, schema_text)`. Schema parsing currently uses [`jsonschema/v5 v5.3.1`](https://github.com/santhosh-tekuri/jsonschema/blob/v5.3.1/resource.go#L269). This limitation does not describe `decode` / `try_decode`.

Schemas must be **self-contained**: an external `$ref` (`file://`, `http://`) is rejected (`not allowed`) — this is what keeps the module pure. A bad schema or malformed data text reports `invalid schema` / `invalid data`. Compiled schemas are cached (bounded), so repeated validation against the same schema text avoids recompilation.

Expand Down
68 changes: 68 additions & 0 deletions lib/json/json_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package json_test

import (
"strings"
"testing"

"github.com/1set/starlet/dataconv"
Expand Down Expand Up @@ -1193,3 +1194,70 @@ func TestJSONValidate(t *testing.T) {
})
}
}

// JSON document boundaries are cannot-run failures, not schema violations.
func TestValidateSingleDocument(t *testing.T) {
members, err := itn.NewAssertLoader(json.ModuleName, json.LoadModule)(&starlark.Thread{}, json.ModuleName)
if err != nil {
t.Fatal(err)
}
for _, input := range []string{
`1]`, `1}`, `null]`, `null }`, `1] false`, " \t1 \r\n]", `{"a":1}]`, `{"a":1}}`, `[1]]`, `[1]}`,
`1 2`, `{} {}`, `1 junk`, `1 [`,
"1\v", "1\f", "1\u00a0", "1\ufeff",
"1 [" + strings.Repeat("0,", 8192) + "0]",
} {
name := input
if len(name) > 80 {
name = "large second document"
}
t.Run(name, func(t *testing.T) {
for _, source := range []starlark.Value{starlark.String(input), starlark.Bytes(input)} {
for _, name := range []string{"validate", "try_validate"} {
t.Run(name+"/"+source.Type(), func(t *testing.T) {
got, err := starlark.Call(&starlark.Thread{}, members[name], starlark.Tuple{source, starlark.String(`{}`)}, nil)
if name == "validate" {
if err == nil || !strings.Contains(err.Error(), "json.validate: invalid data: unexpected trailing content") {
t.Fatalf("validate(%q) = %v, %v; want invalid-data error", input, got, err)
}
return
}
if err != nil {
t.Fatalf("try_validate(%q): %v", input, err)
}
pair, ok := got.(starlark.Tuple)
if !ok || len(pair) != 2 || pair[0] != starlark.None || !strings.Contains(pair[1].String(), "invalid data: unexpected trailing content") {
t.Fatalf("try_validate(%q) = %v; want (None, invalid-data message)", input, got)
}
})
}
}
})
}
for _, input := range []string{"123456789012345678901234567890", " \t\r\n123456789012345678901234567890 \t\r\n"} {
t.Run(input, func(t *testing.T) {
for _, source := range []starlark.Value{starlark.String(input), starlark.Bytes(input)} {
for _, name := range []string{"validate", "try_validate"} {
got, err := starlark.Call(&starlark.Thread{}, members[name], starlark.Tuple{
source, starlark.String(`{"const":123456789012345678901234567890}`),
}, nil)
if err != nil {
t.Fatal(err)
}
if name == "validate" && got != starlark.None {
t.Fatalf("validate(%q) = %v, want None", input, got)
}
if name == "try_validate" {
pair, ok := got.(starlark.Tuple)
if !ok || len(pair) != 2 {
t.Fatalf("try_validate(%q) = %v; want result/error tuple", input, got)
}
if pair[0] != starlark.True || pair[1] != starlark.None {
t.Fatalf("try_validate(%q) = %v, want (True, None)", input, got)
}
}
}
}
})
}
}
13 changes: 6 additions & 7 deletions lib/json/validate.go
Original file line number Diff line number Diff line change
Expand Up @@ -84,8 +84,9 @@ func formatValidationError(ve *jsonschema.ValidationError) string {
// pointer-per-violation message otherwise. try_validate distinguishes the
// three outcomes: (True, None) valid, (False, details) invalid, and
// (None, error) when validation could not run (bad schema, bad arguments,
// malformed JSON text). Known limitation: malformed trailing closing delimiters
// can be accepted in data or schema text; this is not strict syntax validation.
// malformed JSON text). Data text must contain exactly one complete JSON value.
// Known limitation: malformed trailing closing delimiters can still be accepted
// in schema text.
func generateValidate(try bool) func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) {
return func(_ *starlark.Thread, fn *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) {
var data, schema starlark.Value
Expand Down Expand Up @@ -137,11 +138,9 @@ func prepareValidation(data, schema starlark.Value) (*jsonschema.Schema, interfa
if err := dec.Decode(&doc); err != nil {
return nil, nil, fmt.Errorf("invalid data: %w", err)
}
// Validate a single JSON document. Trailing content (a second document
// or garbage) means the whole input was never checked, so reject it
// rather than silently validating only the first value. dec.More()
// stays false for trailing whitespace.
if dec.More() {
// Only JSON whitespace may follow the first value. Inspect the suffix
// directly so rejected input does not decode a second document.
if len(bytes.TrimLeft(dataBytes[dec.InputOffset():], " \t\r\n")) != 0 {
return nil, nil, fmt.Errorf("invalid data: unexpected trailing content after JSON document")
}
return compiled, doc, nil
Expand Down
2 changes: 1 addition & 1 deletion lib/serial/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ print(val, e1, bad, e2 != None)
# Output: [1, 2, 3] None None True
```

**Known limit:** `loads` / `try_loads` can accept malformed trailing closing delimiters, for example `1]`. They are not strict JSON syntax validators. Validate input syntax in the host first if that guarantee is required; the [tail-check fix](https://github.com/1set/starlet/pull/185) is separate from this documentation update.
`loads` / `try_loads` require one complete JSON value; only JSON whitespace (space, tab, carriage return, newline) may follow it.

## Notes / boundaries

Expand Down
7 changes: 3 additions & 4 deletions lib/serial/serial.go
Original file line number Diff line number Diff line change
Expand Up @@ -124,10 +124,9 @@ func generateLoads(try bool) func(*starlark.Thread, *starlark.Builtin, starlark.
if err := dec.Decode(&raw); err != nil {
return failResult(try, err, fn, true)
}
// Check for further input after the decoded value. More does not
// reject every malformed suffix, including stray closing delimiters;
// trailing whitespace/newlines still pass.
if dec.More() {
// Only JSON whitespace may follow the first value. Inspect the suffix
// directly so rejected input does not decode a second document.
if len(strings.TrimLeft(s[dec.InputOffset():], " \t\r\n")) != 0 {
return failResult(try, fmt.Errorf("unexpected trailing data after JSON value"), fn, true)
}
val, err := decode(raw)
Expand Down
63 changes: 63 additions & 0 deletions lib/serial/serial_test.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package serial_test

import (
"strings"
"testing"
"time"

Expand Down Expand Up @@ -427,3 +428,65 @@ func TestLoadModule_Serial(t *testing.T) {
})
}
}

// JSON document boundaries preserve the throwing and try_loads error contracts.
func TestLoadsSingleDocument(t *testing.T) {
members, err := itn.NewAssertLoader(serial.ModuleName, serial.LoadModule)(&starlark.Thread{}, serial.ModuleName)
if err != nil {
t.Fatal(err)
}
for _, input := range []string{
`1]`, `1}`, `null]`, `null }`, `1] false`, " \t1 \r\n]", `{"a":1}]`, `{"a":1}}`, `[1]]`, `[1]}`,
`1 2`, `{} {}`, `1 junk`, `1 [`,
"1\v", "1\f", "1\u00a0", "1\ufeff",
"1 [" + strings.Repeat("0,", 8192) + "0]",
} {
name := input
if len(name) > 80 {
name = "large second document"
}
t.Run(name, func(t *testing.T) {
for _, name := range []string{"loads", "try_loads"} {
t.Run(name, func(t *testing.T) {
got, err := starlark.Call(&starlark.Thread{}, members[name], starlark.Tuple{starlark.String(input)}, nil)
if name == "loads" {
if err == nil || !strings.Contains(err.Error(), "serial.loads: unexpected trailing data") {
t.Fatalf("loads(%q) = %v, %v; want trailing-data error", input, got, err)
}
return
}
if err != nil {
t.Fatalf("try_loads(%q): %v", input, err)
}
pair, ok := got.(starlark.Tuple)
if !ok || len(pair) != 2 || pair[0] != starlark.None || !strings.Contains(pair[1].String(), "unexpected trailing data") {
t.Fatalf("try_loads(%q) = %v; want (None, trailing-data message)", input, got)
}
})
}
})
}
for _, input := range []string{"123456789012345678901234567890", " \t\r\n123456789012345678901234567890 \t\r\n"} {
t.Run(input, func(t *testing.T) {
for _, name := range []string{"loads", "try_loads"} {
got, err := starlark.Call(&starlark.Thread{}, members[name], starlark.Tuple{starlark.String(input)}, nil)
if err != nil {
t.Fatal(err)
}
if name == "try_loads" {
pair, ok := got.(starlark.Tuple)
if !ok || len(pair) != 2 {
t.Fatalf("try_loads(%q) = %v; want result/error tuple", input, got)
}
if pair[1] != starlark.None {
t.Fatalf("try_loads(%q) error = %v", input, pair[1])
}
got = pair[0]
}
if got.String() != "123456789012345678901234567890" || got.Type() != "int" {
t.Fatalf("%s(%q) = %v; want exact integer", name, input, got)
}
}
})
}
}
Loading