From 1eeaa971c5afe9b2dea20641be58667007afd073 Mon Sep 17 00:00:00 2001 From: Kevin Tang <73975146+vt128@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:23:40 +0800 Subject: [PATCH 1/3] [fix] preserve snapshot paths across container wrappers (LET-40) Keep one formatting path across SharedDict, Struct and Set values so container cycles terminate without starting a fresh String traversal. Preserve external synchronization for nested values and release set iterators before host callbacks. Add bounded cycle and compatibility tests. --- USAGE.md | 8 + dataconv/share.go | 87 +++++++++- dataconv/share_test.go | 370 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 458 insertions(+), 7 deletions(-) diff --git a/USAGE.md b/USAGE.md index 828348b..17aca0d 100644 --- a/USAGE.md +++ b/USAGE.md @@ -179,6 +179,14 @@ outside callbacks from operations that hold it, such as key `Hash`/`Equal` or value `Freeze`. This Go method does not add a Starlark builtin or change default printing. See [shared_dict methods and callback rules](lib/goidiomatic/README.md#shared_dict). +Cycle detection follows list/dict/set references through tuples, nested +`SharedDict` values and structs with unique field names. Nested `SharedDict` +values are not locked; synchronize them separately. Sets capture their elements +before formatting callbacks run. Custom values and Go-created structs with +duplicate field names keep their own `String` behavior. Go-created cycles +consisting only of tuples are not detected. There is no general depth or output +limit. + Snapshot formatting copies entries into temporary memory. Larger or nested dictionaries can use more bytes than direct formatting; avoiding intermediate strings and repeated path allocations can offset some work. The net cost depends diff --git a/dataconv/share.go b/dataconv/share.go index 3da011a..f522ae5 100644 --- a/dataconv/share.go +++ b/dataconv/share.go @@ -10,6 +10,7 @@ import ( itn "github.com/1set/starlet/internal" stdjson "go.starlark.net/lib/json" "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" "go.starlark.net/syntax" ) @@ -83,6 +84,10 @@ func (s *SharedDict) String() string { // SnapshotString formats a shallow snapshot of the mapping in insertion order. // It releases the mapping lock before invoking formatting callbacks. Mutable // nested values remain shared and require their own synchronization. +// Nested SharedDict values are not locked by this method. Cycle detection spans +// native lists/dictionaries/sets, SharedDict and structs with unique field names, +// following references via tuples. Other values retain their own String behavior. +// Set elements are captured before formatting callbacks run; the values remain shared. // Call it without holding this dictionary's mutex, and outside callbacks from // operations that hold it (for example a key's Hash/Equal or a value's Freeze). func (s *SharedDict) SnapshotString() string { @@ -94,7 +99,7 @@ func (s *SharedDict) SnapshotString() string { } s.RUnlock() - f := sharedDictFormatter{root: dict, items: items} + f := sharedDictFormatter{shared: s, name: name, root: dict, items: items} f.WriteString(name) f.WriteByte('(') if dict != nil { @@ -109,8 +114,10 @@ func (s *SharedDict) SnapshotString() string { // keys or invoking host callbacks under the mapping lock. type sharedDictFormatter struct { strings.Builder - root *starlark.Dict - items []starlark.Tuple + shared *SharedDict + name string + root *starlark.Dict + items []starlark.Tuple } func (f *sharedDictFormatter) write(value starlark.Value, path []starlark.Value) { @@ -143,6 +150,77 @@ func (f *sharedDictFormatter) write(value starlark.Value, path []starlark.Value) f.WriteByte(',') } f.WriteByte(')') + case *SharedDict: + dict, name := f.root, f.name + if value != f.shared { + dict, name = value.dict, value.getTypeName() + } + f.WriteString(name) + f.WriteByte('(') + if dict != nil { + f.write(dict, path) + } + f.WriteByte(')') + case *starlark.Set: + f.WriteString("set([") + if sharedDictPathContains(path, value) { + f.WriteString("...") + } else { + elements := make([]starlark.Value, 0, value.Len()) + iter := value.Iterate() + var element starlark.Value + for iter.Next(&element) { + elements = append(elements, element) + } + iter.Done() + path = append(path, value) + for i, element := range elements { + if i > 0 { + f.WriteString(", ") + } + f.write(element, path) + } + } + f.WriteString("])") + case *starlarkstruct.Struct: + names := value.AttrNames() + for i := 1; i < len(names); i++ { + if names[i] == names[i-1] { + // Attr returns only the first entry of duplicate Go fields. + f.WriteString(value.String()) + return + } + } + if sharedDictPathContains(path, value) { + f.WriteString("...") + return + } + path = append(path, value) + constructor := value.Constructor() + if name, ok := constructor.(starlark.String); ok { + f.WriteString(name.GoString()) + } else { + if constructor == nil { + f.WriteString(value.String()) + return + } + f.write(constructor, path) + } + f.WriteByte('(') + for i, name := range names { + if i > 0 { + f.WriteString(", ") + } + f.WriteString(name) + f.WriteString(" = ") + field, _ := value.Attr(name) + if field == nil { + f.WriteString(value.String()) + return + } + f.write(field, path) + } + f.WriteByte(')') case *starlark.Dict: f.WriteByte('{') if sharedDictPathContains(path, value) { @@ -153,11 +231,12 @@ func (f *sharedDictFormatter) write(value starlark.Value, path []starlark.Value) items = value.Items() } valuePath := append(path, value) + keyPath := path[:len(path):len(path)] for i, item := range items { if i > 0 { f.WriteString(", ") } - f.write(item[0], path) + f.write(item[0], keyPath) f.WriteString(": ") f.write(item[1], valuePath) } diff --git a/dataconv/share_test.go b/dataconv/share_test.go index b8b32a5..d4d9bb8 100644 --- a/dataconv/share_test.go +++ b/dataconv/share_test.go @@ -4,9 +4,13 @@ package dataconv // concurrent access, formatting snapshots, and JSON conversion. import ( + "context" "fmt" "math" + "os" + "os/exec" "reflect" + "runtime/debug" "strings" "sync" "testing" @@ -280,6 +284,334 @@ func TestSharedDict_SnapshotStringConcurrent(t *testing.T) { workers.Wait() } +func TestSharedDict_SnapshotStringFormattingGraph(t *testing.T) { + const childEnv = "STARLET_TEST_SNAPSHOT_GRAPH_CHILD" + if os.Getenv(childEnv) != "1" { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, os.Args[0], "-test.run=^TestSharedDict_SnapshotStringFormattingGraph$", "-test.v") + cmd.Env = append(os.Environ(), childEnv+"=1") + output, err := cmd.CombinedOutput() + if err != nil { + t.Fatalf("bounded formatting child: %v (timeout: %v)\n%s", err, ctx.Err(), output) + } + // Exercise the same guarded cases here so Go 1.19 records coverage. + } else { + debug.SetMaxStack(128 << 10) + } + put := func(t *testing.T, dict starlark.HasSetKey, key string, value starlark.Value) { + t.Helper() + if err := dict.SetKey(starlark.String(key), value); err != nil { + t.Fatal(err) + } + } + check := func(t *testing.T, dict *SharedDict, want string) { + t.Helper() + if got := dict.SnapshotString(); got != want { + t.Fatalf("SnapshotString() = %q; want %q", got, want) + } + } + t.Run("shared_dict", func(t *testing.T) { + for _, test := range []struct{ name, want string }{ + {"list", `root({"items": [root({...})]})`}, + {"dict", `root({"items": {"back": root({...})}})`}, + {"tuple", `root({"items": (root({...}),)})`}, + } { + t.Run(test.name, func(t *testing.T) { + root := NewNamedSharedDict("root") + var value starlark.Value + switch test.name { + case "list": + value = starlark.NewList([]starlark.Value{root}) + case "dict": + dict := starlark.NewDict(1) + put(t, dict, "back", root) + value = dict + case "tuple": + value = starlark.Tuple{root} + } + put(t, root, "items", value) + check(t, root, test.want) + }) + } + t.Run("mutual", func(t *testing.T) { + a, b, c := NewNamedSharedDict("a"), NewNamedSharedDict("b"), NewNamedSharedDict("c") + back := starlark.NewDict(1) + put(t, back, "back", a) + put(t, a, "items", starlark.NewList([]starlark.Value{b})) + put(t, b, "items", starlark.Tuple{c}) + put(t, c, "items", back) + check(t, a, `a({"items": [b({"items": (c({"items": {"back": a({...})}}),)})]})`) + }) + t.Run("aliases_and_empty", func(t *testing.T) { + root, child := NewNamedSharedDict("root"), NewNamedSharedDict("child") + put(t, child, "n", starlark.MakeInt(1)) + put(t, root, "items", starlark.NewList([]starlark.Value{child, child, NewNamedSharedDict("empty"), &SharedDict{name: "zero"}})) + check(t, root, `root({"items": [child({"n": 1}), child({"n": 1}), empty({}), zero()]})`) + }) + }) + t.Run("root_snapshot", func(t *testing.T) { + root := NewNamedSharedDict("root") + locked, calls := false, 0 + defer func() { + if locked { + root.Unlock() + } + }() + callback := &sharedDictFormatValue{stringFunc: func() string { + calls++ + put(t, root, "later", starlark.True) + root.Lock() + locked = true + return "changed" + }} + put(t, root, "change", callback) + put(t, root, "back", starlark.NewList([]starlark.Value{root})) + put(t, root, "native", root.dict) + check(t, root, `root({"change": changed, "back": [root({...})], "native": {...}})`) + if calls != 1 { + t.Fatalf("format callback calls = %d; want 1", calls) + } + }) + t.Run("external_nested_lock", func(t *testing.T) { + root, child := NewNamedSharedDict("root"), NewNamedSharedDict("child") + put(t, root, "next", starlark.NewList([]starlark.Value{child})) + put(t, child, "next", starlark.NewList([]starlark.Value{root})) + child.Lock() + defer child.Unlock() + check(t, root, `root({"next": [child({"next": [root({...})]})]})`) + }) + t.Run("callbacks", func(t *testing.T) { + for _, mode := range []string{"String", "Type"} { + t.Run(mode, func(t *testing.T) { + root, child := NewNamedSharedDict("root"), NewNamedSharedDict("child") + calls := 0 + callback := func() string { + calls++ + for _, dict := range []*SharedDict{root, child} { + if !dict.TryLock() { + t.Fatal("format callback ran with a mapping lock held") + } + dict.Unlock() + if dict.Len() != 1 { + t.Fatal("unexpected mapping length") + } + } + put(t, root, "later", starlark.True) + return "host" + } + host := &sharedDictFormatValue{stringFunc: func() string { return "receiver" }} + var value starlark.Value = host + wantValue := "host" + if mode == "String" { + host.stringFunc = callback + } else { + host.typeFunc = callback + value = starlark.NewBuiltin("method", func(*starlark.Thread, *starlark.Builtin, starlark.Tuple, []starlark.Tuple) (starlark.Value, error) { + return starlark.None, nil + }).BindReceiver(host) + wantValue = "" + } + put(t, child, "value", value) + put(t, root, "items", starlark.NewList([]starlark.Value{child})) + check(t, root, `root({"items": [child({"value": `+wantValue+`})]})`) + if calls != 1 || root.Len() != 2 { + t.Fatalf("callback calls=%d, root length=%d; want 1 and 2", calls, root.Len()) + } + }) + } + }) + t.Run("concurrent_root_writer", func(t *testing.T) { + root, child := NewNamedSharedDict("root"), NewNamedSharedDict("child") + put(t, child, "n", starlark.MakeInt(1)) + put(t, root, "items", starlark.NewList([]starlark.Value{child})) + put(t, root, "counter", starlark.MakeInt(0)) + var workers sync.WaitGroup + workers.Add(2) + go func() { + defer workers.Done() + for i := 0; i < 128; i++ { + if err := root.SetKey(starlark.String("counter"), starlark.MakeInt(i)); err != nil { + t.Error(err) + return + } + } + }() + go func() { + defer workers.Done() + for i := 0; i < 128; i++ { + if got := root.SnapshotString(); !strings.HasPrefix(got, `root({"items": [child({"n": 1})], "counter": `) || !strings.HasSuffix(got, "})") { + t.Errorf("malformed snapshot: %q", got) + return + } + } + }() + workers.Wait() + }) + t.Run("opposite_snapshots", func(t *testing.T) { + a, b := NewNamedSharedDict("a"), NewNamedSharedDict("b") + put(t, a, "next", starlark.NewList([]starlark.Value{b})) + put(t, b, "next", starlark.NewList([]starlark.Value{a})) + var workers sync.WaitGroup + for _, test := range []struct { + dict *SharedDict + want string + }{{a, `a({"next": [b({"next": [a({...})]})]})`}, {b, `b({"next": [a({"next": [b({...})]})]})`}} { + workers.Add(1) + go func(dict *SharedDict, want string) { + defer workers.Done() + for i := 0; i < 128; i++ { + if got := dict.SnapshotString(); got != want { + t.Errorf("SnapshotString() = %q; want %q", got, want) + return + } + } + }(test.dict, test.want) + } + workers.Wait() + }) + t.Run("struct", func(t *testing.T) { + t.Run("list_cycle", func(t *testing.T) { + root := NewNamedSharedDict("root") + list := starlark.NewList(nil) + record := starlarkstruct.FromStringDict(starlark.String("record"), starlark.StringDict{"items": list}) + if err := list.Append(record); err != nil { + t.Fatal(err) + } + put(t, root, "items", list) + check(t, root, `root({"items": [record(items = [...])]})`) + }) + t.Run("root_wrapper", func(t *testing.T) { + root := NewNamedSharedDict("root") + put(t, root, "items", starlarkstruct.FromStringDict(starlark.String("record"), starlark.StringDict{"root": root})) + check(t, root, `root({"items": record(root = root({...}))})`) + }) + t.Run("constructor_cycle", func(t *testing.T) { + root := NewNamedSharedDict("root") + constructor := starlark.NewList(nil) + record := starlarkstruct.FromStringDict(constructor, starlark.StringDict{"n": starlark.MakeInt(1)}) + if err := constructor.Append(record); err != nil { + t.Fatal(err) + } + put(t, root, "items", record) + check(t, root, `root({"items": [...](n = 1)})`) + }) + t.Run("oracle", func(t *testing.T) { + for _, constructor := range []starlark.Value{starlarkstruct.Default, starlark.String("record\"\n"), starlark.MakeInt(7), starlark.Tuple{starlark.String("kind")}} { + root := NewSharedDict() + record := starlarkstruct.FromStringDict(constructor, starlark.StringDict{"z": starlark.None, "a": starlark.Tuple{starlark.True}}) + put(t, root, "items", starlark.NewList([]starlark.Value{record, record})) + check(t, root, "shared_dict("+root.dict.String()+")") + } + }) + t.Run("duplicate_fields_oracle", func(t *testing.T) { + root := NewSharedDict() + record := starlarkstruct.FromKeywords(starlark.String("record"), []starlark.Tuple{{starlark.String("a"), starlark.MakeInt(1)}, {starlark.String("a"), starlark.MakeInt(2)}}) + put(t, root, "items", record) + check(t, root, "shared_dict("+root.dict.String()+")") + }) + for _, tupleKey := range []bool{false, true} { + t.Run(fmt.Sprintf("key_path/tuple=%v", tupleKey), func(t *testing.T) { + root := NewNamedSharedDict("root") + record := starlarkstruct.FromStringDict(starlark.String("key"), starlark.StringDict{"n": starlark.MakeInt(1)}) + var key starlark.Value = record + wantKey := "key(n = 1)" + if tupleKey { + key = starlark.Tuple{record} + wantKey = "(key(n = 1),)" + } + bottom := starlark.NewDict(2) + if err := bottom.SetKey(key, record); err != nil { + t.Fatal(err) + } + put(t, bottom, "self", bottom) + put(t, root, "items", starlark.NewList([]starlark.Value{starlark.NewList([]starlark.Value{bottom})})) + check(t, root, `root({"items": [[{`+wantKey+`: key(n = 1), "self": {...}}]]})`) + }) + } + for _, test := range []struct { + name string + value starlark.Value + }{{"nil_field", starlarkstruct.FromStringDict(starlarkstruct.Default, starlark.StringDict{"value": nil})}, {"zero_value", new(starlarkstruct.Struct)}} { + t.Run(test.name, func(t *testing.T) { + root := NewSharedDict() + put(t, root, "items", test.value) + defer func() { + if recover() == nil { + t.Fatal("invalid Go struct no longer preserves upstream String panic") + } + }() + root.SnapshotString() + }) + } + }) + t.Run("set", func(t *testing.T) { + t.Run("constructor_cycle", func(t *testing.T) { + root := NewNamedSharedDict("root") + set, constructor := starlark.NewSet(1), starlark.NewList(nil) + record := starlarkstruct.FromStringDict(constructor, nil) + if err := set.Insert(record); err != nil { + t.Fatal(err) + } + if err := constructor.Append(set); err != nil { + t.Fatal(err) + } + put(t, root, "items", set) + check(t, root, `root({"items": set([[set([...])]()])})`) + }) + t.Run("oracle", func(t *testing.T) { + for _, values := range [][]starlark.Value{nil, {starlark.None, starlark.True, starlark.Tuple{starlark.MakeInt(1)}}} { + root, set := NewSharedDict(), starlark.NewSet(len(values)) + for _, value := range values { + if err := set.Insert(value); err != nil { + t.Fatal(err) + } + } + put(t, root, "items", set) + check(t, root, "shared_dict("+root.dict.String()+")") + } + }) + t.Run("callback_insert", func(t *testing.T) { + root, set := NewSharedDict(), starlark.NewSet(1) + calls := 0 + value := &sharedDictFormatValue{stringFunc: func() string { + calls++ + if err := set.Insert(starlark.String("later")); err != nil { + t.Fatal(err) + } + return "first" + }} + if err := set.Insert(value); err != nil { + t.Fatal(err) + } + put(t, root, "items", set) + check(t, root, `shared_dict({"items": set([first])})`) + if calls != 1 || set.Len() != 2 { + t.Fatalf("format calls=%d, set length=%d; want 1 and 2", calls, set.Len()) + } + }) + t.Run("callback_panic", func(t *testing.T) { + root, set := NewSharedDict(), starlark.NewSet(1) + value := &sharedDictFormatValue{stringFunc: func() string { panic("set format panic") }} + if err := set.Insert(value); err != nil { + t.Fatal(err) + } + put(t, root, "items", set) + func() { + defer func() { + if got := recover(); got != "set format panic" { + t.Fatalf("format panic = %v", got) + } + }() + root.SnapshotString() + }() + if err := set.Insert(starlark.True); err != nil { + t.Fatalf("set iterator remained active after formatting panic: %v", err) + } + }) + }) +} + type sharedDictFormatValue struct { stringFunc func() string hashFunc func() (uint32, error) @@ -420,6 +752,36 @@ func BenchmarkSharedDictReadPaths(b *testing.B) { if err := nested.SetKey(starlark.String("nested"), flat8.dict); err != nil { b.Fatal(err) } + nestedShared := NewSharedDict() + if err := nestedShared.SetKey(starlark.String("nested"), starlark.NewList([]starlark.Value{flat8})); err != nil { + b.Fatal(err) + } + repeatedShared := NewSharedDict() + if err := repeatedShared.SetKey(starlark.String("nested"), starlark.NewList([]starlark.Value{flat8, flat8})); err != nil { + b.Fatal(err) + } + record := NewSharedDict() + if err := record.SetKey(starlark.String("record"), starlarkstruct.FromStringDict(starlark.String("record"), starlark.StringDict{"entries": flat8.dict})); err != nil { + b.Fatal(err) + } + plainSet := NewSharedDict() + setValues, structValues := starlark.NewSet(8), starlark.NewSet(8) + for i := 0; i < 8; i++ { + if err := setValues.Insert(starlark.MakeInt(i)); err != nil { + b.Fatal(err) + } + key := starlarkstruct.FromStringDict(starlark.String("key"), starlark.StringDict{"id": starlark.MakeInt(i)}) + if err := structValues.Insert(key); err != nil { + b.Fatal(err) + } + } + if err := plainSet.SetKey(starlark.String("set"), setValues); err != nil { + b.Fatal(err) + } + structSet := NewSharedDict() + if err := structSet.SetKey(starlark.String("set"), structValues); err != nil { + b.Fatal(err) + } cycle := NewSharedDict() if err := cycle.SetKey(starlark.String("self"), cycle.dict); err != nil { b.Fatal(err) @@ -430,11 +792,13 @@ func BenchmarkSharedDictReadPaths(b *testing.B) { }{ {"Flat0", makeFlat(0)}, {"Flat1", makeFlat(1)}, {"Flat8", flat8}, {"Flat128", makeFlat(128)}, {"Nested8", nested}, {"Cycle", cycle}, + {"NestedShared8", nestedShared}, {"RepeatedShared8", repeatedShared}, {"Struct8", record}, + {"Set8", plainSet}, {"SetStruct8", structSet}, } { b.Run("Snapshot/"+test.name, func(b *testing.B) { - format := test.dict.String - if snapshot, ok := interface{}(test.dict).(interface{ SnapshotString() string }); ok { - format = snapshot.SnapshotString + format := test.dict.SnapshotString + if got, want := format(), test.dict.String(); got != want { + b.Fatalf("SnapshotString() = %q; want %q", got, want) } b.ReportAllocs() b.ResetTimer() From 015686d733a8de978f1f8ac5ec28f87cccc22c25 Mon Sep 17 00:00:00 2001 From: Kevin Tang <73975146+vt128@users.noreply.github.com> Date: Mon, 9 Nov 2026 15:45:13 +0800 Subject: [PATCH 2/3] [perf] reduce snapshot formatter stack and temporary storage (LET-40) Move Set and Struct traversal outside the common formatter frame. Capture small sets and ancestor paths in bounded local buffers, and skip the formatter for empty root snapshots. Preserve cycle and callback behavior with spill and deep-ancestor tests. --- dataconv/share.go | 153 ++++++++++++++++++++++++----------------- dataconv/share_test.go | 79 +++++++++++++++++---- 2 files changed, 155 insertions(+), 77 deletions(-) diff --git a/dataconv/share.go b/dataconv/share.go index f522ae5..2dffe1d 100644 --- a/dataconv/share.go +++ b/dataconv/share.go @@ -99,12 +99,20 @@ func (s *SharedDict) SnapshotString() string { } s.RUnlock() - f := sharedDictFormatter{shared: s, name: name, root: dict, items: items} + if dict == nil { + return name + "()" + } + if len(items) == 0 { + return name + "({})" + } + var f sharedDictFormatter + f.shared = s + f.name = name + f.root = dict + f.items = items f.WriteString(name) f.WriteByte('(') - if dict != nil { - f.write(dict, nil) - } + f.write(dict, nil) f.WriteByte(')') return f.String() } @@ -162,65 +170,9 @@ func (f *sharedDictFormatter) write(value starlark.Value, path []starlark.Value) } f.WriteByte(')') case *starlark.Set: - f.WriteString("set([") - if sharedDictPathContains(path, value) { - f.WriteString("...") - } else { - elements := make([]starlark.Value, 0, value.Len()) - iter := value.Iterate() - var element starlark.Value - for iter.Next(&element) { - elements = append(elements, element) - } - iter.Done() - path = append(path, value) - for i, element := range elements { - if i > 0 { - f.WriteString(", ") - } - f.write(element, path) - } - } - f.WriteString("])") + f.writeSet(value, path) case *starlarkstruct.Struct: - names := value.AttrNames() - for i := 1; i < len(names); i++ { - if names[i] == names[i-1] { - // Attr returns only the first entry of duplicate Go fields. - f.WriteString(value.String()) - return - } - } - if sharedDictPathContains(path, value) { - f.WriteString("...") - return - } - path = append(path, value) - constructor := value.Constructor() - if name, ok := constructor.(starlark.String); ok { - f.WriteString(name.GoString()) - } else { - if constructor == nil { - f.WriteString(value.String()) - return - } - f.write(constructor, path) - } - f.WriteByte('(') - for i, name := range names { - if i > 0 { - f.WriteString(", ") - } - f.WriteString(name) - f.WriteString(" = ") - field, _ := value.Attr(name) - if field == nil { - f.WriteString(value.String()) - return - } - f.write(field, path) - } - f.WriteByte(')') + f.writeStruct(value, path) case *starlark.Dict: f.WriteByte('{') if sharedDictPathContains(path, value) { @@ -247,6 +199,83 @@ func (f *sharedDictFormatter) write(value starlark.Value, path []starlark.Value) } } +func (f *sharedDictFormatter) writeSet(value *starlark.Set, path []starlark.Value) { + f.WriteString("set([") + if sharedDictPathContains(path, value) { + f.WriteString("...") + } else { + var smallElements [8]starlark.Value + elements := smallElements[:0] + if size := value.Len(); size > len(smallElements) { + elements = make([]starlark.Value, 0, size) + } + iter := value.Iterate() + var element starlark.Value + for iter.Next(&element) { + elements = append(elements, element) + } + iter.Done() + var smallPath [8]starlark.Value + size := len(path) + 1 + nextPath := smallPath[:0] + if size >= len(smallPath) { + nextPath = make([]starlark.Value, size, size+1) + } else { + nextPath = smallPath[:size] + } + copy(nextPath, path) + nextPath[size-1] = value + for i, element := range elements { + if i > 0 { + f.WriteString(", ") + } + f.write(element, nextPath) + } + } + f.WriteString("])") +} + +func (f *sharedDictFormatter) writeStruct(value *starlarkstruct.Struct, path []starlark.Value) { + names := value.AttrNames() + for i := 1; i < len(names); i++ { + if names[i] == names[i-1] { + // Attr returns only the first entry of duplicate Go fields. + f.WriteString(value.String()) + return + } + } + if sharedDictPathContains(path, value) { + f.WriteString("...") + return + } + path = append(path, value) + constructor := value.Constructor() + if name, ok := constructor.(starlark.String); ok { + f.WriteString(name.GoString()) + } else { + if constructor == nil { + f.WriteString(value.String()) + return + } + f.write(constructor, path) + } + f.WriteByte('(') + for i, name := range names { + if i > 0 { + f.WriteString(", ") + } + f.WriteString(name) + f.WriteString(" = ") + field, _ := value.Attr(name) + if field == nil { + f.WriteString(value.String()) + return + } + f.write(field, path) + } + f.WriteByte(')') +} + func sharedDictPathContains(path []starlark.Value, value starlark.Value) bool { for _, ancestor := range path { if ancestor == value { diff --git a/dataconv/share_test.go b/dataconv/share_test.go index d4d9bb8..2bac4e2 100644 --- a/dataconv/share_test.go +++ b/dataconv/share_test.go @@ -121,6 +121,19 @@ func TestSharedDict_SnapshotString(t *testing.T) { if got := (&SharedDict{}).String(); got != "shared_dict()" { t.Fatalf("String() = %q; want shared_dict()", got) } + if got := (&SharedDict{}).SnapshotString(); got != "shared_dict()" { + t.Fatalf("SnapshotString() = %q; want shared_dict()", got) + } + }) + t.Run("empty_root", func(t *testing.T) { + for _, test := range []struct { + dict *SharedDict + want string + }{{NewSharedDict(), "shared_dict({})"}, {NewNamedSharedDict("named"), "named({})"}, {&SharedDict{name: "zero"}, "zero()"}} { + if got := test.dict.SnapshotString(); got != test.want { + t.Fatalf("SnapshotString() = %q; want %q", got, test.want) + } + } }) } @@ -570,6 +583,43 @@ func TestSharedDict_SnapshotStringFormattingGraph(t *testing.T) { put(t, root, "items", set) check(t, root, "shared_dict("+root.dict.String()+")") } + for _, size := range []int{8, 9, 32} { + t.Run(fmt.Sprintf("size=%d", size), func(t *testing.T) { + root, set := NewSharedDict(), starlark.NewSet(size) + for i := 0; i < size; i++ { + if err := set.Insert(starlark.MakeInt(i)); err != nil { + t.Fatal(err) + } + } + put(t, root, "items", set) + check(t, root, "shared_dict("+root.dict.String()+")") + }) + } + }) + t.Run("deep_path", func(t *testing.T) { + root, set := NewSharedDict(), starlark.NewSet(1) + if err := set.Insert(starlark.MakeInt(1)); err != nil { + t.Fatal(err) + } + var value starlark.Value = set + for i := 0; i < 9; i++ { + value = starlark.NewList([]starlark.Value{value}) + } + put(t, root, "items", value) + check(t, root, "shared_dict("+root.dict.String()+")") + }) + t.Run("deep_ancestor_cycle", func(t *testing.T) { + root, set := NewSharedDict(), starlark.NewSet(1) + var value starlark.Value = set + for i := 0; i < 9; i++ { + value = starlark.NewList([]starlark.Value{value}) + } + record := starlarkstruct.FromStringDict(value, nil) + if err := set.Insert(record); err != nil { + t.Fatal(err) + } + put(t, root, "items", value) + check(t, root, `shared_dict({"items": `+strings.Repeat("[", 9)+`set([[...]()])`+strings.Repeat("]", 9)+`})`) }) t.Run("callback_insert", func(t *testing.T) { root, set := NewSharedDict(), starlark.NewSet(1) @@ -764,23 +814,21 @@ func BenchmarkSharedDictReadPaths(b *testing.B) { if err := record.SetKey(starlark.String("record"), starlarkstruct.FromStringDict(starlark.String("record"), starlark.StringDict{"entries": flat8.dict})); err != nil { b.Fatal(err) } - plainSet := NewSharedDict() - setValues, structValues := starlark.NewSet(8), starlark.NewSet(8) - for i := 0; i < 8; i++ { - if err := setValues.Insert(starlark.MakeInt(i)); err != nil { - b.Fatal(err) + makeSet := func(n int, records bool) *SharedDict { + dict, set := NewSharedDict(), starlark.NewSet(n) + for i := 0; i < n; i++ { + var value starlark.Value = starlark.MakeInt(i) + if records { + value = starlarkstruct.FromStringDict(starlark.String("key"), starlark.StringDict{"id": value}) + } + if err := set.Insert(value); err != nil { + b.Fatal(err) + } } - key := starlarkstruct.FromStringDict(starlark.String("key"), starlark.StringDict{"id": starlark.MakeInt(i)}) - if err := structValues.Insert(key); err != nil { + if err := dict.SetKey(starlark.String("set"), set); err != nil { b.Fatal(err) } - } - if err := plainSet.SetKey(starlark.String("set"), setValues); err != nil { - b.Fatal(err) - } - structSet := NewSharedDict() - if err := structSet.SetKey(starlark.String("set"), structValues); err != nil { - b.Fatal(err) + return dict } cycle := NewSharedDict() if err := cycle.SetKey(starlark.String("self"), cycle.dict); err != nil { @@ -793,7 +841,8 @@ func BenchmarkSharedDictReadPaths(b *testing.B) { {"Flat0", makeFlat(0)}, {"Flat1", makeFlat(1)}, {"Flat8", flat8}, {"Flat128", makeFlat(128)}, {"Nested8", nested}, {"Cycle", cycle}, {"NestedShared8", nestedShared}, {"RepeatedShared8", repeatedShared}, {"Struct8", record}, - {"Set8", plainSet}, {"SetStruct8", structSet}, + {"Set8", makeSet(8, false)}, {"SetStruct8", makeSet(8, true)}, + {"Set32", makeSet(32, false)}, {"SetStruct32", makeSet(32, true)}, } { b.Run("Snapshot/"+test.name, func(b *testing.B) { format := test.dict.SnapshotString From 9200c15e265d622722759fe65d753e35c2fd7503 Mon Sep 17 00:00:00 2001 From: Kevin Tang <73975146+vt128@users.noreply.github.com> Date: Mon, 9 Nov 2026 17:39:23 +0800 Subject: [PATCH 3/3] docs: clarify on-demand diagnostic snapshot costs --- USAGE.md | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/USAGE.md b/USAGE.md index 17aca0d..b0f9c1f 100644 --- a/USAGE.md +++ b/USAGE.md @@ -171,9 +171,10 @@ Own or synchronize mutable nested values separately. Default Starlark formatting/printing and the Go `String`/`ToJSON` methods still require external synchronization with mapping writers. -From Go, use `text := shared.SnapshotString()` to format a snapshot of the -mapping's entries without racing with mapping updates. Formatting callbacks run -after the mapping lock is released; mutable nested values remain shared. +For on-demand debugging and diagnostics in Go, use +`text := shared.SnapshotString()` to capture and format the mapping's entries +without racing with mapping updates. Formatting callbacks run after the mapping +lock is released; mutable nested values remain shared. Call `SnapshotString` without already holding that dictionary's mutex, and outside callbacks from operations that hold it, such as key `Hash`/`Equal` or value `Freeze`. This Go method does not add a Starlark builtin or change default @@ -187,10 +188,12 @@ duplicate field names keep their own `String` behavior. Go-created cycles consisting only of tuples are not detected. There is no general depth or output limit. -Snapshot formatting copies entries into temporary memory. Larger or nested -dictionaries can use more bytes than direct formatting; avoiding intermediate -strings and repeated path allocations can offset some work. The net cost depends -on the values, Go version and machine. Run the included comparisons with: +Snapshot formatting copies entries and tracks container paths, adding temporary +memory and work. Large sets and nested containers can take longer and use more +bytes than direct formatting. Call it only when diagnostic output is needed; +avoid high-frequency business loops. Normal execution and default printing do +not call it automatically. Costs depend on the values, Go version and machine. +Run the included comparisons with: ```bash go test ./dataconv -run '^$' -bench '^BenchmarkSharedDictReadPaths$' -benchmem