From 7a7f98766887f06a58e936e6cf4ad7c65c471085 Mon Sep 17 00:00:00 2001 From: Khasbulat <92339556+Yakwilik@users.noreply.github.com> Date: Mon, 26 Oct 2026 04:32:16 +0300 Subject: [PATCH 1/5] fix: reuse verified dependency snapshots and source inventories --- .github/workflows/tests.yml | 22 +++ .spec/ARCHITECTURE.md | 4 +- .spec/config/dependency.md | 16 +- internal/adapters/gitmodules/bsr_test.go | 6 +- internal/adapters/gitmodules/cache.go | 9 +- internal/adapters/gitmodules/cache_run.go | 99 +++++++++++++ .../adapters/gitmodules/cache_run_test.go | 128 ++++++++++++++++ .../adapters/gitmodules/cache_verification.go | 44 +++--- internal/adapters/gitmodules/checkout_test.go | 2 +- .../adapters/gitmodules/diagnostics_test.go | 2 +- internal/adapters/gitmodules/download.go | 35 ++++- internal/adapters/gitmodules/migration.go | 2 +- internal/adapters/gitmodules/migration_bsr.go | 4 +- .../adapters/gitmodules/object_cache_test.go | 2 +- .../gitmodules/root_selection_test.go | 2 + internal/adapters/gitmodules/snapshot.go | 99 ++++++------- .../adapters/gitmodules/snapshot_checkout.go | 41 ++++-- .../adapters/gitmodules/snapshot_reuse.go | 107 ++++++++++++++ .../gitmodules/snapshot_reuse_test.go | 55 +++++++ .../adapters/gitmodules/source_binding.go | 4 + internal/adapters/gitsnapshot/destination.go | 77 +++++++++- .../adapters/gitsnapshot/destination_test.go | 60 ++++++++ internal/adapters/go_git/snapshot.go | 2 +- internal/core/core.go | 8 +- internal/core/generate.go | 12 +- internal/core/generate_inputs.go | 40 +++++ internal/core/generate_inputs_test.go | 86 +++++++++++ internal/generation/descriptor_set.go | 11 +- internal/generation/frozen.go | 6 +- internal/generation/module.go | 33 +++-- internal/generation/packages.go | 10 +- internal/generation/selection.go | 12 +- internal/generation/source_index_test.go | 82 +++++++++++ internal/modules/collisions.go | 22 ++- internal/modules/effective_graph.go | 14 +- internal/modules/frozen.go | 4 +- internal/modules/locked_sources.go | 4 +- internal/modules/operations_test.go | 3 + internal/modules/source_index.go | 125 ++++++++++++++++ internal/modules/source_index_test.go | 137 ++++++++++++++++++ internal/modules/sources.go | 74 ++++++---- internal/modules/tidy_input_verification.go | 63 +++++++- internal/modules/versionless_update_test.go | 2 + internal/sourceview/immutable_walk_test.go | 134 +++++++++++++++++ internal/sourceview/view.go | 41 +++++- 45 files changed, 1548 insertions(+), 197 deletions(-) create mode 100644 internal/adapters/gitmodules/cache_run.go create mode 100644 internal/adapters/gitmodules/cache_run_test.go create mode 100644 internal/adapters/gitmodules/snapshot_reuse.go create mode 100644 internal/adapters/gitmodules/snapshot_reuse_test.go create mode 100644 internal/adapters/gitsnapshot/destination_test.go create mode 100644 internal/core/generate_inputs.go create mode 100644 internal/core/generate_inputs_test.go create mode 100644 internal/generation/source_index_test.go create mode 100644 internal/modules/source_index.go create mode 100644 internal/modules/source_index_test.go create mode 100644 internal/sourceview/immutable_walk_test.go diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5816c70c..5fa9aa4e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,3 +58,25 @@ jobs: - name: Validate nightly release configuration run: goreleaser check + + windows-dependency-regressions: + runs-on: windows-latest + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Verify dependency snapshot and source traversal regressions + run: >- + go test -count=1 + ./internal/adapters/gitmodules + ./internal/adapters/gitsnapshot + ./internal/modules + ./internal/sourceview + ./internal/core + ./internal/generation + -run 'Test(InstalledSnapshot|SnapshotVerificationWait|SourceFilesDistinguishes|FetchRetainsSnapshotForOfflineInstall|DestinationValidatorRetains|SnapshotSourceIndex|LocalSourceIndex|ImmutableWalk|MutableWalk|PrepareGenerationUsesVerifiedInventory|PrepareGenerationWithoutInventoryKeepsLocalSourcesFresh|GenerationSelectsIndexedDependencyInputs)' diff --git a/.spec/ARCHITECTURE.md b/.spec/ARCHITECTURE.md index e23d5e39..afe87faf 100644 --- a/.spec/ARCHITECTURE.md +++ b/.spec/ARCHITECTURE.md @@ -87,11 +87,13 @@ mod vendor EnsureLockedSources may install dependencies. ReadManifest, ReadLock, LocalSources, and CachedSources do not download or rewrite project files. CachedSources assumes installation/hash verification has already occurred. +Verified installed modules can supply an immutable regular-file inventory through the optional SourceFiles cache capability. SourceRoots uses it for dependency discovery and nested boundaries without repeated filesystem traversal. Local roots and replacements continue to resolve current filesystem state. Generation computes collision checks and file ownership together and shares the result with managed options and descriptor output. + Generation, policy commands, and module operations share root/collision checks. WalkProtoFiles owns dependency-source traversal, including exclusion of hidden directories, vendored sources and nested module boundaries. Generator discovery and policy ancestor traversal remain separate: their inclusion and inheritance rules differ. Nested protobuf module roots are rebased by module_config from the nested manifest directory into the checkout directory. Import names remain relative to those roots; repository/module prefixes do not become part of an import. -The Git cache lives beneath the CLI-supplied EasyP directory in v1/git; internal/adapters/gitmodules/object_cache.go maintains reusable bare object repositories with OS locks. Cache.Cached reads installed metadata; Install supplies content verification before cached metadata is trusted. +The Git cache lives beneath the CLI-supplied EasyP directory in v1/git; internal/adapters/gitmodules/object_cache.go maintains reusable bare object repositories with OS locks. Cache.Cached reads installed metadata; Install supplies content verification before cached metadata is trusted. Successful verification and the inventory are reused within one cache handle, keyed by source, commit, content hash and selected roots. A new handle rechecks the snapshot; explicit VerifyCached bypasses the in-memory result. Revision resolution atomically retains completed snapshots so installation does not materialize them a second time. Local replacements use an ephemeral main-module overlay graph and never alter the shared lock. Tidy validates the overlay; get/update may edit explicit manifest requirements. Vendor copies the effective local graph without touching the lock. Explicit frozen operation verifies existing native manifests and locked graphs; automatic unknown-import-to-module discovery is intentionally not implemented. diff --git a/.spec/config/dependency.md b/.spec/config/dependency.md index 24205f26..977cfbca 100644 --- a/.spec/config/dependency.md +++ b/.spec/config/dependency.md @@ -60,7 +60,13 @@ See [Go major version suffixes](https://go.dev/ref/mod#major-version-suffixes) a The CLI resolves EASYPPATH once for a command that needs the cache (default $HOME/.easyp). gitmodules.Cache owns <EASYPPATH>/v1/git, source keys, temporary checkout names and installation paths. Installed snapshots live under its modules/<source-key>/<snapshot-key> layout, where the snapshot key hashes commit plus content hash; reusable bare object stores live under objects/<remote-key>, with OS locks for concurrent fetches. Source/commit bindings under sources/ identify these stores for read-only policy access. Application callers request cached module metadata and its physical directory; they do not assemble cache paths. -Cache.Install verifies locked contents, including cache hits. A newly installed snapshot is checked with the full h1: directory hash. On Darwin/Linux, later unchanged hits use a sidecar verification stamp keyed by the expected lock hash plus an inode/ctime metadata fingerprint; any metadata change invalidates the stamp and forces the full directory hash again. Content changes therefore remain detectable even when size and mtime are restored. Platforms without a strong change token keep the full-hash path. Stamp hits first check the current snapshot selection policy. Read-only validation never writes verification stamps. The stamp is an optimization of the per-user local cache, not an additional source of dependency identity. Cached objects support shallow pinned-commit fetches with an advertised-history fallback; neither path substitutes HEAD for an unavailable pin. Cache.Cached reads installed metadata without downloading or rewriting module contents and requires prior verification. Git credentials and transport configuration remain the responsibility of system Git. Ordinary native source use does not require objects for unselected policies. Explicit mod download repairs missing policy object sources, preserving every reached pin in a shared repository; validation reports missing sources without fetching or writing. +Cache.Install verifies a snapshot on its first use by a cache handle, then reuses that successful verification and its file inventory within the command. The key includes source, commit, content hash and root selection. Installed snapshots are immutable for that lifetime; local replacements are never admitted to this cache. A new command creates a new handle and checks installed contents again. Cache.VerifyCached explicitly rechecks contents even on the same handle, without downloading or writing verification stamps; a failed check invalidates the remembered result. + +A newly acquired snapshot is checked with the full h1: directory hash. On Darwin/Linux, an unchanged snapshot from a previous command can use a sidecar stamp keyed by the expected lock hash plus an inode/ctime metadata fingerprint; any metadata change forces the full directory hash again. Content changes remain detectable on the next command or explicit verification even when size and mtime are restored. Platforms without a strong change token keep the full-hash path on first use. Stamp hits first check the current snapshot selection policy. The stamp is an optimization of the per-user local cache, not an additional source of dependency identity. A completed, hashed snapshot produced during revision resolution is installed atomically and reused by the following install step; provisional root inspections cannot publish an incomplete snapshot. + +The optional Cache.SourceFiles capability exposes a copy of the verified snapshot's regular-file inventory, including configuration markers needed for nested module boundaries. Dependency source discovery, global import-collision checks and module ownership use this inventory instead of walking the installed filesystem for each target. Generation shares one collision/ownership index across managed options and descriptor attribution. This does not limit collision detection to files reached by compilation. + +Cached objects support shallow pinned-commit fetches with an advertised-history fallback; neither path substitutes HEAD for an unavailable pin. Cache.Cached reads installed metadata without downloading or rewriting module contents and requires prior verification. Git credentials and transport configuration remain the responsibility of system Git. Ordinary native source use does not require objects for unselected policies. Explicit mod download repairs missing policy object sources, preserving every reached pin in a shared repository; validation reports missing sources without fetching or writing. EASYP_PROXY, when set to a host or host:port, replaces the host of Git remotes used to fetch dependencies. A module declared as github.com/group/package is cloned from https://gitlab.com/group/package when the variable is gitlab.com. Explicit https and ssh URLs keep their path and user; only the host changes, including a port supplied by the variable. Absolute local paths are unchanged. The published module identity, lock source, and content hash stay the original module path. An empty value disables substitution. config.Config.LoadEnvironment reads the variable into Config.Proxy; it is not an easyp.yaml field. @@ -81,8 +87,10 @@ phases. Start/finish records include status and elapsed time; inventories includ file counts and selected roots/filters. Slow phases report elapsed progress every 15 seconds at info level, including without --debug. These logs do not change source selection or dependency identity. Indexed tree and blob -metadata are reused only within one immutable Git filesystem; mutable local -source resolution remains uncached. +metadata are reused within one immutable Git filesystem. Verified installed +snapshot inventories are reused within one command. Mutable local source +resolution remains uncached between calls. A walk resolves its local boundary +once; reusing descendant metadata is restricted to explicitly immutable views. Deleting the old easyp.lock is not required for migration. Its absence does not change an explicit git_repo.url: repo@<commit> pin. @@ -163,7 +171,7 @@ Explicit --frozen requires protobuf.mod and prot All root replacement directives are forbidden, even when unused. Validation rejects them before opening replacement directories or accessing dependency caches/remotes. Dependency manifests still supply transitive requirements; dependency-local replacements remain ignored. The locked closure must satisfy direct and transitive requirements with no missing or stale entries. Malformed locks, incompatible versions/commits and unexpected graph entries fail instead of triggering resolution. -The frozen path does not call the version resolver, query HEAD or tags, or create/rewrite manifests and locks. Exact locked commits may be downloaded when absent from cache; installed sources are lock-hash verified on cold paths, while unchanged Darwin/Linux warm hits reuse the verified metadata stamp and fall back to a full h1: check after any detected filesystem change. This permits reproducible dependency selection with a cold cache and does not imply offline execution. Existing Git credentials, transport configuration, and plugin network requirements still apply. +The frozen path does not call the version resolver, query HEAD or tags, or create/rewrite manifests and locks. Exact locked commits may be downloaded when absent from cache. First use by the command verifies the snapshot against the lock, using the Darwin/Linux metadata stamp where available and falling back to a full h1: check after a detected filesystem change. Later targets reuse that verified immutable snapshot within the same command. This permits reproducible dependency selection with a cold cache and does not imply offline execution. Existing Git credentials, transport configuration, and plugin network requirements still apply. - mod download validates and installs the locked graph; mod vendor validates it before replacing vendor output. - mod tidy, get, mod update, and init refuse frozen operation. migrate, including preview and interactive mode, is also rejected. diff --git a/internal/adapters/gitmodules/bsr_test.go b/internal/adapters/gitmodules/bsr_test.go index c2d66b0f..b043aef3 100644 --- a/internal/adapters/gitmodules/bsr_test.go +++ b/internal/adapters/gitmodules/bsr_test.go @@ -56,7 +56,11 @@ func TestFetchBSRDependencies(t *testing.T) { entries, readErr := os.ReadDir(cache.root) require.NoError(t, readErr) for _, entry := range entries { - assert.Equal(t, "objects", entry.Name(), "temporary checkout survived Fetch") + allowed := []string{"objects"} + if err == nil { + allowed = append(allowed, "modules", "sources") + } + assert.Contains(t, allowed, entry.Name(), "temporary checkout survived Fetch") } if tt.wantErrIs != nil { require.ErrorIs(t, err, tt.wantErrIs) diff --git a/internal/adapters/gitmodules/cache.go b/internal/adapters/gitmodules/cache.go index 9f1ffec7..41106356 100644 --- a/internal/adapters/gitmodules/cache.go +++ b/internal/adapters/gitmodules/cache.go @@ -6,6 +6,7 @@ import ( "fmt" "log/slog" "path/filepath" + "sync" "github.com/easyp-tech/easyp/internal/adapters/bsr" "github.com/easyp-tech/easyp/internal/adapters/gitcommand" @@ -21,14 +22,16 @@ type Cache struct { bsrResolver modules.BSRResolver logger logger.Logger proxy string + stateMu sync.Mutex + state *cacheRunState } // WithLogger returns an isolated cache handle with dependency progress logging. -// Historical-pin scopes preserve this logger through their existing shallow copy. +// Logger and historical-pin scopes share verified immutable snapshot state. func (c *Cache) WithLogger(log logger.Logger) *Cache { - scoped := *c + scoped := c.scoped() scoped.logger = log - return &scoped + return scoped } func (c *Cache) operationContext(ctx context.Context, source, version string) context.Context { diff --git a/internal/adapters/gitmodules/cache_run.go b/internal/adapters/gitmodules/cache_run.go new file mode 100644 index 00000000..1536d301 --- /dev/null +++ b/internal/adapters/gitmodules/cache_run.go @@ -0,0 +1,99 @@ +package gitmodules + +import ( + "context" + "fmt" + "os" + "sync" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" +) + +// A cache handle belongs to one command. Only its successfully installed, +// immutable snapshots share verification and file inventories within that run. +type cacheRunState struct { + mu sync.Mutex + snapshots map[string]*cachedSnapshot +} + +type cachedSnapshot struct { + gate chan struct{} + verified bool + files []string + identity os.FileInfo +} + +func (snapshot *cachedSnapshot) lock(ctx context.Context) error { + select { + case snapshot.gate <- struct{}{}: + if err := ctx.Err(); err != nil { + snapshot.unlock() + return err + } + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (snapshot *cachedSnapshot) unlock() { <-snapshot.gate } + +// accept records a snapshot hashed before its atomic publication. The caller +// holds the per-snapshot gate; no second file-content pass is needed. +func (snapshot *cachedSnapshot) accept(installed string, files []string) error { + info, err := os.Lstat(installed) + if err != nil { + return fmt.Errorf("Lstat: %w", err) + } + // Persistent verification stamps require matching fingerprints before and + // after the content hash. Publication has only the already computed hash; + // a later command verifies the installed tree before writing a stamp. + snapshot.files, snapshot.identity, snapshot.verified = files, info, true + return nil +} + +func (c *Cache) runState() *cacheRunState { + c.stateMu.Lock() + defer c.stateMu.Unlock() + if c.state == nil { + c.state = &cacheRunState{snapshots: make(map[string]*cachedSnapshot)} + } + return c.state +} + +func (c *Cache) scoped() *Cache { + return &Cache{root: c.root, bsrResolver: c.bsrResolver, logger: c.logger, proxy: c.proxy, state: c.runState()} +} + +func (c *Cache) snapshotState(installed string) *cachedSnapshot { + state := c.runState() + state.mu.Lock() + defer state.mu.Unlock() + if snapshot := state.snapshots[installed]; snapshot != nil { + return snapshot + } + snapshot := &cachedSnapshot{gate: make(chan struct{}, 1)} + state.snapshots[installed] = snapshot + return snapshot +} + +// SourceFiles returns the sorted logical file inventory of a snapshot verified +// by this cache handle. Paths are relative to its installation directory and +// include metadata so consumers can preserve nested module ownership. +// The returned slice is independent of the command's immutable cached inventory. +// A nil slice means this handle has no verified inventory and callers must use +// their ordinary source traversal. A verified empty snapshot returns an empty, +// non-nil slice. This method never verifies or trusts a directory's existence. +func (c *Cache) SourceFiles(entry v1.LockedModule) ([]string, error) { + snapshot := c.snapshotState(v1ModuleCachePath(c.root, entry)) + if err := snapshot.lock(context.Background()); err != nil { + return nil, fmt.Errorf("lock: %w", err) + } + defer snapshot.unlock() + if !snapshot.verified { + return nil, nil + } + files := make([]string, len(snapshot.files)) + copy(files, snapshot.files) + return files, nil +} diff --git a/internal/adapters/gitmodules/cache_run_test.go b/internal/adapters/gitmodules/cache_run_test.go new file mode 100644 index 00000000..5826cd1f --- /dev/null +++ b/internal/adapters/gitmodules/cache_run_test.go @@ -0,0 +1,128 @@ +package gitmodules + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" +) + +func TestSnapshotVerificationWaitHonorsCancellation(t *testing.T) { + t.Parallel() + cache := New(t.TempDir()) + entry := v1.LockedModule{ + Source: "example.test/module", Version: "v1.0.0", + Commit: "1111111111111111111111111111111111111111", + Hash: "h1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + } + snapshot := cache.snapshotState(v1ModuleCachePath(cache.root, entry)) + require.NoError(t, snapshot.lock(t.Context())) + defer snapshot.unlock() + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond) + defer cancel() + err := cache.Install(ctx, v1.Lock{Version: 1, Modules: []v1.LockedModule{entry}}) + require.ErrorIs(t, err, context.DeadlineExceeded) +} + +func TestSourceFilesDistinguishesMissingAndEmptyInventory(t *testing.T) { + t.Parallel() + cache := New(t.TempDir()) + entry := v1.LockedModule{ + Source: "example.test/module", Version: "v1.0.0", + Commit: "1111111111111111111111111111111111111111", + } + files, err := cache.SourceFiles(entry) + require.NoError(t, err) + require.Nil(t, files) + // Compute the actual empty-directory hash before selecting the cache path. + entry.Hash, err = hashSnapshotV1Files(t.TempDir()) + require.NoError(t, err) + directory := v1ModuleCachePath(cache.root, entry) + require.NoError(t, os.MkdirAll(directory, 0o755)) + require.NoError(t, cache.VerifyCached(t.Context(), v1.Lock{Version: 1, Modules: []v1.LockedModule{entry}})) + files, err = cache.SourceFiles(entry) + require.NoError(t, err) + require.NotNil(t, files) + require.Empty(t, files) +} + +func TestInstalledSnapshotInventoryIsReusedWithinRun(t *testing.T) { + t.Parallel() + remote, commit := diagnosticRepository(t) + cache := New(t.TempDir()) + fetched, err := cache.Fetch(t.Context(), remote, commit) + require.NoError(t, err) + lock := v1.Lock{Version: 1, Modules: []v1.LockedModule{fetched.Lock}} + require.NoError(t, cache.Install(t.Context(), lock)) + files, err := cache.SourceFiles(fetched.Lock) + require.NoError(t, err) + assert.Equal(t, []string{"api.proto", "easyp.yaml"}, files) + files[0] = "caller mutation" + files, err = cache.WithLogger(nil).SourceFiles(fetched.Lock) + require.NoError(t, err) + assert.Equal(t, []string{"api.proto", "easyp.yaml"}, files) + + other := fetched.Lock + other.Roots = []string{"elsewhere"} + files, err = cache.SourceFiles(other) + require.NoError(t, err) + assert.Nil(t, files) + + directory, _, err := cache.Cached(fetched.Lock) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(directory, "api.proto"), []byte("tampered"), 0o644)) + // Ordinary use trusts an installed snapshot for this command. A fresh handle + // and explicit verification still detect corruption, without changing files. + require.NoError(t, cache.Install(t.Context(), lock)) + fresh := &Cache{root: cache.root} + files, err = fresh.SourceFiles(fetched.Lock) + require.NoError(t, err) + assert.Nil(t, files) + require.ErrorContains(t, fresh.Install(t.Context(), lock), "hash mismatch") + require.ErrorContains(t, cache.VerifyCached(t.Context(), lock), "hash mismatch") + files, err = cache.SourceFiles(fetched.Lock) + require.NoError(t, err) + assert.Nil(t, files) + require.ErrorContains(t, cache.Install(t.Context(), lock), "hash mismatch") +} + +func TestInstalledSnapshotConcurrentReaders(t *testing.T) { + t.Parallel() + remote, commit := diagnosticRepository(t) + producer := New(t.TempDir()) + fetched, err := producer.Fetch(t.Context(), remote, commit) + require.NoError(t, err) + lock := v1.Lock{Version: 1, Modules: []v1.LockedModule{fetched.Lock}} + cache := &Cache{root: producer.root} + var group sync.WaitGroup + failures := make(chan error, 16) + for range 16 { + group.Go(func() { + if err := cache.Install(t.Context(), lock); err != nil { + failures <- err + return + } + files, err := cache.SourceFiles(fetched.Lock) + if err != nil { + failures <- err + return + } + if len(files) != 2 || files[0] != "api.proto" || files[1] != "easyp.yaml" { + failures <- fmt.Errorf("unexpected inventory %v", files) + } + }) + } + group.Wait() + close(failures) + for err := range failures { + require.NoError(t, err) + } +} diff --git a/internal/adapters/gitmodules/cache_verification.go b/internal/adapters/gitmodules/cache_verification.go index ae3496a9..44f96c55 100644 --- a/internal/adapters/gitmodules/cache_verification.go +++ b/internal/adapters/gitmodules/cache_verification.go @@ -11,8 +11,6 @@ import ( "path" "path/filepath" - "golang.org/x/mod/sumdb/dirhash" - v1 "github.com/easyp-tech/easyp/internal/config/v1" ) @@ -26,41 +24,42 @@ type cacheVerificationStamp struct { } func verifyInstalledV1Module(installed string, entry v1.LockedModule, writeStamp bool) error { - files, err := snapshotV1Files(installed) + _, err := verifiedSnapshotFiles(installed, entry, writeStamp) + return err +} + +func verifiedSnapshotFiles(installed string, entry v1.LockedModule, writeStamp bool) ([]string, error) { + files, fingerprint, fast, err := cacheTreeInventory(installed) if err != nil { - return fmt.Errorf("snapshotV1Files: %w", err) + return nil, fmt.Errorf("cacheTreeInventory: %w", err) } for _, name := range files { if path.Ext(name) != ".proto" && !snapshotConfigFile(name) { - return fmt.Errorf("unrelated snapshot file %q; regenerate the unreleased v1 lock with easyp mod tidy", name) + return nil, fmt.Errorf("unrelated snapshot file %q; regenerate the unreleased v1 lock with easyp mod tidy", name) } } - fingerprint, fast, err := cacheTreeFingerprint(installed) - if err != nil { - return fmt.Errorf("cacheTreeFingerprint: %w", err) - } if fast { stamp, readErr := readCacheVerificationStamp(installed) if readErr == nil && stamp.Version == cacheVerificationVersion && stamp.Hash == entry.Hash && stamp.Roots == v1RootSelectionKey(entry.Roots) && stamp.Fingerprint == fingerprint { - return nil + return files, nil } } - actual, err := dirhash.HashDir(installed, "", dirhash.Hash1) + actual, err := hashV1Files(installed, files) if err != nil { - return fmt.Errorf("HashDir: %w", err) + return nil, fmt.Errorf("hashV1Files: %w", err) } if actual != entry.Hash { - return fmt.Errorf("cached %s@%s hash mismatch: got %s, want %s; inspect or quarantine only cache directory %q, then rerun easyp mod download; keep protobuf.lock unchanged", entry.Source, entry.Commit, actual, entry.Hash, installed) + return nil, fmt.Errorf("cached %s@%s hash mismatch: got %s, want %s; inspect or quarantine only cache directory %q, then rerun easyp mod download; keep protobuf.lock unchanged", entry.Source, entry.Commit, actual, entry.Hash, installed) } if fast { after, afterFast, fingerprintErr := cacheTreeFingerprint(installed) if fingerprintErr != nil { - return fmt.Errorf("cacheTreeFingerprint after hash: %w", fingerprintErr) + return nil, fmt.Errorf("cacheTreeFingerprint after hash: %w", fingerprintErr) } if !afterFast || after != fingerprint { - return fmt.Errorf("cached %s@%s changed while its lock hash was being verified; retry after concurrent cache writes stop", entry.Source, entry.Commit) + return nil, fmt.Errorf("cached %s@%s changed while its lock hash was being verified; retry after concurrent cache writes stop", entry.Source, entry.Commit) } if writeStamp { _ = writeCacheVerificationStamp(installed, cacheVerificationStamp{ @@ -71,7 +70,7 @@ func verifyInstalledV1Module(installed string, entry v1.LockedModule, writeStamp }) } } - return nil + return files, nil } func cacheVerificationStampPath(installed string) string { @@ -120,6 +119,12 @@ func writeCacheVerificationStamp(installed string, stamp cacheVerificationStamp) } func cacheTreeFingerprint(root string) (string, bool, error) { + _, fingerprint, supported, err := cacheTreeInventory(root) + return fingerprint, supported, err +} + +func cacheTreeInventory(root string) ([]string, string, bool, error) { + var files []string hash := sha256.New() supported := true err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error { @@ -145,6 +150,9 @@ func cacheTreeFingerprint(root string) (string, bool, error) { relative = "" } name := filepath.ToSlash(relative) + if info.Mode().IsRegular() { + files = append(files, name) + } if err := binary.Write(hash, binary.LittleEndian, uint32(len(name))); err != nil { return err } @@ -167,7 +175,7 @@ func cacheTreeFingerprint(root string) (string, bool, error) { return nil }) if err != nil { - return "", false, err + return nil, "", false, err } - return "m1:" + base64.StdEncoding.EncodeToString(hash.Sum(nil)), supported, nil + return files, "m1:" + base64.StdEncoding.EncodeToString(hash.Sum(nil)), supported, nil } diff --git a/internal/adapters/gitmodules/checkout_test.go b/internal/adapters/gitmodules/checkout_test.go index e8933930..e8ef5d8f 100644 --- a/internal/adapters/gitmodules/checkout_test.go +++ b/internal/adapters/gitmodules/checkout_test.go @@ -62,7 +62,7 @@ func TestFetchV1ModuleRemovesTemporaryCheckouts(t *testing.T) { entries, err := os.ReadDir(cache) require.NoError(t, err) for _, entry := range entries { - assert.Equal(t, "objects", entry.Name(), "temporary checkout was not removed") + assert.Contains(t, []string{"objects", "modules", "sources"}, entry.Name(), "temporary checkout was not removed") } }) } diff --git a/internal/adapters/gitmodules/diagnostics_test.go b/internal/adapters/gitmodules/diagnostics_test.go index b05c575f..38f60f9b 100644 --- a/internal/adapters/gitmodules/diagnostics_test.go +++ b/internal/adapters/gitmodules/diagnostics_test.go @@ -39,7 +39,7 @@ func TestCacheIntegrityDiagnosticIdentifiesRecoveryPath(t *testing.T) { require.NoError(t, err) name := filepath.Join(installed, "api.proto") require.NoError(t, os.WriteFile(name, []byte("tampered"), 0o644)) - err = cache.Install(t.Context(), lock) + err = (&Cache{root: cache.root}).Install(t.Context(), lock) require.ErrorContains(t, err, "hash mismatch") assert.Contains(t, err.Error(), installed) assert.Contains(t, err.Error(), "protobuf.lock unchanged") diff --git a/internal/adapters/gitmodules/download.go b/internal/adapters/gitmodules/download.go index 5667598f..88ca3dbe 100644 --- a/internal/adapters/gitmodules/download.go +++ b/internal/adapters/gitmodules/download.go @@ -20,8 +20,8 @@ func v1ModuleCachePath(cacheRoot string, entry v1.LockedModule) string { return filepath.Join(cacheRoot, "modules", v1CacheSourceKey(entry.Source), v1CacheSourceKey(identity)) } -// Install installs each locked Git commit, verifying its content hash -// before accepting a downloaded or already cached module. +// Install installs each locked Git commit and verifies its contents once per +// cache handle. Installed snapshots are immutable for the command's lifetime. func (c *Cache) Install(ctx context.Context, lock v1.Lock) error { return c.install(ctx, lock, true) } @@ -57,7 +57,20 @@ func (c *Cache) installEntry(ctx context.Context, entry v1.LockedModule, acquire finish := gitcommand.Start(ctx, "install module") defer func() { finish(err) }() installed := v1ModuleCachePath(c.root, entry) + snapshot := c.snapshotState(installed) + if err := snapshot.lock(ctx); err != nil { + return fmt.Errorf("lock: %w", err) + } + defer snapshot.unlock() + if !acquire { + // Explicit read-only verification must not reuse an earlier success. + // A failed check also invalidates later Install and SourceFiles calls. + snapshot.verified = false + } info, err := os.Lstat(installed) + if err != nil || !os.SameFile(snapshot.identity, info) { + snapshot.verified = false + } if errors.Is(err, os.ErrNotExist) { if !acquire { return fmt.Errorf("policy module %s@%s is not cached; run easyp mod download before validate-config: %w", entry.Source, entry.Commit, err) @@ -76,8 +89,12 @@ func (c *Cache) installEntry(ctx context.Context, entry v1.LockedModule, acquire // rather than racing a second rename into the same cache directory. info, err = os.Lstat(installed) if errors.Is(err, os.ErrNotExist) { - if err := fetchPinnedV1Module(ctx, entry, c.root, installed, c.proxy); err != nil { - return err + files, fetchErr := fetchPinnedV1Module(ctx, entry, c.root, installed, c.proxy) + if fetchErr != nil { + return fetchErr + } + if err := snapshot.accept(installed, files); err != nil { + return fmt.Errorf("accept: %w", err) } info, err = os.Lstat(installed) } @@ -88,10 +105,16 @@ func (c *Cache) installEntry(ctx context.Context, entry v1.LockedModule, acquire if !info.IsDir() { return fmt.Errorf("%s: cached path %q is not a directory; inspect this path and keep protobuf.lock unchanged", entry.Source, installed) } - if err := verifyInstalledV1Module(installed, entry, acquire); err != nil { - return fmt.Errorf("verify cached %s at %q: %w", entry.Source, installed, err) + if !snapshot.verified { + files, err := verifiedSnapshotFiles(installed, entry, acquire) + if err != nil { + return fmt.Errorf("verify cached %s at %q: %w", entry.Source, installed, err) + } + snapshot.files, snapshot.identity = files, info + snapshot.verified = true } if err := moduleconfig.ValidateLegacyMajor(installed, entry.Source, entry.Version); err != nil { + snapshot.verified = false return fmt.Errorf("ValidateLegacyMajor: %w", err) } return nil diff --git a/internal/adapters/gitmodules/migration.go b/internal/adapters/gitmodules/migration.go index 05b4bc10..74c536ec 100644 --- a/internal/adapters/gitmodules/migration.go +++ b/internal/adapters/gitmodules/migration.go @@ -156,7 +156,7 @@ func (c *Cache) fetchMigration(ctx context.Context, source, version, legacyHash if err != nil { return modules.Fetched{}, fmt.Errorf("sourceV1SnapshotView: %w", err) } - if err := materializeSelectedSnapshotAliases(ctx, view, checkout.snapshot, checkout.module, nil); err != nil { + if err := materializeSelectedSnapshotAliases(ctx, newSnapshotWriter(view, checkout.snapshot), checkout.module, nil, true); err != nil { return modules.Fetched{}, fmt.Errorf("materializeSelectedSnapshotAliases: %w", err) } checkout.inspection, err = inspectV1Snapshot(ctx, view, checkout.snapshot, checkout.module, checkout.commit) diff --git a/internal/adapters/gitmodules/migration_bsr.go b/internal/adapters/gitmodules/migration_bsr.go index 088cc865..2140ded2 100644 --- a/internal/adapters/gitmodules/migration_bsr.go +++ b/internal/adapters/gitmodules/migration_bsr.go @@ -16,9 +16,9 @@ func (c *Cache) WithMigrationPins(pins []v1.Requirement) modules.MigrationReposi for _, pin := range pins { historical[pin.Module] = pin.Version } - scoped := *c + scoped := c.scoped() scoped.bsrResolver = migrationBSRResolver{base: c.bsrResolver, pins: historical} - return &scoped + return scoped } type migrationBSRResolver struct { diff --git a/internal/adapters/gitmodules/object_cache_test.go b/internal/adapters/gitmodules/object_cache_test.go index a7bd9640..a8b43263 100644 --- a/internal/adapters/gitmodules/object_cache_test.go +++ b/internal/adapters/gitmodules/object_cache_test.go @@ -35,7 +35,7 @@ func TestPinnedObjectCacheWorksWithoutRemote(t *testing.T) { dir, _, err := cache.Cached(first.Lock) require.NoError(t, err) require.NoError(t, os.WriteFile(filepath.Join(dir, "api.proto"), []byte("tampered"), 0o644)) - require.ErrorContains(t, cache.Install(t.Context(), v1.Lock{Version: 1, Modules: []v1.LockedModule{first.Lock}}), "hash mismatch") + require.ErrorContains(t, cache.VerifyCached(t.Context(), v1.Lock{Version: 1, Modules: []v1.LockedModule{first.Lock}}), "hash mismatch") } func TestObjectRepositoryLockHonorsCancellation(t *testing.T) { diff --git a/internal/adapters/gitmodules/root_selection_test.go b/internal/adapters/gitmodules/root_selection_test.go index 0c977dea..03bb4a51 100644 --- a/internal/adapters/gitmodules/root_selection_test.go +++ b/internal/adapters/gitmodules/root_selection_test.go @@ -56,6 +56,8 @@ func TestFetchWithRootsReplaysColdAndWarmSelections(t *testing.T) { lock := v1.Lock{Version: 1, Modules: []v1.LockedModule{entry}} require.NoError(t, cache.Install(t.Context(), lock)) require.NoError(t, cache.Install(t.Context(), lock)) + // A new command independently verifies the published tree and may stamp it. + require.NoError(t, (&Cache{root: cache.root}).Install(t.Context(), lock)) require.NoError(t, cache.VerifyCached(t.Context(), lock)) directory, module, err := cache.Cached(entry) require.NoError(t, err) diff --git a/internal/adapters/gitmodules/snapshot.go b/internal/adapters/gitmodules/snapshot.go index 47c598a0..6f0fc34d 100644 --- a/internal/adapters/gitmodules/snapshot.go +++ b/internal/adapters/gitmodules/snapshot.go @@ -34,7 +34,7 @@ func prepareV1Snapshot(ctx context.Context, checkout, commit, source, subdir str if err != nil { return "", v1.Module{}, nil, fmt.Errorf("sourceV1SnapshotTree: %w", err) } - view := sourceview.New(tree) + view := sourceview.NewImmutable(tree) directory, err = os.MkdirTemp(filepath.Dir(checkout), "snapshot-*") if err != nil { return "", v1.Module{}, nil, fmt.Errorf("MkdirTemp: %w", err) @@ -48,7 +48,8 @@ func prepareV1Snapshot(ctx context.Context, checkout, commit, source, subdir str } } }() - inventory, err := stageSnapshotMetadata(ctx, tree, view, directory, subdir) + writer := newSnapshotWriter(view, directory) + inventory, err := stageSnapshotMetadata(ctx, tree, writer, subdir) if err != nil { return "", v1.Module{}, nil, fmt.Errorf("stageSnapshotMetadata: %w", err) } @@ -85,21 +86,23 @@ func prepareV1Snapshot(ctx context.Context, checkout, commit, source, subdir str return "", v1.Module{}, nil, fmt.Errorf("pruneEmptySnapshotParents: %w", err) } } + // Ignored metadata may have removed directories with a different host spelling. + writer.destination.Reset() // Stage ordinary files only after metadata has selected Buf source paths. // Discarded proto names must not collide with retained names on the host. for _, name := range regular { if path.Ext(name) == ".proto" && len(selectV1ProtoFiles([]string{name}, module.ProtoFilters)) == 0 { continue } - if err := materializeSnapshotFile(ctx, view, directory, name); err != nil { - return "", v1.Module{}, nil, fmt.Errorf("materializeSnapshotFile: %w", err) + if err := writer.file(ctx, name); err != nil { + return "", v1.Module{}, nil, fmt.Errorf("file: %w", err) } } - if err := materializeSnapshotAuxiliaryAliases(ctx, view, directory, aliases); err != nil { + if err := materializeSnapshotAuxiliaryAliases(ctx, writer, aliases); err != nil { return "", v1.Module{}, nil, fmt.Errorf("materializeSnapshotAuxiliaryAliases: %w", err) } if !provisional { - if err := materializeSelectedSnapshotAliases(ctx, view, directory, module, ignoredMetadata); err != nil { + if err := materializeSelectedSnapshotAliases(ctx, writer, module, ignoredMetadata, len(aliases) > 0); err != nil { return "", v1.Module{}, nil, fmt.Errorf("materializeSelectedSnapshotAliases: %w", err) } } @@ -110,26 +113,6 @@ func prepareV1Snapshot(ctx context.Context, checkout, commit, source, subdir str } inspection.Provisional = provisional } - // Buf filters select the same logical paths for hashing and installation. - err = filepath.WalkDir(directory, func(file string, entry fs.DirEntry, walkErr error) error { - if walkErr != nil { - return walkErr - } - if entry.IsDir() || filepath.Ext(file) != ".proto" { - return nil - } - name, err := filepath.Rel(directory, file) - if err != nil { - return fmt.Errorf("Rel: %w", err) - } - if len(selectV1ProtoFiles([]string{filepath.ToSlash(name)}, module.ProtoFilters)) == 0 { - return os.Remove(file) - } - return nil - }) - if err != nil { - return "", v1.Module{}, nil, fmt.Errorf("WalkDir: %w", err) - } return directory, module, inspection, nil } @@ -160,18 +143,19 @@ func pruneEmptySnapshotParents(directory, name string) error { // stageSnapshotMetadata discovers immutable metadata before source filtering. // Failed markers preserve ownership and parser precedence without pointer reads. -func stageSnapshotMetadata(ctx context.Context, tree *gitsnapshot.FS, view *sourceview.View, directory, subdir string) (snapshotInventory, error) { +func stageSnapshotMetadata(ctx context.Context, tree *gitsnapshot.FS, writer *snapshotWriter, subdir string) (snapshotInventory, error) { + view := writer.view var aliases, regular []string var metadataFailures []snapshotMetadataFailure failedMarker := func(name string, cause error) error { failure := snapshotMetadataFailure{name: name, err: cause} - err := makeSnapshotDirectory(directory, name) + err := writer.mkdir(name) if errors.Is(err, gitsnapshot.ErrPathCollision) { // A failed alias has no retained bytes. Keep its witness separate // if the host cannot represent it beside staged metadata. failure.unstaged = true } else if err != nil { - return fmt.Errorf("makeSnapshotDirectory: %w", err) + return fmt.Errorf("mkdir: %w", err) } metadataFailures = append(metadataFailures, failure) return nil @@ -203,7 +187,7 @@ func stageSnapshotMetadata(ctx context.Context, tree *gitsnapshot.FS, view *sour } return nil } - return materializeSnapshotFile(ctx, view, directory, name) + return writer.file(ctx, name) }) if err != nil { return snapshotInventory{}, fmt.Errorf("WalkDir: %w", err) @@ -248,7 +232,7 @@ func stageSnapshotMetadata(ctx context.Context, tree *gitsnapshot.FS, view *sour return nil } if moduleconfig.IsGitDependencyConfigFile(path.Base(logical)) && !snapshotMetadataExcluded(logical, subdir) { - return materializeSnapshotFile(ctx, view, directory, logical) + return writer.file(ctx, logical) } return nil }) @@ -258,8 +242,8 @@ func stageSnapshotMetadata(ctx context.Context, tree *gitsnapshot.FS, view *sour continue } if moduleconfig.IsGitDependencyConfigFile(path.Base(name)) { - if err := materializeSnapshotFile(ctx, view, directory, name); err != nil { - return snapshotInventory{}, fmt.Errorf("materializeSnapshotFile: %w", err) + if err := writer.file(ctx, name); err != nil { + return snapshotInventory{}, fmt.Errorf("file: %w", err) } } } @@ -268,13 +252,14 @@ func stageSnapshotMetadata(ctx context.Context, tree *gitsnapshot.FS, view *sour // materializeSelectedSnapshotAliases validates declared roots and copies aliases // selected by source ownership and Buf filters. Auxiliary links are handled separately. -func materializeSelectedSnapshotAliases(ctx context.Context, view *sourceview.View, directory string, module v1.Module, ignoredMetadata map[string]bool) error { +func materializeSelectedSnapshotAliases(ctx context.Context, writer *snapshotWriter, module v1.Module, ignoredMetadata map[string]bool, hasAliases bool) error { + view, directory := writer.view, writer.directory for _, root := range module.Roots { logicalRoot := filepath.ToSlash(root) resolution, resolveErr := view.Resolve(ctx, logicalRoot) if resolveErr != nil { if errors.Is(resolveErr, fs.ErrNotExist) && len(resolution.Links) == 0 { - if err := makeSnapshotDirectory(directory, root); err != nil { + if err := writer.mkdir(root); err != nil { return fmt.Errorf("MkdirAll: %w", err) } continue @@ -284,9 +269,12 @@ func materializeSelectedSnapshotAliases(ctx context.Context, view *sourceview.Vi if !resolution.Info.IsDir() { return fmt.Errorf("module %s has invalid root %q: not a directory", module.Name, root) } - if err := makeSnapshotDirectory(directory, root); err != nil { + if err := writer.mkdir(root); err != nil { return fmt.Errorf("MkdirAll: %w", err) } + if !hasAliases { + continue + } err := view.WalkSelected(ctx, logicalRoot, snapshotSourceCandidate, func(logical string, resolved sourceview.Resolution, walkErr error) error { if ignoredMetadata[logical] { return nil @@ -317,7 +305,7 @@ func materializeSelectedSnapshotAliases(ctx context.Context, view *sourceview.Vi if !snapshotSelectedAlias(logical, module) { return nil } - return materializeSnapshotFile(ctx, view, directory, logical) + return writer.file(ctx, logical) }) if err != nil { return fmt.Errorf("Walk: root %q: %w", root, err) @@ -331,7 +319,7 @@ func sourceV1SnapshotView(ctx context.Context, checkout, commit string) (*source if err != nil { return nil, fmt.Errorf("sourceV1SnapshotTree: %w", err) } - return sourceview.New(tree), nil + return sourceview.NewImmutable(tree), nil } func sourceV1SnapshotTree(ctx context.Context, checkout, commit string) (*gitsnapshot.FS, error) { @@ -420,7 +408,8 @@ func snapshotFailedMetadataOwnsSources(name, directory string, module v1.Module) // materializeSnapshotAuxiliaryAliases retains bounded regular non-proto aliases. // Consumers can select arbitrary policy fragments; Fetch has no consumer context // with which to narrow those paths. Failed auxiliary aliases remain omitted. -func materializeSnapshotAuxiliaryAliases(ctx context.Context, view *sourceview.View, directory string, aliases []string) error { +func materializeSnapshotAuxiliaryAliases(ctx context.Context, writer *snapshotWriter, aliases []string) error { + view := writer.view for _, name := range aliases { resolved, err := view.Resolve(ctx, name) if err != nil { @@ -437,7 +426,7 @@ func materializeSnapshotAuxiliaryAliases(ctx context.Context, view *sourceview.V if resolved.Info.IsDir() || !snapshotConfigFile(logical) { return nil } - return materializeSnapshotFile(ctx, view, directory, logical) + return writer.file(ctx, logical) }) if err != nil { return fmt.Errorf("Walk: %w", err) @@ -447,8 +436,8 @@ func materializeSnapshotAuxiliaryAliases(ctx context.Context, view *sourceview.V if !snapshotConfigFile(name) { continue } - if err := materializeSnapshotFile(ctx, view, directory, name); err != nil { - return fmt.Errorf("materializeSnapshotFile: %w", err) + if err := writer.file(ctx, name); err != nil { + return fmt.Errorf("file: %w", err) } } return nil @@ -528,8 +517,18 @@ func snapshotExcludedDirectory(name, root, directory string, module v1.Module) b return true } -func materializeSnapshotFile(ctx context.Context, view *sourceview.View, directory, name string) error { - file, err := view.Open(ctx, name) +type snapshotWriter struct { + view *sourceview.View + directory string + destination *gitsnapshot.DestinationValidator +} + +func newSnapshotWriter(view *sourceview.View, directory string) *snapshotWriter { + return &snapshotWriter{view: view, directory: directory, destination: gitsnapshot.NewDestinationValidator(directory)} +} + +func (writer *snapshotWriter) file(ctx context.Context, name string) error { + file, err := writer.view.Open(ctx, name) if err != nil { return fmt.Errorf("Open: %w", err) } @@ -542,8 +541,8 @@ func materializeSnapshotFile(ctx context.Context, view *sourceview.View, directo _ = file.Close() return fmt.Errorf("non-regular snapshot file %q", name) } - destination := filepath.Join(directory, filepath.FromSlash(name)) - if err := gitsnapshot.ValidateDestination(directory, name); err != nil { + destination := filepath.Join(writer.directory, filepath.FromSlash(name)) + if err := writer.destination.Validate(name); err != nil { closeErr := file.Close() return errors.Join(fmt.Errorf("ValidateDestination: %w", err), closeErr) } @@ -561,16 +560,18 @@ func materializeSnapshotFile(ctx context.Context, view *sourceview.View, directo if err := errors.Join(copyErr, closeErr, ctx.Err()); err != nil { return fmt.Errorf("Copy: %w", err) } + writer.destination.Record(name) return nil } -func makeSnapshotDirectory(directory, name string) error { - if err := gitsnapshot.ValidateDestination(directory, name); err != nil { +func (writer *snapshotWriter) mkdir(name string) error { + if err := writer.destination.Validate(name); err != nil { return fmt.Errorf("ValidateDestination: %w", err) } - if err := os.MkdirAll(filepath.Join(directory, filepath.FromSlash(name)), 0o755); err != nil { + if err := os.MkdirAll(filepath.Join(writer.directory, filepath.FromSlash(name)), 0o755); err != nil { return fmt.Errorf("MkdirAll: %w", err) } + writer.destination.Record(name) return nil } diff --git a/internal/adapters/gitmodules/snapshot_checkout.go b/internal/adapters/gitmodules/snapshot_checkout.go index 21de7bb2..6daf5173 100644 --- a/internal/adapters/gitmodules/snapshot_checkout.go +++ b/internal/adapters/gitmodules/snapshot_checkout.go @@ -64,10 +64,15 @@ func (c *Cache) fetchWithRootSelection(ctx context.Context, source, version stri return modules.Fetched{}, fmt.Errorf("ValidateLegacyMajor: %w", err) } var hash string + var files []string if checkout.inspection == nil || !checkout.inspection.Provisional { - hash, err = hashSnapshotV1Files(checkout.snapshot) + files, err = snapshotV1Files(checkout.snapshot) if err != nil { - return modules.Fetched{}, fmt.Errorf("hashSnapshotV1Files: %w", err) + return modules.Fetched{}, fmt.Errorf("snapshotV1Files: %w", err) + } + hash, err = hashV1Files(checkout.snapshot, files) + if err != nil { + return modules.Fetched{}, fmt.Errorf("hashV1Files: %w", err) } } if version == "" { @@ -77,7 +82,13 @@ func (c *Cache) fetchWithRootSelection(ctx context.Context, source, version stri if err != nil { return modules.Fetched{}, fmt.Errorf("resolveBSRDependencies: %w", err) } - return modules.Fetched{Module: module, Lock: v1.LockedModule{Source: source, Version: version, Commit: checkout.commit, Hash: hash, Roots: lockedV1ModuleRoots(module, roots), BSR: bindings}, Inspection: checkout.inspection}, nil + entry := v1.LockedModule{Source: source, Version: version, Commit: checkout.commit, Hash: hash, Roots: lockedV1ModuleRoots(module, roots), BSR: bindings} + if hash != "" { + if err := c.retainFetchedSnapshot(ctx, checkout, entry, files); err != nil { + return modules.Fetched{}, fmt.Errorf("retainFetchedSnapshot: %w", err) + } + } + return modules.Fetched{Module: module, Lock: entry, Inspection: checkout.inspection}, nil } // checkoutV1ModuleWithRoots retains Git objects and an index for historical @@ -180,10 +191,10 @@ func checkoutSnapshotCandidate(ctx context.Context, dir, source, version string, return v1ModuleCheckout{dir: dir, module: module, commit: commit, snapshot: stage, inspection: inspection}, true, nil } -func fetchPinnedV1Module(ctx context.Context, entry v1.LockedModule, cacheRoot, installed, proxy string) (err error) { +func fetchPinnedV1Module(ctx context.Context, entry v1.LockedModule, cacheRoot, installed, proxy string) (files []string, err error) { checkout, err := checkoutV1ModuleWithRoots(ctx, entry.Source, entry.Commit, cacheRoot, entry.Roots, false, proxy) if err != nil { - return fmt.Errorf("checkoutV1ModuleWithRoots: %w", err) + return nil, fmt.Errorf("checkoutV1ModuleWithRoots: %w", err) } defer func() { for _, directory := range []string{checkout.snapshot, checkout.dir} { @@ -193,24 +204,28 @@ func fetchPinnedV1Module(ctx context.Context, entry v1.LockedModule, cacheRoot, } } }() - actual, err := hashSnapshotV1Files(checkout.snapshot) + files, err = snapshotV1Files(checkout.snapshot) + if err != nil { + return nil, fmt.Errorf("snapshotV1Files: %w", err) + } + actual, err := hashV1Files(checkout.snapshot, files) if err != nil { - return fmt.Errorf("hashSnapshotV1Files: %w", err) + return nil, fmt.Errorf("hashV1Files: %w", err) } if !strings.EqualFold(checkout.commit, entry.Commit) { - return fmt.Errorf("%s: snapshot commit does not match lock", entry.Source) + return nil, fmt.Errorf("%s: snapshot commit does not match lock", entry.Source) } if actual != entry.Hash { - return fmt.Errorf("%s@%s hash mismatch: got %s, want %s; downloaded contents do not match the pinned hash; investigate repository integrity and keep protobuf.lock unchanged", entry.Source, entry.Commit, actual, entry.Hash) + return nil, fmt.Errorf("%s@%s hash mismatch: got %s, want %s; downloaded contents do not match the pinned hash; investigate repository integrity and keep protobuf.lock unchanged", entry.Source, entry.Commit, actual, entry.Hash) } if err := moduleconfig.ValidateLegacyMajor(checkout.snapshot, entry.Source, entry.Version); err != nil { - return fmt.Errorf("ValidateLegacyMajor: %w", err) + return nil, fmt.Errorf("ValidateLegacyMajor: %w", err) } if err := writeSourceBinding(ctx, cacheRoot, checkout.dir, entry); err != nil { - return fmt.Errorf("writeSourceBinding: %w", err) + return nil, fmt.Errorf("writeSourceBinding: %w", err) } if err := os.Rename(checkout.snapshot, installed); err != nil { - return fmt.Errorf("Rename: %w", err) + return nil, fmt.Errorf("Rename: %w", err) } - return nil + return files, nil } diff --git a/internal/adapters/gitmodules/snapshot_reuse.go b/internal/adapters/gitmodules/snapshot_reuse.go new file mode 100644 index 00000000..7930aa9b --- /dev/null +++ b/internal/adapters/gitmodules/snapshot_reuse.go @@ -0,0 +1,107 @@ +package gitmodules + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" +) + +// retainFetchedSnapshot publishes the bytes just hashed by Fetch. Install can +// reuse this complete snapshot instead of materializing the same commit again. +// Provisional root inspections never reach this path. +func (c *Cache) retainFetchedSnapshot(ctx context.Context, checkout v1ModuleCheckout, entry v1.LockedModule, files []string) error { + installed := v1ModuleCachePath(c.root, entry) + snapshot := c.snapshotState(installed) + if err := snapshot.lock(ctx); err != nil { + return fmt.Errorf("lock: %w", err) + } + defer snapshot.unlock() + if err := os.MkdirAll(filepath.Dir(installed), 0o755); err != nil { + return fmt.Errorf("MkdirAll: %w", err) + } + unlock, err := lockObjectRepository(ctx, installed+".install.lock") + if err != nil { + return fmt.Errorf("lockObjectRepository: %w", err) + } + defer unlock() + info, err := os.Lstat(installed) + if err == nil { + if !info.IsDir() { + return fmt.Errorf("cached path %q is not a directory", installed) + } + if !snapshot.verified || !os.SameFile(snapshot.identity, info) { + snapshot.verified = false + files, err = verifiedSnapshotFiles(installed, entry, true) + if err != nil { + return fmt.Errorf("verifiedSnapshotFiles: %w", err) + } + } + } else if errors.Is(err, os.ErrNotExist) { + if err := retainFetchedSource(ctx, c.root, checkout.dir, entry); err != nil { + return fmt.Errorf("retainFetchedSource: %w", err) + } + if err := os.Rename(checkout.snapshot, installed); err != nil { + return fmt.Errorf("Rename: %w", err) + } + return snapshot.accept(installed, files) + } else { + return fmt.Errorf("Lstat: %w", err) + } + snapshot.files, snapshot.identity, snapshot.verified = files, info, true + return nil +} + +// retainFetchedSource copies already fetched objects locally. HEAD/tag Fetch +// must not download the revision a second time just to preserve policy objects. +func retainFetchedSource(ctx context.Context, root, checkout string, entry v1.LockedModule) error { + absoluteCheckout, err := filepath.Abs(checkout) + if err != nil { + return fmt.Errorf("Abs: %w", err) + } + checkout = absoluteCheckout + raw, err := gitV1(ctx, checkout, "config", "--get", "remote.origin.url") + if err != nil { + return fmt.Errorf("gitV1: %w", err) + } + origin := strings.TrimSpace(raw) + objectsRoot, err := filepath.Abs(filepath.Join(root, "objects")) + if err != nil { + return fmt.Errorf("Abs: %w", err) + } + relative, relErr := filepath.Rel(objectsRoot, origin) + if relErr == nil && filepath.IsLocal(relative) && !strings.ContainsAny(relative, "/\\") { + return writeSourceBindingRepository(root, origin, entry) + } + if err := os.MkdirAll(objectsRoot, 0o700); err != nil { + return fmt.Errorf("MkdirAll: %w", err) + } + repository := filepath.Join(objectsRoot, v1CacheSourceKey(origin)) + unlock, err := lockObjectRepository(ctx, repository+".lock") + if err != nil { + return fmt.Errorf("lockObjectRepository: %w", err) + } + defer unlock() + if _, err := os.Stat(filepath.Join(repository, "HEAD")); errors.Is(err, os.ErrNotExist) { + if _, err := gitV1(ctx, "", "clone", "--quiet", "--bare", "--no-hardlinks", "--", checkout, repository); err != nil { + return fmt.Errorf("gitV1: %w", err) + } + if _, err := gitV1(ctx, repository, "config", "gc.auto", "0"); err != nil { + return fmt.Errorf("gitV1: %w", err) + } + } else if err != nil { + return fmt.Errorf("Stat: %w", err) + } else { + if _, err := gitV1(ctx, repository, "fetch", "--quiet", "--no-tags", "--no-write-fetch-head", "--depth=1", "--", checkout, entry.Commit); err != nil { + return fmt.Errorf("gitV1: %w", err) + } + } + if _, err := gitV1(ctx, repository, "update-ref", "refs/easyp/"+strings.ToLower(entry.Commit), entry.Commit); err != nil { + return fmt.Errorf("gitV1: %w", err) + } + return writeSourceBindingRepository(root, repository, entry) +} diff --git a/internal/adapters/gitmodules/snapshot_reuse_test.go b/internal/adapters/gitmodules/snapshot_reuse_test.go new file mode 100644 index 00000000..bff70187 --- /dev/null +++ b/internal/adapters/gitmodules/snapshot_reuse_test.go @@ -0,0 +1,55 @@ +package gitmodules + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" +) + +func TestFetchRetainsSnapshotForOfflineInstall(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + name, version string + commit bool + }{ + {name: "head"}, + {name: "tag", version: "v1.0.0"}, + {name: "commit", commit: true}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + remote, commit := diagnosticRepository(t) + version := tt.version + if tt.commit { + version = commit + } + cache := New(t.TempDir()) + fetched, err := cache.Fetch(t.Context(), remote, version) + require.NoError(t, err) + directory, _, err := cache.Cached(fetched.Lock) + require.NoError(t, err) + before, err := os.Stat(filepath.Join(directory, "api.proto")) + require.NoError(t, err) + lock := v1.Lock{Version: 1, Modules: []v1.LockedModule{fetched.Lock}} + require.NoError(t, os.Rename(remote, remote+"-offline")) + require.NoError(t, cache.Install(t.Context(), lock)) + assert.NoFileExists(t, cacheVerificationStampPath(directory), "publication must not attest a fingerprint collected after hashing") + after, err := os.Stat(filepath.Join(directory, "api.proto")) + require.NoError(t, err) + assert.True(t, os.SameFile(before, after), "Install must retain the already materialized file") + require.NoError(t, (&Cache{root: cache.root}).VerifyCached(t.Context(), lock)) + + // Retained local objects also reconstruct the same pinned snapshot + // after removing the installation, without the original repository. + require.NoError(t, os.RemoveAll(directory)) + fresh := &Cache{root: cache.root} + require.NoError(t, fresh.Install(t.Context(), lock)) + require.NoError(t, fresh.VerifyCached(t.Context(), lock)) + }) + } +} diff --git a/internal/adapters/gitmodules/source_binding.go b/internal/adapters/gitmodules/source_binding.go index b9f03be7..3e494d4d 100644 --- a/internal/adapters/gitmodules/source_binding.go +++ b/internal/adapters/gitmodules/source_binding.go @@ -48,6 +48,10 @@ func writeSourceBinding(ctx context.Context, root, checkout string, entry v1.Loc return fmt.Errorf("gitV1: %w", err) } repository := strings.TrimSpace(data) + return writeSourceBindingRepository(root, repository, entry) +} + +func writeSourceBindingRepository(root, repository string, entry v1.LockedModule) (err error) { objectsRoot, err := filepath.Abs(filepath.Join(root, "objects")) if err != nil { return fmt.Errorf("Abs: %w", err) diff --git a/internal/adapters/gitsnapshot/destination.go b/internal/adapters/gitsnapshot/destination.go index 3f576a26..94655613 100644 --- a/internal/adapters/gitsnapshot/destination.go +++ b/internal/adapters/gitsnapshot/destination.go @@ -14,12 +14,60 @@ var ErrPathCollision = errors.New("snapshot path collision") // ValidateDestination rejects distinct Git paths that the host filesystem aliases. // Git paths stay case-sensitive even on a case-insensitive staging filesystem. func ValidateDestination(directory, name string) error { + return validateDestination(directory, name, nil, nil) +} + +// DestinationValidator reuses validated path prefixes while one writer builds a +// private snapshot. It must be reset after removing or renaming staged paths. +type DestinationValidator struct { + directory string + checked map[string]bool + names map[string]map[string]bool +} + +// NewDestinationValidator creates a validator for an exclusively owned stage. +func NewDestinationValidator(directory string) *DestinationValidator { + return &DestinationValidator{directory: directory, checked: make(map[string]bool), names: make(map[string]map[string]bool)} +} + +// Validate rejects host aliases without rereading unchanged parent directories. +func (v *DestinationValidator) Validate(name string) error { + return validateDestination(v.directory, name, v.checked, v.names) +} + +// Record remembers a successfully created path after Validate has accepted it. +// The single snapshot writer must record every creation before the next check. +func (v *DestinationValidator) Record(name string) { + if name == "." || name == "" { + return + } + current := v.directory + for _, component := range strings.Split(filepath.FromSlash(name), string(filepath.Separator)) { + if names := v.names[current]; names != nil { + names[component] = true + } + current = filepath.Join(current, component) + v.checked[current] = true + } +} + +// Reset discards prefixes whose directories may have been removed or renamed. +func (v *DestinationValidator) Reset() { + clear(v.checked) + clear(v.names) +} + +func validateDestination(directory, name string, checked map[string]bool, directories map[string]map[string]bool) error { if name == "." || name == "" { return nil } current := directory for _, component := range strings.Split(filepath.FromSlash(name), string(filepath.Separator)) { requested := filepath.Join(current, component) + if checked[requested] { + current = requested + continue + } _, err := os.Lstat(requested) if errors.Is(err, os.ErrNotExist) { return nil @@ -27,17 +75,34 @@ func ValidateDestination(directory, name string) error { if err != nil { return fmt.Errorf("Lstat: %w", err) } - entries, err := os.ReadDir(current) - if err != nil { - return fmt.Errorf("ReadDir: %w", err) - } + names := directories[current] exact := false - for _, entry := range entries { - exact = exact || entry.Name() == component + if names == nil { + entries, err := os.ReadDir(current) + if err != nil { + return fmt.Errorf("ReadDir: %w", err) + } + if directories != nil { + names = make(map[string]bool, len(entries)) + } + for _, entry := range entries { + exact = exact || entry.Name() == component + if names != nil { + names[entry.Name()] = true + } + } + if directories != nil { + directories[current] = names + } + } else { + exact = names[component] } if !exact { return fmt.Errorf("%w: %q resolves to a different spelling below %q", ErrPathCollision, name, current) } + if checked != nil { + checked[requested] = true + } current = requested } return nil diff --git a/internal/adapters/gitsnapshot/destination_test.go b/internal/adapters/gitsnapshot/destination_test.go new file mode 100644 index 00000000..3de674e8 --- /dev/null +++ b/internal/adapters/gitsnapshot/destination_test.go @@ -0,0 +1,60 @@ +package gitsnapshot + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDestinationValidatorRetainsHostCollisionChecks(t *testing.T) { + t.Parallel() + directory := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(directory, "Case", "api"), 0o755)) + validator := NewDestinationValidator(directory) + require.NoError(t, validator.Validate("Case/api/first.proto")) + require.NoError(t, validator.Validate("Case/api/second.proto")) + _, caseErr := os.Stat(filepath.Join(directory, "case")) + err := validator.Validate("case/api/other.proto") + if caseErr == nil { + require.ErrorIs(t, err, ErrPathCollision) + } else { + require.NoError(t, err) + } + require.NoError(t, os.RemoveAll(filepath.Join(directory, "Case"))) + require.NoError(t, os.MkdirAll(filepath.Join(directory, "case", "api"), 0o755)) + validator.Reset() + err = validator.Validate("Case/api/other.proto") + if caseErr == nil { + require.ErrorIs(t, err, ErrPathCollision) + } else { + require.NoError(t, err) + } +} + +func BenchmarkDestinationValidation(b *testing.B) { + directory := b.TempDir() + for i := range 1000 { + require.NoError(b, os.MkdirAll(filepath.Join(directory, "google", fmt.Sprintf("api%04d", i), "v1"), 0o755)) + } + for _, name := range []string{"stateless", "snapshot"} { + b.Run(name, func(b *testing.B) { + validator := NewDestinationValidator(directory) + b.ReportAllocs() + for i := range b.N { + path := fmt.Sprintf("google/api%04d/v1/file.proto", i%1000) + var err error + if name == "stateless" { + err = ValidateDestination(directory, path) + } else { + err = validator.Validate(path) + } + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/internal/adapters/go_git/snapshot.go b/internal/adapters/go_git/snapshot.go index 61140eda..772c1974 100644 --- a/internal/adapters/go_git/snapshot.go +++ b/internal/adapters/go_git/snapshot.go @@ -79,7 +79,7 @@ func SnapshotRevision(ctx context.Context, directory, ref string) (_ *Snapshot, _ = snapshot.Close() } }() - view := sourceview.New(tree) + view := sourceview.NewImmutable(tree) err = view.Walk(ctx, ".", func(name string, resolved sourceview.Resolution, walkErr error) error { if walkErr != nil { if !snapshotInput(name) || snapshotAdditionalPolicy(name) { diff --git a/internal/core/core.go b/internal/core/core.go index 316b3830..531d18f6 100644 --- a/internal/core/core.go +++ b/internal/core/core.go @@ -26,6 +26,7 @@ type Core struct { importRoots []string importFileAllowed func(string) bool sourceFileOpen func(string) (io.ReadCloser, error) + sourceFilesWalk func(string, func(string) error) (bool, error) sourceBoundary string fileModules map[string]string managedMode ManagedModeConfig @@ -62,7 +63,11 @@ type Options struct { // files selected by dependency metadata. Nil permits every file. ImportFileAllowed func(string) bool // OpenSourceFile opens logical sources within their owning bounded roots. - OpenSourceFile func(string) (io.ReadCloser, error) + OpenSourceFile func(string) (io.ReadCloser, error) + // WalkSourceFiles optionally visits verified immutable proto inputs under an + // absolute import root. Paths passed to the visitor are absolute. Returning + // false selects the normal local filesystem walk instead. + WalkSourceFiles func(string, func(string) error) (bool, error) FileModules map[string]string CurrentProjectGitWalker CurrentProjectGitWalker BreakingCheckConfig BreakingCheckConfig @@ -88,6 +93,7 @@ func New(options Options) *Core { importRoots: slices.Clone(options.ImportRoots), importFileAllowed: options.ImportFileAllowed, sourceFileOpen: options.OpenSourceFile, + sourceFilesWalk: options.WalkSourceFiles, fileModules: maps.Clone(options.FileModules), currentProjectGitWalker: options.CurrentProjectGitWalker, breakingCheckConfig: options.BreakingCheckConfig, diff --git a/internal/core/generate.go b/internal/core/generate.go index 5969d249..cda73678 100644 --- a/internal/core/generate.go +++ b/internal/core/generate.go @@ -25,7 +25,6 @@ import ( pluginexecutor "github.com/easyp-tech/easyp/internal/adapters/plugin" "github.com/easyp-tech/easyp/internal/core/path_helpers" - "github.com/easyp-tech/easyp/internal/fs/fs" "github.com/easyp-tech/easyp/internal/sourceview" "github.com/easyp-tech/easyp/internal/version" ) @@ -85,14 +84,15 @@ func (c *Core) prepareGeneration(ctx context.Context, root string, selected []st found := make(map[string]bool) for _, inputFilesDir := range c.inputs.InputFilesDir { - searchPath := filepath.Join(inputFilesDir.Root, inputFilesDir.Path) - fsWalker := fs.NewFSWalker(root, searchPath) importRoot := filepath.Join(root, inputFilesDir.Root) if !slices.Contains(imports, importRoot) { imports = append(imports, importRoot) } - err := fsWalker.WalkDir(func(walkPath string, err error) error { + err := c.walkGenerationInputs(root, inputFilesDir, func(walkPath string, err error, indexed bool) error { + if err := ctx.Err(); err != nil { + return err + } addedFile := stripPrefix(walkPath, inputFilesDir.Root) if selected != nil && filepath.Ext(walkPath) == ".proto" && !selection[addedFile] { return nil @@ -108,7 +108,7 @@ func (c *Core) prepareGeneration(ctx context.Context, root string, selected []st return err case ctx.Err() != nil: return ctx.Err() - case path_helpers.ShouldSkipV1SourceDir(importRoot, filepath.Join(root, walkPath)): + case !indexed && path_helpers.ShouldSkipV1SourceDir(importRoot, filepath.Join(root, walkPath)): return stdfs.SkipDir case filepath.Ext(walkPath) != ".proto": return nil @@ -130,7 +130,7 @@ func (c *Core) prepareGeneration(ctx context.Context, root string, selected []st return nil }) if err != nil { - return nil, fmt.Errorf("WalkDir: %w", err) + return nil, fmt.Errorf("walkGenerationInputs: %w", err) } } diff --git a/internal/core/generate_inputs.go b/internal/core/generate_inputs.go new file mode 100644 index 00000000..33701353 --- /dev/null +++ b/internal/core/generate_inputs.go @@ -0,0 +1,40 @@ +package core + +import ( + "fmt" + "path/filepath" + + "github.com/easyp-tech/easyp/internal/fs/fs" +) + +func (c *Core) walkGenerationInputs(root string, input InputFilesDir, visit func(string, error, bool) error) error { + searchPath := filepath.Join(input.Root, input.Path) + if c.sourceFilesWalk != nil { + indexed, err := c.sourceFilesWalk(filepath.Join(root, input.Root), func(path string) error { + within, err := filepath.Rel(filepath.Join(root, searchPath), path) + if err != nil { + return fmt.Errorf("Rel: %w", err) + } + if !filepath.IsLocal(within) { + return nil + } + relative, err := filepath.Rel(root, path) + if err != nil { + return fmt.Errorf("Rel: %w", err) + } + if !filepath.IsLocal(relative) { + return fmt.Errorf("indexed source %q is outside the module", path) + } + return visit(filepath.ToSlash(relative), nil, true) + }) + if err != nil { + return fmt.Errorf("WalkSourceFiles: %w", err) + } + if indexed { + return nil + } + } + return fs.NewFSWalker(root, searchPath).WalkDir(func(path string, err error) error { + return visit(path, err, false) + }) +} diff --git a/internal/core/generate_inputs_test.go b/internal/core/generate_inputs_test.go new file mode 100644 index 00000000..e09e6a71 --- /dev/null +++ b/internal/core/generate_inputs_test.go @@ -0,0 +1,86 @@ +package core + +import ( + "io" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/easyp-tech/easyp/internal/logger" +) + +func TestPrepareGenerationUsesVerifiedInventory(t *testing.T) { + t.Parallel() + root := t.TempDir() + contents := map[string]string{ + filepath.Join(root, "proto/api/item.proto"): "syntax = \"proto3\"; package api; import \"shared.proto\"; message Item { shared.Shared value = 1; }", + filepath.Join(root, "proto/shared.proto"): "syntax = \"proto3\"; package shared; message Shared {}", + filepath.Join(root, "proto/api/unused.proto"): "invalid unselected protobuf", + } + var unusedChecks atomic.Int64 + walks := 0 + app := New(Options{ + Logger: logger.NewNop(), + Inputs: Inputs{InputFilesDir: []InputFilesDir{{Root: "proto", Path: "api"}}}, + WalkSourceFiles: func(importRoot string, visit func(string) error) (bool, error) { + assert.Equal(t, filepath.Join(root, "proto"), importRoot) + walks++ + for _, name := range []string{"api/item.proto", "api/unused.proto", "shared.proto"} { + if err := visit(filepath.Join(importRoot, name)); err != nil { + return true, err + } + } + return true, nil + }, + ImportFileAllowed: func(path string) bool { + if filepath.Base(path) == "unused.proto" { + unusedChecks.Add(1) + } + _, ok := contents[filepath.Clean(path)] + return ok + }, + OpenSourceFile: func(path string) (io.ReadCloser, error) { + body, ok := contents[filepath.Clean(path)] + if !ok { + return nil, os.ErrNotExist + } + return io.NopCloser(strings.NewReader(body)), nil + }, + }) + // No source directories exist on disk. Both input discovery and imports + // must use the configured snapshot inventory and source accessor. + plan, err := app.PrepareGenerationFiles(t.Context(), root, []string{"api/item.proto"}) + require.NoError(t, err) + assert.Equal(t, 1, walks) + assert.Zero(t, unusedChecks.Load(), "selection runs before file metadata or membership checks") + assert.ElementsMatch(t, []string{"api/item.proto", "shared.proto"}, descriptorNames(plan.DescriptorSet(true).File)) + assert.Equal(t, []string{"api/item.proto"}, descriptorNames(plan.DescriptorSet(false).File)) + _, err = app.PrepareGenerationFiles(t.Context(), root, []string{"shared.proto"}) + require.ErrorContains(t, err, "outside the module inputs") +} + +func TestPrepareGenerationWithoutInventoryKeepsLocalSourcesFresh(t *testing.T) { + t.Parallel() + root := t.TempDir() + writeGenerationSources(t, root, map[string]string{"first.proto": "syntax = \"proto3\"; message First {}"}) + app := New(Options{ + Logger: logger.NewNop(), + Inputs: Inputs{InputFilesDir: []InputFilesDir{{Root: ".", Path: "."}}}, + WalkSourceFiles: func(string, func(string) error) (bool, error) { + return false, nil + }, + }) + first, err := app.PrepareGeneration(t.Context(), root) + require.NoError(t, err) + assert.Equal(t, []string{"first.proto"}, descriptorNames(first.DescriptorSet(false).File)) + require.NoError(t, os.Remove(filepath.Join(root, "first.proto"))) + writeGenerationSources(t, root, map[string]string{"second.proto": "syntax = \"proto3\"; message Second {}"}) + second, err := app.PrepareGeneration(t.Context(), root) + require.NoError(t, err) + assert.Equal(t, []string{"second.proto"}, descriptorNames(second.DescriptorSet(false).File)) +} diff --git a/internal/generation/descriptor_set.go b/internal/generation/descriptor_set.go index 9893414c..98215f03 100644 --- a/internal/generation/descriptor_set.go +++ b/internal/generation/descriptor_set.go @@ -99,7 +99,7 @@ func prepareDescriptorTargets(ctx context.Context, log logger.Logger, cache modu } } label := fmt.Sprintf("project %q, module %q (%s)", relativeDescriptorPath(request.WorkDir, filepath.Dir(target.configPath)), selected.module.Name, relativeDescriptorPath(request.WorkDir, selected.directory)) - app, err := prepareV1ModuleCore(log, request, target.configPath, selected.directory, target.config, selected.module, selected.dependencies) + app, owners, err := prepareV1ModuleCore(log, request, target.configPath, selected.directory, target.config, selected.module, selected.dependencies, selected.sources) if err != nil { return nil, fmt.Errorf("prepareV1ModuleCore for %s: %w", label, err) } @@ -116,15 +116,6 @@ func prepareDescriptorTargets(ctx context.Context, log logger.Logger, cache modu if _, err := protodesc.NewFiles(full); err != nil { return nil, fmt.Errorf("validate descriptor set for %s: %w", label, err) } - roots, err := modules.ModuleSources(selected.directory, selected.module) - if err != nil { - return nil, fmt.Errorf("ModuleSources: %w", err) - } - roots = append(roots, selected.dependencies...) - owners, err := roots.FileModules() - if err != nil { - return nil, fmt.Errorf("FileModules: %w", err) - } versions := descriptorVersions(request, selected) prepared = append(prepared, preparedDescriptorTarget{target: target, selected: selected, plan: plan, full: full, owners: owners, versions: versions, label: label}) } diff --git a/internal/generation/frozen.go b/internal/generation/frozen.go index a5882bc6..3aeb9f88 100644 --- a/internal/generation/frozen.go +++ b/internal/generation/frozen.go @@ -50,13 +50,15 @@ func resolveFrozenGenerationModule(ctx context.Context, cache modules.Cache, rep if err != nil { return v1GenerationModule{}, fmt.Errorf("Cached: %w", err) } - var others modules.SourceRoots + var sources, others modules.SourceRoots for _, root := range consumer.dependencies { if root.Module != selection.source { others = append(others, root) + } else { + sources = append(sources, root) } } - return v1GenerationModule{directory: directory, resolutionDir: consumerDir, module: module, dependencies: others}, nil + return v1GenerationModule{directory: directory, resolutionDir: consumerDir, module: module, sources: sources, dependencies: others}, nil } localDir, err := findV1LocalModuleByName(repoRoot, selection.source) if err != nil { diff --git a/internal/generation/module.go b/internal/generation/module.go index 2c554562..794b1d86 100644 --- a/internal/generation/module.go +++ b/internal/generation/module.go @@ -11,7 +11,7 @@ import ( ) func generateV1ModuleWithRoots(ctx context.Context, log logger.Logger, request Request, configPath, moduleDir string, gen v1.Generate, module v1.Module, importRoots modules.SourceRoots) error { - app, err := prepareV1ModuleCore(log, request, configPath, moduleDir, gen, module, importRoots) + app, _, err := prepareV1ModuleCore(log, request, configPath, moduleDir, gen, module, importRoots, nil) if err != nil { return fmt.Errorf("prepareV1ModuleCore: %w", err) } @@ -21,31 +21,34 @@ func generateV1ModuleWithRoots(ctx context.Context, log logger.Logger, request R return nil } -func prepareV1ModuleCore(log logger.Logger, request Request, configPath, moduleDir string, gen v1.Generate, module v1.Module, importRoots modules.SourceRoots) (*core.Core, error) { - sources, err := modules.ModuleSources(moduleDir, module) - if err != nil { - return nil, fmt.Errorf("ModuleSources: %w", err) +func prepareV1ModuleCore(log logger.Logger, request Request, configPath, moduleDir string, gen v1.Generate, module v1.Module, importRoots, sources modules.SourceRoots) (*core.Core, map[string]string, error) { + if sources == nil { + var err error + sources, err = modules.ModuleSources(moduleDir, module) + if err != nil { + return nil, nil, fmt.Errorf("ModuleSources: %w", err) + } } allSources := append(append(modules.SourceRoots(nil), sources...), importRoots...) - if err := modules.CheckSourceCollisions(allSources); err != nil { - return nil, fmt.Errorf("CheckSourceCollisions: %w", err) + // The consumer retains ownership when the same physical root is also + // present in its dependency graph. Indexing also validates every import. + indexRoots := append(append(modules.SourceRoots(nil), importRoots...), sources...) + owners, err := indexRoots.IndexFileModules() + if err != nil { + return nil, nil, fmt.Errorf("IndexFileModules: %w", err) } options, err := prepareV1GeneratorConfig(configPath, moduleDir, gen, module) if err != nil { - return nil, fmt.Errorf("prepareV1GeneratorConfig: %w", err) + return nil, nil, fmt.Errorf("prepareV1GeneratorConfig: %w", err) } options.Logger = log options.PluginWorkDir = request.WorkDir options.ImportRoots = importRoots.Paths() options.ImportFileAllowed = allSources.FileAllowed() options.OpenSourceFile = allSources.OpenSourceFile + options.WalkSourceFiles = sources.WalkSnapshotFiles if options.ManagedModeConfig.Enabled || options.ManagedModeConfig.GoPackageOnly { - roots := append(modules.SourceRoots(nil), importRoots...) - roots = append(roots, sources...) - options.FileModules, err = roots.FileModules() - if err != nil { - return nil, fmt.Errorf("FileModules: %w", err) - } + options.FileModules = owners } - return core.New(options), nil + return core.New(options), owners, nil } diff --git a/internal/generation/packages.go b/internal/generation/packages.go index a7150071..0d6fcaae 100644 --- a/internal/generation/packages.go +++ b/internal/generation/packages.go @@ -56,9 +56,13 @@ func (s *sourceSelectorMatches) validateMatches(section string) error { // names without compiling unrelated files. Package declarations are read only // when needed and after the path filter; required imports are compiled later. func selectedSourceFiles(ctx context.Context, selected v1GenerationModule, project, module *sourceSelectorMatches) ([]string, error) { - roots, err := modules.ModuleSources(selected.directory, selected.module) - if err != nil { - return nil, fmt.Errorf("ModuleSources: %w", err) + roots := selected.sources + if roots == nil { + var err error + roots, err = modules.ModuleSources(selected.directory, selected.module) + if err != nil { + return nil, fmt.Errorf("ModuleSources: %w", err) + } } allRoots := append(append(modules.SourceRoots(nil), roots...), selected.dependencies...) seen := make(map[string]bool) diff --git a/internal/generation/selection.go b/internal/generation/selection.go index 95fdda75..a7e95e8a 100644 --- a/internal/generation/selection.go +++ b/internal/generation/selection.go @@ -91,6 +91,7 @@ type v1GenerationModule struct { resolutionDir string directory string module v1.Module + sources modules.SourceRoots dependencies modules.SourceRoots } @@ -134,13 +135,15 @@ func resolveV1GenerationModule(ctx context.Context, cache modules.Cache, repoRoo return v1GenerationModule{}, fmt.Errorf("EnsureEffectiveGraph: %w", err) } if selected, ok := graph.Modules[selection.source]; ok { - var others modules.SourceRoots + var sources, others modules.SourceRoots for _, root := range graph.Sources { if root.Module != selection.source { others = append(others, root) + } else { + sources = append(sources, root) } } - return v1GenerationModule{directory: selected.Directory, resolutionDir: consumerDir, module: selected.Module, dependencies: others}, nil + return v1GenerationModule{directory: selected.Directory, resolutionDir: consumerDir, module: selected.Module, sources: sources, dependencies: others}, nil } for _, replacement := range consumer.Replaces { if replacement.Module == selection.source { @@ -180,12 +183,15 @@ func resolveV1GenerationModule(ctx context.Context, cache modules.Cache, repoRoo } otherRoots := make(modules.SourceRoots, 0, len(dependencyRoots)) + var sources modules.SourceRoots for _, root := range dependencyRoots { if root.Module != selection.source { otherRoots = append(otherRoots, root) + } else { + sources = append(sources, root) } } - return v1GenerationModule{directory: moduleDir, resolutionDir: consumerDir, module: module, dependencies: otherRoots}, nil + return v1GenerationModule{directory: moduleDir, resolutionDir: consumerDir, module: module, sources: sources, dependencies: otherRoots}, nil } func readV1GenerationModule(ctx context.Context, cache modules.Cache, directory string) (v1GenerationModule, error) { diff --git a/internal/generation/source_index_test.go b/internal/generation/source_index_test.go new file mode 100644 index 00000000..0313f2c0 --- /dev/null +++ b/internal/generation/source_index_test.go @@ -0,0 +1,82 @@ +package generation + +import ( + "context" + "fmt" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" + "github.com/easyp-tech/easyp/internal/logger" +) + +func TestGenerationSelectsIndexedDependencyInputs(t *testing.T) { + t.Parallel() + root, snapshot := t.TempDir(), t.TempDir() + const source = "example.test/dependency" + files := map[string]string{ + v1.ModuleFile: "module " + source + "\nroots proto\n", + "proto/api/item.proto": "syntax = \"proto3\"; package api; import \"shared.proto\"; message Item { shared.Shared value = 1; }", + "proto/api/unused.proto": "invalid unselected protobuf", + "proto/shared.proto": "syntax = \"proto3\"; package shared; message Shared {}", + "proto/nested/protobuf.mod": "module example.test/nested\n", + "proto/nested/ignored.proto": "invalid nested module protobuf", + } + var names []string + for name, body := range files { + writeV1GenerateFixture(t, snapshot, name, body) + names = append(names, name) + } + slices.Sort(names) + writeV1GenerateFixture(t, root, v1.ModuleFile, "module example.test/consumer\nrequire "+source+" v1.0.0\n") + writeV1GenerateFixture(t, root, v1.GenerateFile, `version: v1 +generate: + modules: + - module: example.test/dependency + paths: [proto/api/item.proto] + managed: + enabled: true + override: + - module: example.test/dependency + file_option: go_package_prefix + value: example.test/generated +`) + entry := v1.LockedModule{Source: source, Version: "v1.0.0", Commit: strings.Repeat("a", 40), Hash: "h1:" + strings.Repeat("A", 43) + "="} + require.NoError(t, writeTestLock(root, v1.Lock{Version: 1, Modules: []v1.LockedModule{entry}})) + cache := &indexedGenerationCache{directory: snapshot, module: v1.Module{Name: source, Roots: []string{"proto"}}, files: names} + output := filepath.Join(root, "selected.pb") + err := Run(t.Context(), logger.NewNop(), cache, Request{WorkDir: root, WorkspaceRoot: root, DescriptorSetOut: output, IncludeImports: true}) + require.NoError(t, err) + assert.Equal(t, 1, cache.inventoryReads) + descriptors := readDescriptorSet(t, output) + assert.ElementsMatch(t, []string{"api/item.proto", "shared.proto"}, descriptorNames(descriptors)) + for _, file := range descriptors.File { + assert.Contains(t, file.GetOptions().GetGoPackage(), "example.test/generated") + } +} + +type indexedGenerationCache struct { + directory string + module v1.Module + files []string + inventoryReads int +} + +func (c *indexedGenerationCache) Install(context.Context, v1.Lock) error { return nil } + +func (c *indexedGenerationCache) Cached(entry v1.LockedModule) (string, v1.Module, error) { + if entry.Source != c.module.Name { + return "", v1.Module{}, fmt.Errorf("unknown cached module %s", entry.Source) + } + return c.directory, c.module, nil +} + +func (c *indexedGenerationCache) SourceFiles(v1.LockedModule) ([]string, error) { + c.inventoryReads++ + return slices.Clone(c.files), nil +} diff --git a/internal/modules/collisions.go b/internal/modules/collisions.go index 2d389081..ccbfce11 100644 --- a/internal/modules/collisions.go +++ b/internal/modules/collisions.go @@ -7,27 +7,37 @@ import ( // CheckSourceCollisions respects each module's source selection when comparing import names. func CheckSourceCollisions(roots SourceRoots) error { + _, err := roots.indexFiles(true) + return err +} + +func (roots SourceRoots) indexFiles(checkCollisions bool) (map[string]string, error) { seen := make(map[string]string) + owners := make(map[string]string) allowed := roots.FileAllowed() for _, root := range roots { err := root.walk(allowed, true, roots.ownsSelectedPhysical, func(path string) error { - if allowed != nil && !allowed(path) { - return nil - } importPath, err := filepath.Rel(root.Path, path) if err != nil { return err } importPath = filepath.ToSlash(importPath) - if previous, ok := seen[importPath]; ok && previous != path { + if previous, ok := seen[importPath]; checkCollisions && ok && previous != path { return fmt.Errorf("duplicate import path %q: %s and %s", importPath, previous, path) } seen[importPath] = path + owners[importPath] = root.Module return nil }) if err != nil { - return err + return nil, err } } - return nil + return owners, nil +} + +// IndexFileModules checks import collisions and records module ownership in a +// single source traversal. The returned map can be shared by generation stages. +func (roots SourceRoots) IndexFileModules() (map[string]string, error) { + return roots.indexFiles(true) } diff --git a/internal/modules/effective_graph.go b/internal/modules/effective_graph.go index 79635335..78c5a14e 100644 --- a/internal/modules/effective_graph.go +++ b/internal/modules/effective_graph.go @@ -63,8 +63,14 @@ func ensureEffectiveGraphWithRoots(ctx context.Context, root string, module v1.M if err != nil { return graph, fmt.Errorf("finalizeEffectiveImportRoots: %w", err) } - appendModule := func(name string, entry EffectiveModule) error { - roots, err := ModuleSources(entry.Directory, entry.Module) + appendModule := func(name string, entry EffectiveModule, locked *v1.LockedModule) error { + var roots SourceRoots + var err error + if locked == nil { + roots, err = ModuleSources(entry.Directory, entry.Module) + } else { + roots, err = cachedModuleSources(entry.Directory, entry.Module, *locked, cache) + } if err != nil { return fmt.Errorf("ModuleSources: %s: %w", name, err) } @@ -73,7 +79,7 @@ func ensureEffectiveGraphWithRoots(ctx context.Context, root string, module v1.M return nil } for _, name := range pass.locals { - if err := appendModule(name, locals.modules[name]); err != nil { + if err := appendModule(name, locals.modules[name], nil); err != nil { return graph, err } } @@ -82,7 +88,7 @@ func ensureEffectiveGraphWithRoots(ctx context.Context, root string, module v1.M if err != nil { return graph, fmt.Errorf("installed: %w", err) } - if err := appendModule(entry.Source, selected); err != nil { + if err := appendModule(entry.Source, selected, &entry); err != nil { return graph, err } } diff --git a/internal/modules/frozen.go b/internal/modules/frozen.go index d1fcf12b..d765f80d 100644 --- a/internal/modules/frozen.go +++ b/internal/modules/frozen.go @@ -89,9 +89,9 @@ func frozenCachedSources(ctx context.Context, module v1.Module, lock v1.Lock, re if err != nil { return nil, fmt.Errorf("dependency %s: validateFrozenRequirements: %w", dependency.Name, err) } - dependencyRoots, err := ModuleSources(directory, dependency) + dependencyRoots, err := cachedModuleSources(directory, dependency, entry, repository) if err != nil { - return nil, fmt.Errorf("ModuleSources: %w", err) + return nil, fmt.Errorf("cachedModuleSources: %w", err) } sources[requirement.Module] = dependencyRoots queue = append(queue, dependency.Requires...) diff --git a/internal/modules/locked_sources.go b/internal/modules/locked_sources.go index 1115cb6f..2182a39a 100644 --- a/internal/modules/locked_sources.go +++ b/internal/modules/locked_sources.go @@ -102,9 +102,9 @@ func cachedSources(lock v1.Lock, replaced map[string]bool, repository Cache) (So if err := ValidateRequirements(unreplacedRequirements(dependency.Requires, replaced), lock); err != nil { return nil, fmt.Errorf("dependency %s: %w", dependency.Name, err) } - dependencyRoots, err := ModuleSources(installDir, dependency) + dependencyRoots, err := cachedModuleSources(installDir, dependency, entry, repository) if err != nil { - return nil, fmt.Errorf("ModuleSources: %w", err) + return nil, fmt.Errorf("cachedModuleSources: %w", err) } roots = append(roots, dependencyRoots...) } diff --git a/internal/modules/operations_test.go b/internal/modules/operations_test.go index 941e9d5e..1f8338ef 100644 --- a/internal/modules/operations_test.go +++ b/internal/modules/operations_test.go @@ -241,6 +241,9 @@ func (r *fakeRepository) Install(context.Context, v1.Lock) error { r.installed = true return r.installErr } +func (r *fakeRepository) VerifyCached(context.Context, v1.Lock) error { + return r.installErr +} func (r *fakeRepository) Cached(v1.LockedModule) (string, v1.Module, error) { return r.directory, r.module, r.cachedErr } diff --git a/internal/modules/source_index.go b/internal/modules/source_index.go new file mode 100644 index 00000000..6e0170c1 --- /dev/null +++ b/internal/modules/source_index.go @@ -0,0 +1,125 @@ +package modules + +import ( + "fmt" + "path/filepath" + "slices" + "strings" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" + "github.com/easyp-tech/easyp/internal/core/path_helpers" +) + +// snapshotFileSource exposes names captured while verifying installed contents. +// Names are relative to the installed module directory and include metadata. +// Local sources never implement or use this immutable snapshot capability. +type snapshotFileSource interface { + SourceFiles(v1.LockedModule) ([]string, error) +} + +type sourceFileIndex struct { + paths []string + names map[string]bool + physicalRoot string +} + +// WalkSnapshotFiles visits the verified regular proto inventory for importRoot. +// It returns false without visiting anything when that root is a local source +// or its cache did not supply an immutable inventory. +func (roots SourceRoots) WalkSnapshotFiles(importRoot string, visit func(string) error) (bool, error) { + for _, root := range roots { + if root.index == nil || filepath.Clean(root.Path) != filepath.Clean(importRoot) { + continue + } + for _, path := range root.index.paths { + if err := visit(path); err != nil { + return true, err + } + } + return true, nil + } + return false, nil +} + +func cachedModuleSources(directory string, module v1.Module, entry v1.LockedModule, repository Cache) (SourceRoots, error) { + roots, err := ModuleSources(directory, module) + if err != nil { + return nil, fmt.Errorf("ModuleSources: %w", err) + } + indexed, ok := repository.(snapshotFileSource) + if !ok { + return roots, nil + } + files, err := indexed.SourceFiles(entry) + if err != nil { + return nil, fmt.Errorf("SourceFiles: %w", err) + } + if files == nil { + return roots, nil + } + return indexSnapshotSources(directory, roots, files) +} + +func indexSnapshotSources(directory string, roots SourceRoots, files []string) (SourceRoots, error) { + boundaries := make(map[string]bool) + var protos []string + for _, name := range files { + name = filepath.FromSlash(name) + if !filepath.IsLocal(name) || filepath.Clean(name) != name || name == "." { + return nil, fmt.Errorf("invalid snapshot file %q", name) + } + path := filepath.Join(directory, name) + switch filepath.Base(name) { + case ".git", v1.ModuleFile, "buf.work.yaml", "buf.yaml": + boundaries[filepath.Dir(path)] = true + } + if filepath.Ext(name) == ".proto" { + protos = append(protos, path) + } + } + slices.SortFunc(protos, compareSourcePaths) + for index := range roots { + root := &roots[index] + catalog := &sourceFileIndex{names: make(map[string]bool), physicalRoot: physicalSourcePath(root.Path)} + for _, path := range protos { + if !sourcePathWithin(path, root.Path) || path_helpers.HiddenOrVendorSourcePath(root.Path, filepath.Dir(path)) || + (root.fileAllowed != nil && !root.fileAllowed(path)) { + continue + } + nested := false + for parent := filepath.Dir(path); parent != filepath.Clean(root.Path); parent = filepath.Dir(parent) { + if boundaries[parent] { + nested = true + break + } + } + if nested { + continue + } + name, err := filepath.Rel(root.Path, path) + if err != nil { + return nil, fmt.Errorf("Rel: %w", err) + } + catalog.paths = append(catalog.paths, path) + catalog.names[name] = true + } + root.index = catalog + } + return roots, nil +} + +// Directory walks sort each component, so a/child.proto precedes a.proto. +// Preserve that order when an immutable inventory replaces filesystem walks. +func compareSourcePaths(first, second string) int { + for { + a, remainingA, nestedA := strings.Cut(first, string(filepath.Separator)) + b, remainingB, nestedB := strings.Cut(second, string(filepath.Separator)) + if compared := strings.Compare(a, b); compared != 0 { + return compared + } + if !nestedA || !nestedB { + return strings.Compare(remainingA, remainingB) + } + first, second = remainingA, remainingB + } +} diff --git a/internal/modules/source_index_test.go b/internal/modules/source_index_test.go new file mode 100644 index 00000000..fe634bdb --- /dev/null +++ b/internal/modules/source_index_test.go @@ -0,0 +1,137 @@ +package modules + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" +) + +func TestSnapshotSourceIndexPreservesSelection(t *testing.T) { + t.Parallel() + files := map[string]string{ + v1.ModuleFile: "module example.test/dependency\n", + "proto/api/model.proto": "syntax = \"proto3\";\n", + "proto/api.proto": "syntax = \"proto3\";\n", + "proto/excluded/model.proto": "syntax = \"proto3\";\n", + "proto/.hidden.proto": "syntax = \"proto3\";\n", + "proto/.hidden/ignored.proto": "syntax = \"proto3\";\n", + "proto/easyp_vendor/ignored.proto": "syntax = \"proto3\";\n", + "proto/native/protobuf.mod": "module example.test/nested\n", + "proto/native/ignored.proto": "syntax = \"proto3\";\n", + "proto/buf/buf.yaml": "version: v1\n", + "proto/buf/ignored.proto": "syntax = \"proto3\";\n", + "proto/work/buf.work.yaml": "version: v1\ndirectories: [nested]\n", + "proto/work/nested/ignored.proto": "syntax = \"proto3\";\n", + "proto/repository/.git": "gitdir: /unused\n", + "proto/repository/ignored.proto": "syntax = \"proto3\";\n", + "other/shared.proto": "syntax = \"proto3\";\n", + } + tests := []struct { + name string + roots []string + filters []v1.ProtoFileFilter + }{ + {name: "default root", roots: []string{"."}}, + {name: "multiple roots", roots: []string{"proto", "other"}}, + {name: "explicit hidden root", roots: []string{"proto/.hidden"}}, + {name: "explicit nested module root", roots: []string{"proto/buf"}}, + {name: "filtered root", roots: []string{"proto"}, filters: []v1.ProtoFileFilter{{Root: "proto", Includes: []string{"proto/api", "proto/.hidden.proto", "proto/excluded"}, Excludes: []string{"proto/excluded"}}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + directory := t.TempDir() + var names []string + for name, body := range files { + writeV1GenerateFixture(t, directory, name, body) + names = append(names, name) + } + slices.Sort(names) + module := v1.Module{Name: "example.test/dependency", Roots: tt.roots, ProtoFilters: tt.filters} + local, err := ModuleSources(directory, module) + require.NoError(t, err) + want, err := local.IndexFileModules() + require.NoError(t, err) + indexed, err := ModuleSources(directory, module) + require.NoError(t, err) + indexed, err = indexSnapshotSources(directory, indexed, names) + require.NoError(t, err) + got, err := indexed.IndexFileModules() + require.NoError(t, err) + assert.Equal(t, want, got) + for index, root := range local { + var localOrder, indexedOrder []string + require.NoError(t, root.Walk(func(path string) error { localOrder = append(localOrder, path); return nil })) + require.NoError(t, indexed[index].Walk(func(path string) error { indexedOrder = append(indexedOrder, path); return nil })) + assert.Equal(t, localOrder, indexedOrder, "snapshot traversal keeps directory-walk order") + } + localAllowed, indexedAllowed := local.FileAllowed(), indexed.FileAllowed() + for _, root := range local { + for _, name := range names { + path := filepath.Join(directory, filepath.FromSlash(name)) + if sourcePathWithin(path, root.Path) && filepath.Ext(path) == ".proto" { + assert.Equal(t, localAllowed(path), indexedAllowed(path), name) + } + } + } + }) + } +} + +func TestSnapshotSourceIndexReusesVerifiedInventory(t *testing.T) { + t.Parallel() + directory := t.TempDir() + writeV1GenerateFixture(t, directory, "api/one.proto", "syntax = \"proto3\";\n") + module := v1.Module{Name: "example.test/dependency", Roots: []string{"."}} + roots, err := ModuleSources(directory, module) + require.NoError(t, err) + roots, err = indexSnapshotSources(directory, roots, []string{"api/one.proto"}) + require.NoError(t, err) + // Make traversal impossible after indexing. Looking up ownership must use + // the inventory already supplied by the verified snapshot boundary. + require.NoError(t, os.Rename(filepath.Join(directory, "api"), filepath.Join(directory, "relocated"))) + for range 3 { + owners, err := roots.IndexFileModules() + require.NoError(t, err) + assert.Equal(t, map[string]string{"api/one.proto": module.Name}, owners) + } +} + +func TestLocalSourceIndexObservesChanges(t *testing.T) { + t.Parallel() + directory := t.TempDir() + writeV1GenerateFixture(t, directory, "one.proto", "syntax = \"proto3\";\n") + module := v1.Module{Name: "example.test/local", Roots: []string{"."}} + roots, err := ModuleSources(directory, module) + require.NoError(t, err) + before, err := roots.IndexFileModules() + require.NoError(t, err) + assert.Equal(t, map[string]string{"one.proto": module.Name}, before) + require.NoError(t, os.Remove(filepath.Join(directory, "one.proto"))) + writeV1GenerateFixture(t, directory, "two.proto", "syntax = \"proto3\";\n") + after, err := roots.IndexFileModules() + require.NoError(t, err) + assert.Equal(t, map[string]string{"two.proto": module.Name}, after) +} + +func TestSnapshotSourceIndexChecksUnusedCollisions(t *testing.T) { + t.Parallel() + var all SourceRoots + for _, module := range []string{"example.test/a", "example.test/b"} { + directory := t.TempDir() + writeV1GenerateFixture(t, directory, "unused.proto", "syntax = \"proto3\";\n") + roots, err := ModuleSources(directory, v1.Module{Name: module, Roots: []string{"."}}) + require.NoError(t, err) + roots, err = indexSnapshotSources(directory, roots, []string{"unused.proto"}) + require.NoError(t, err) + all = append(all, roots...) + } + _, err := all.IndexFileModules() + require.ErrorContains(t, err, "duplicate import path \"unused.proto\"") +} diff --git a/internal/modules/sources.go b/internal/modules/sources.go index 5e1f2c6c..70c8fb67 100644 --- a/internal/modules/sources.go +++ b/internal/modules/sources.go @@ -23,6 +23,7 @@ type SourceRoot struct { fileAllowed func(string) bool directoryAllowed func(string) bool directory string + index *sourceFileIndex } // SourceRoots preserves import precedence and module ownership. @@ -51,7 +52,19 @@ func (roots SourceRoots) WalkSelected(root SourceRoot, selected func(string) boo } func (root SourceRoot) walk(selected func(string) bool, namesOnly bool, targetAllowed func(string) bool, visit func(string) error) error { + if root.index != nil { + for _, path := range root.index.paths { + if selected != nil && !selected(path) { + continue + } + if err := visit(path); err != nil { + return err + } + } + return nil + } boundary := root.boundary() + physicalBoundary := physicalSourcePath(boundary) relative, err := filepath.Rel(boundary, root.Path) if err != nil { return fmt.Errorf("Rel: %w", err) @@ -74,13 +87,13 @@ func (root SourceRoot) walk(selected func(string) bool, namesOnly bool, targetAl if root.directoryAllowed != nil && resolved.Info != nil && resolved.Info.IsDir() && !root.directoryAllowed(path) { return fs.SkipDir } - if path_helpers.ShouldSkipV1SourceDir(root.Path, path) { + if resolved.Info != nil && resolved.Info.IsDir() && path_helpers.ShouldSkipV1SourceDir(root.Path, path) { return fs.SkipDir } if filepath.Ext(path) == ".proto" && selected != nil && !selected(path) { return nil } - if errors.Is(walkErr, sourceview.ErrNestedRepository) && targetAllowed != nil && targetAllowed(filepath.Join(physicalSourcePath(boundary), filepath.FromSlash(resolved.Path))) { + if errors.Is(walkErr, sourceview.ErrNestedRepository) && targetAllowed != nil && targetAllowed(filepath.Join(physicalBoundary, filepath.FromSlash(resolved.Path))) { walkErr = nil } if walkErr != nil && namesOnly && filepath.Ext(path) == ".proto" { @@ -92,12 +105,12 @@ func (root SourceRoot) walk(selected func(string) bool, namesOnly bool, targetAl } return walkErr } - physical := filepath.Join(physicalSourcePath(boundary), filepath.FromSlash(resolved.Path)) + physical := filepath.Join(physicalBoundary, filepath.FromSlash(resolved.Path)) if !resolved.Info.IsDir() && root.fileAllowed != nil && !root.fileAllowed(physical) { return nil } if resolved.Info.IsDir() { - if path_helpers.ShouldSkipV1SourceDir(filepath.Join(physicalSourcePath(boundary), filepath.FromSlash(physicalRoot.Path)), physical) { + if path_helpers.ShouldSkipV1SourceDir(filepath.Join(physicalBoundary, filepath.FromSlash(physicalRoot.Path)), physical) { return fs.SkipDir } return nil @@ -162,6 +175,10 @@ func (root SourceRoot) allows(path, importRoot string) bool { } func (root SourceRoot) allowsPath(path, importRoot string) bool { + if root.index != nil { + relative, err := filepath.Rel(importRoot, path) + return err == nil && root.index.names[relative] + } if root.fileAllowed != nil && !root.fileAllowed(path) { return false } @@ -188,8 +205,13 @@ func (roots SourceRoots) FileAllowed() func(string) bool { selections := make([]selection, 0, len(roots)) for _, root := range roots { selections = append(selections, selection{path: filepath.Clean(root.Path), root: root}) - canonical, err := filepath.EvalSymlinks(root.Path) - if err == nil && canonical != root.Path { + canonical := "" + if root.index != nil { + canonical = root.index.physicalRoot + } else if resolved, err := filepath.EvalSymlinks(root.Path); err == nil { + canonical = resolved + } + if canonical != "" && canonical != root.Path { selections = append(selections, selection{path: canonical, root: root}) } } @@ -207,7 +229,18 @@ func (roots SourceRoots) FileAllowed() func(string) bool { return !inside } return func(path string) bool { - return allowedPath(filepath.Clean(path)) && allowedPath(physicalSourcePath(path)) + path = filepath.Clean(path) + if !allowedPath(path) { + return false + } + for _, selection := range selections { + if selection.root.index != nil && sourcePathWithin(path, selection.path) { + // Installed snapshots materialize aliases as regular files. Their + // verified inventory cannot acquire new links during this run. + return true + } + } + return allowedPath(physicalSourcePath(path)) } } @@ -294,25 +327,7 @@ func moduleFileSelection(directory string, filters []v1.ProtoFileFilter, include // FileModules maps import paths to their owning modules for managed selectors. // Callers check import collisions before using this mapping. func (roots SourceRoots) FileModules() (map[string]string, error) { - modules := make(map[string]string) - allowed := roots.FileAllowed() - for _, root := range roots { - err := root.walk(allowed, true, roots.ownsSelectedPhysical, func(path string) error { - if allowed != nil && !allowed(path) { - return nil - } - relative, err := filepath.Rel(root.Path, path) - if err != nil { - return err - } - modules[filepath.ToSlash(relative)] = root.Module - return nil - }) - if err != nil { - return nil, err - } - } - return modules, nil + return roots.indexFiles(false) } // ModuleSources validates module roots and resolves them relative to its directory. @@ -344,7 +359,12 @@ func ModuleSources(directory string, module v1.Module) (SourceRoots, error) { func (roots SourceRoots) ownsSelectedPhysical(path string) bool { for _, root := range roots { - physicalRoot := physicalSourcePath(root.Path) + physicalRoot := "" + if root.index != nil { + physicalRoot = root.index.physicalRoot + } else { + physicalRoot = physicalSourcePath(root.Path) + } if sourcePathWithin(path, physicalRoot) && root.allowsPath(path, physicalRoot) { return true } diff --git a/internal/modules/tidy_input_verification.go b/internal/modules/tidy_input_verification.go index eb523f0a..86ec6bbf 100644 --- a/internal/modules/tidy_input_verification.go +++ b/internal/modules/tidy_input_verification.go @@ -3,8 +3,13 @@ package modules import ( "context" "fmt" + "io" + "io/fs" + "os" "path/filepath" + "golang.org/x/mod/sumdb/dirhash" + v1 "github.com/easyp-tech/easyp/internal/config/v1" ) @@ -111,12 +116,66 @@ func verifyTidyCache(ctx context.Context, lock v1.Lock, repository Repository) e return fmt.Errorf("VerifyCached: %w", err) } } else { - // The base Cache contract verifies installed bytes against the lock. - // Callers recheck snapshots after this potentially repairing operation. + // Install may reuse an immutable snapshot verified earlier in the + // command. Tidy requires a fresh content check before rewriting inputs, + // including when a wrapper hides the optional explicit verifier. err = repository.Install(ctx, lock) if err != nil { return fmt.Errorf("Install: %w", err) } + for _, entry := range lock.Modules { + directory, _, err := repository.Cached(entry) + if err != nil { + return fmt.Errorf("Cached: %w", err) + } + if err := verifyTidySnapshot(ctx, directory, entry); err != nil { + return fmt.Errorf("verifyTidySnapshot: %w", err) + } + } + } + return nil +} + +func verifyTidySnapshot(ctx context.Context, directory string, entry v1.LockedModule) error { + var files []string + err := filepath.WalkDir(directory, func(path string, item fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := ctx.Err(); err != nil { + return fmt.Errorf("Err: %w", err) + } + if item.IsDir() { + return nil + } + if !item.Type().IsRegular() { + return fmt.Errorf("non-regular cached path %q", path) + } + name, err := filepath.Rel(directory, path) + if err != nil { + return fmt.Errorf("Rel: %w", err) + } + files = append(files, filepath.ToSlash(name)) + return nil + }) + if err != nil { + return fmt.Errorf("WalkDir: %w", err) + } + actual, err := dirhash.Hash1(files, func(name string) (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("Err: %w", err) + } + file, err := os.Open(filepath.Join(directory, filepath.FromSlash(name))) + if err != nil { + return nil, fmt.Errorf("Open: %w", err) + } + return file, nil + }) + if err != nil { + return fmt.Errorf("Hash1: %w", err) + } + if actual != entry.Hash { + return fmt.Errorf("cached %s@%s hash mismatch: got %s, want %s; keep protobuf.lock unchanged", entry.Source, entry.Commit, actual, entry.Hash) } return nil } diff --git a/internal/modules/versionless_update_test.go b/internal/modules/versionless_update_test.go index ca25fc09..6a616c47 100644 --- a/internal/modules/versionless_update_test.go +++ b/internal/modules/versionless_update_test.go @@ -203,6 +203,8 @@ func (r *movingHeadRepository) Fetch(_ context.Context, source, version string) func (r *movingHeadRepository) Install(context.Context, v1.Lock) error { return nil } +func (r *movingHeadRepository) VerifyCached(context.Context, v1.Lock) error { return nil } + func (r *movingHeadRepository) Cached(entry v1.LockedModule) (string, v1.Module, error) { switch entry.Source { case "example.com/A": diff --git a/internal/sourceview/immutable_walk_test.go b/internal/sourceview/immutable_walk_test.go new file mode 100644 index 00000000..730bfd02 --- /dev/null +++ b/internal/sourceview/immutable_walk_test.go @@ -0,0 +1,134 @@ +package sourceview + +import ( + "fmt" + "io/fs" + "strings" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestImmutableWalkReusesParentResolution(t *testing.T) { + t.Parallel() + for _, depth := range []int{1, 8, 24} { + t.Run(fmt.Sprintf("depth_%d", depth), func(t *testing.T) { + t.Parallel() + backend := &countedWalkFS{MapFS: make(fstest.MapFS)} + prefix := strings.Repeat("nested/", depth) + const count = 100 + for index := range count { + backend.MapFS[fmt.Sprintf("%sfile-%d.proto", prefix, index)] = &fstest.MapFile{Data: []byte("contract")} + } + visited := 0 + err := NewImmutable(backend).Walk(t.Context(), ".", func(_ string, _ Resolution, err error) error { + visited++ + return err + }) + require.NoError(t, err) + assert.Equal(t, count+depth+1, visited) + assert.Equal(t, 1, backend.lstats, "only the walk root requires path resolution") + assert.Equal(t, count+depth, backend.infos, "each entry supplies its own metadata once") + }) + } +} + +func TestImmutableWalkPreservesInheritedAliases(t *testing.T) { + t.Parallel() + view := NewImmutable(fstest.MapFS{ + "real/nested/a.proto": {Data: []byte("a")}, + "real/nested/b.proto": {Data: []byte("b")}, + "alias": {Mode: fs.ModeSymlink, Data: []byte("real")}, + "chain": {Mode: fs.ModeSymlink, Data: []byte("alias")}, + }) + var files []string + err := view.Walk(t.Context(), "chain", func(logical string, resolved Resolution, err error) error { + require.NoError(t, err) + require.Len(t, resolved.Links, 2) + assert.Equal(t, "chain", resolved.Links[0].Path) + assert.Equal(t, "alias", resolved.Links[1].Path) + if !resolved.Info.IsDir() { + files = append(files, logical+"="+resolved.Path) + } + return nil + }) + require.NoError(t, err) + assert.Equal(t, []string{"chain/nested/a.proto=real/nested/a.proto", "chain/nested/b.proto=real/nested/b.proto"}, files) +} + +func TestMutableWalkObservesSourceChanges(t *testing.T) { + t.Parallel() + backend := fstest.MapFS{"directory/first.proto": {Data: []byte("first")}} + view := New(backend) + var first []string + err := view.Walk(t.Context(), ".", func(logical string, _ Resolution, err error) error { + first = append(first, logical) + return err + }) + require.NoError(t, err) + delete(backend, "directory/first.proto") + backend["directory/second.proto"] = &fstest.MapFile{Data: []byte("second")} + var second []string + err = view.Walk(t.Context(), ".", func(logical string, _ Resolution, err error) error { + second = append(second, logical) + return err + }) + require.NoError(t, err) + assert.Equal(t, []string{".", "directory", "directory/first.proto"}, first) + assert.Equal(t, []string{".", "directory", "directory/second.proto"}, second) +} + +func TestImmutableWalkRejectsUnsupportedEntries(t *testing.T) { + t.Parallel() + tests := []struct { + name string + file string + mode fs.FileMode + }{ + {name: "drive path", file: "C:outside.proto"}, + {name: "backslash", file: "unsafe\\outside.proto"}, + {name: "named pipe", file: "fifo.proto", mode: fs.ModeNamedPipe}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + view := NewImmutable(fstest.MapFS{tt.file: {Mode: tt.mode}}) + err := view.Walk(t.Context(), ".", func(_ string, _ Resolution, err error) error { return err }) + require.ErrorIs(t, err, ErrUnsupported) + }) + } +} + +type countedWalkFS struct { + fstest.MapFS + lstats int + infos int +} + +func (f *countedWalkFS) Lstat(name string) (fs.FileInfo, error) { + f.lstats++ + return f.MapFS.Lstat(name) +} + +func (f *countedWalkFS) ReadDir(name string) ([]fs.DirEntry, error) { + entries, err := f.MapFS.ReadDir(name) + if err != nil { + return nil, err + } + for index, entry := range entries { + entries[index] = countedWalkEntry{DirEntry: entry, owner: f} + } + return entries, nil +} + +type countedWalkEntry struct { + fs.DirEntry + owner *countedWalkFS +} + +func (entry countedWalkEntry) Info() (fs.FileInfo, error) { + entry.owner.infos++ + return entry.DirEntry.Info() +} diff --git a/internal/sourceview/view.go b/internal/sourceview/view.go index 116f1b73..c9721187 100644 --- a/internal/sourceview/view.go +++ b/internal/sourceview/view.go @@ -31,6 +31,7 @@ var ( type View struct { fsys fs.FS absoluteRoots []string + immutable bool } // Resolution describes a target and the physical links followed to reach it. @@ -59,6 +60,13 @@ func New(fsys fs.FS) *View { return &View{fsys: fsys} } +// NewImmutable borrows an immutable filesystem, such as a pinned Git tree. +// Walk may reuse resolved parent paths and directory-entry metadata. The caller +// must keep the filesystem unchanged for the lifetime of the view. +func NewImmutable(fsys fs.FS) *View { + return &View{fsys: fsys, immutable: true} +} + // NewLocal borrows fsys and accepts absolute links beneath canonicalRoot. // canonicalRoot must be the absolute, canonical OS path of fsys's boundary. // The caller must canonicalize it before opening the bounded filesystem. @@ -282,12 +290,12 @@ func (v *View) WalkSelected(ctx context.Context, logicalRoot string, selectEntry } func (v *View) walk(ctx context.Context, logical string, selectEntry func(string, fs.DirEntry) bool, fn WalkFunc, active map[string]bool) error { - err := ctx.Err() - if err != nil { - return &fs.PathError{Op: "Walk", Path: logical, Err: err} - } resolved, resolveErr := v.Resolve(ctx, logical) - err = ctx.Err() + return v.walkResolved(ctx, logical, resolved, resolveErr, selectEntry, fn, active) +} + +func (v *View) walkResolved(ctx context.Context, logical string, resolved Resolution, resolveErr error, selectEntry func(string, fs.DirEntry) bool, fn WalkFunc, active map[string]bool) error { + err := ctx.Err() if err != nil { return &fs.PathError{Op: "Walk", Path: logical, Err: err} } @@ -335,7 +343,28 @@ func (v *View) walk(ctx context.Context, logical string, selectEntry func(string if selectEntry != nil && !selectEntry(child, entry) { continue } - err = v.walk(ctx, child, selectEntry, fn, active) + if !v.immutable || entry.Type()&fs.ModeSymlink != 0 { + err = v.walk(ctx, child, selectEntry, fn, active) + } else { + info, infoErr := entry.Info() + if infoErr == nil && info.Mode()&fs.ModeSymlink != 0 { + // The entry became a link after ReadDir. Resolve it through + // the full bounded path instead of trusting stale metadata. + err = v.walk(ctx, child, selectEntry, fn, active) + } else { + childResolution := Resolution{Path: joinName(resolved.Path, name), Info: info, Links: slices.Clone(resolved.Links)} + if infoErr == nil { + infoErr = validatePointer(child) + if infoErr == nil && !info.IsDir() && !info.Mode().IsRegular() { + infoErr = ErrUnsupported + } + } + if infoErr != nil { + infoErr = &fs.PathError{Op: "Walk", Path: child, Err: infoErr} + } + err = v.walkResolved(ctx, child, childResolution, infoErr, selectEntry, fn, active) + } + } } if errors.Is(err, fs.SkipDir) { return nil From f051779308002f4f6324edd73b65f11ee5405b5e Mon Sep 17 00:00:00 2001 From: Khasbulat <92339556+Yakwilik@users.noreply.github.com> Date: Mon, 26 Oct 2026 04:37:24 +0300 Subject: [PATCH 2/5] fix: read shared Git objects across platform-specific paths --- .github/workflows/tests.yml | 2 +- .../adapters/gitmodules/git_object_sizes.go | 19 +++++- .../gitmodules/git_object_sizes_test.go | 66 +++++++++++++++++++ 3 files changed, 84 insertions(+), 3 deletions(-) create mode 100644 internal/adapters/gitmodules/git_object_sizes_test.go diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5fa9aa4e..c2749be3 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -79,4 +79,4 @@ jobs: ./internal/sourceview ./internal/core ./internal/generation - -run 'Test(InstalledSnapshot|SnapshotVerificationWait|SourceFilesDistinguishes|FetchRetainsSnapshotForOfflineInstall|DestinationValidatorRetains|SnapshotSourceIndex|LocalSourceIndex|ImmutableWalk|MutableWalk|PrepareGenerationUsesVerifiedInventory|PrepareGenerationWithoutInventoryKeepsLocalSourcesFresh|GenerationSelectsIndexedDependencyInputs)' + -run 'Test(InstalledSnapshot|SnapshotVerificationWait|SnapshotStorageReadsSharedObjects|SourceFilesDistinguishes|FetchRetainsSnapshotForOfflineInstall|DestinationValidatorRetains|SnapshotSourceIndex|LocalSourceIndex|ImmutableWalk|MutableWalk|PrepareGenerationUsesVerifiedInventory|PrepareGenerationWithoutInventoryKeepsLocalSourcesFresh|GenerationSelectsIndexedDependencyInputs)' diff --git a/internal/adapters/gitmodules/git_object_sizes.go b/internal/adapters/gitmodules/git_object_sizes.go index 85516f47..dad7b347 100644 --- a/internal/adapters/gitmodules/git_object_sizes.go +++ b/internal/adapters/gitmodules/git_object_sizes.go @@ -14,13 +14,28 @@ import ( "github.com/go-git/go-git/v5/storage/filesystem" ) -// Go-git's blob reads follow alternates, but EncodedObjectSize does not. Keep -// header-only size queries working for our shared pinned-commit checkouts too. +// Shared checkouts resolve both objects and their sizes through explicitly +// rooted stores. Go-git's alternate filesystem can lose Windows drive names or +// reinterpret relative paths, and EncodedObjectSize never follows alternates. type snapshotObjectStorage struct { storage.Storer alternates []storage.Storer } +func (s snapshotObjectStorage) EncodedObject(kind plumbing.ObjectType, hash plumbing.Hash) (plumbing.EncodedObject, error) { + object, err := s.Storer.EncodedObject(kind, hash) + if !errors.Is(err, plumbing.ErrObjectNotFound) { + return object, err + } + for _, alternate := range s.alternates { + object, err = alternate.EncodedObject(kind, hash) + if !errors.Is(err, plumbing.ErrObjectNotFound) { + return object, err + } + } + return nil, err +} + func (s snapshotObjectStorage) EncodedObjectSize(hash plumbing.Hash) (int64, error) { size, err := s.Storer.EncodedObjectSize(hash) if !errors.Is(err, plumbing.ErrObjectNotFound) { diff --git a/internal/adapters/gitmodules/git_object_sizes_test.go b/internal/adapters/gitmodules/git_object_sizes_test.go new file mode 100644 index 00000000..15660c8c --- /dev/null +++ b/internal/adapters/gitmodules/git_object_sizes_test.go @@ -0,0 +1,66 @@ +package gitmodules + +import ( + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSnapshotStorageReadsSharedObjects(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + name string + relative bool + }{ + {name: "absolute alternate"}, + {name: "relative alternate", relative: true}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + remote, commit := diagnosticRepository(t) + checkout := filepath.Join(t.TempDir(), "shared checkout") + runTestGit(t, remote, "clone", "--quiet", "--shared", "--no-checkout", "--", remote, checkout) + objects := filepath.Join(checkout, ".git", "objects") + require.NoFileExists(t, filepath.Join(objects, commit[:2], commit[2:])) + packs, err := os.ReadDir(filepath.Join(objects, "pack")) + require.NoError(t, err) + require.Empty(t, packs) + alternate := filepath.Join(remote, ".git", "objects") + if tt.relative { + var err error + alternate, err = filepath.Rel(objects, alternate) + require.NoError(t, err) + } + require.NoError(t, os.WriteFile(filepath.Join(objects, "info", "alternates"), []byte(filepath.ToSlash(alternate)+"\n"), 0o644)) + // The primary checkout has no object bodies. Native Git and EasyP + // must both read the same shared object database on every platform. + runTestGit(t, checkout, "cat-file", "-e", commit+"^{commit}") + store, err := snapshotStorage(checkout) + require.NoError(t, err) + repository, err := git.Open(store, nil) + require.NoError(t, err) + revision, err := repository.CommitObject(plumbing.NewHash(commit)) + require.NoError(t, err) + tree, err := revision.Tree() + require.NoError(t, err) + file, err := tree.File("api.proto") + require.NoError(t, err) + size, err := store.EncodedObjectSize(file.Hash) + require.NoError(t, err) + reader, err := file.Reader() + require.NoError(t, err) + contents, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + assert.Equal(t, int64(len(contents)), size) + assert.True(t, strings.HasPrefix(string(contents), "syntax = \"proto3\";")) + }) + } +} From f5f9e4692e2bb7f21257e3bed39e5b231442e486 Mon Sep 17 00:00:00 2001 From: Khasbulat <92339556+Yakwilik@users.noreply.github.com> Date: Mon, 26 Oct 2026 04:55:23 +0300 Subject: [PATCH 3/5] chore: keep dependency regression CI workflow unchanged --- .github/workflows/tests.yml | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index c2749be3..5816c70c 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -58,25 +58,3 @@ jobs: - name: Validate nightly release configuration run: goreleaser check - - windows-dependency-regressions: - runs-on: windows-latest - steps: - - name: Checkout repository - uses: actions/checkout@v4 - - - name: Install Go - uses: actions/setup-go@v5 - with: - go-version-file: go.mod - - - name: Verify dependency snapshot and source traversal regressions - run: >- - go test -count=1 - ./internal/adapters/gitmodules - ./internal/adapters/gitsnapshot - ./internal/modules - ./internal/sourceview - ./internal/core - ./internal/generation - -run 'Test(InstalledSnapshot|SnapshotVerificationWait|SnapshotStorageReadsSharedObjects|SourceFilesDistinguishes|FetchRetainsSnapshotForOfflineInstall|DestinationValidatorRetains|SnapshotSourceIndex|LocalSourceIndex|ImmutableWalk|MutableWalk|PrepareGenerationUsesVerifiedInventory|PrepareGenerationWithoutInventoryKeepsLocalSourcesFresh|GenerationSelectsIndexedDependencyInputs)' From d48270ba16407a2538dfd1c5f2fe3f57f5d67f39 Mon Sep 17 00:00:00 2001 From: Khasbulat Abdullin Date: Mon, 2 Nov 2026 00:02:43 +0300 Subject: [PATCH 4/5] fix(modules): handle reachable duplicate imports during tidy Coalesce byte-identical reachable proto copies, ignore unused collisions, and preserve observed local replacement metadata and import ownership. Avoid false conflicts from historical namespaces while retaining current-graph validation and snapshot indexing. Add regression coverage for MVS, cold generation, mutation checks, and historical imports. --- .spec/config/dependency.md | 58 ++- .../2026-10-15-tidy-import-collisions.md | 205 ++++++++++ V1_RELEASE_NOTES.md | 2 + .../module_config/local_dependency.go | 28 +- internal/generation/mvs_integration_test.go | 132 ++++++ internal/modules/aliases_test.go | 16 + internal/modules/effective_graph.go | 19 +- internal/modules/import_rewrites_test.go | 30 +- .../modules/import_rewrites_validation.go | 80 +++- internal/modules/import_roots_overlay.go | 2 +- internal/modules/import_roots_tidy.go | 126 +++++- internal/modules/imports.go | 55 ++- internal/modules/local_sources.go | 36 +- .../modules/resolved_file_observations.go | 45 ++- .../modules/tidy_collisions_metadata_test.go | 379 ++++++++++++++++++ internal/modules/tidy_collisions_test.go | 312 ++++++++++++++ .../tidy_historical_collisions_test.go | 125 ++++++ internal/modules/tidy_imports.go | 139 +++++++ internal/modules/tidy_inputs.go | 27 ++ internal/modules/tidy_local_sources.go | 151 +++++++ internal/modules/tidy_operation.go | 18 +- internal/modules/tidy_overlay.go | 93 +++++ internal/sourceview/local.go | 18 + internal/sourceview/view.go | 20 + internal/sourceview/view_test.go | 32 ++ 25 files changed, 2054 insertions(+), 94 deletions(-) create mode 100644 .spec/plans/2026-10-15-tidy-import-collisions.md create mode 100644 internal/generation/mvs_integration_test.go create mode 100644 internal/modules/tidy_collisions_metadata_test.go create mode 100644 internal/modules/tidy_collisions_test.go create mode 100644 internal/modules/tidy_historical_collisions_test.go create mode 100644 internal/modules/tidy_imports.go create mode 100644 internal/modules/tidy_local_sources.go create mode 100644 internal/modules/tidy_overlay.go diff --git a/.spec/config/dependency.md b/.spec/config/dependency.md index 977cfbca..12bca6df 100644 --- a/.spec/config/dependency.md +++ b/.spec/config/dependency.md @@ -29,6 +29,31 @@ require ( Nested modules get separate lock entries even when their commits match. The Git adapter tries repository candidates and the metadata reader confirms the requested module name at the candidate directory. A root manifest does not hide named nested modules. +## Minimal version selection + +Each semantic require declares a minimum for that exact module +identity. EasyP selects the highest of the direct and transitive minima and +records one revision of that module in protobuf.lock. If a consumer +requires common v1.2.0 and its dependency requires +common v1.5.0, both use v1.5.0. An available +v1.9.0 is not selected unless the graph requires it. + +Only the selected revision supplies source roots and proto contents to tidy, +generation and frozen replay. The v1.2.0 and v1.5.0 +files may differ: unselected versions are not competing import candidates. +The reachable byte comparison below applies to multiple copies in the final +selected graph, not to alternative versions of one module. + +Module identity includes the major suffix described below; MVS does not select +the lowest major or merge different identities. Full Git commits are exact +constraints, not semantic minima: different exact commits for one module remain +a conflict, and a tag combined with an exact commit must resolve to that commit. + +The real-Git regression +internal/generation/mvs_integration_test.go covers both directions +of a direct/transitive version difference, requirement ordering, an existing +lower lock, idempotent tidy, and warm/cold frozen generation. + ## Major versions and Git mapping EasyP uses Go-style semantic import versioning for module identities. Native module, require, replace source identities, fetched metadata and lock entries are checked; changing only the consumer manifest cannot bypass the check. @@ -46,7 +71,7 @@ EasyP uses Go-style semantic import versioning for module identities. Native v2/v2.0.0 tag does not release repo/v2. -repo and repo/v2 are separate MVS, lock, cache and replacement identities. They may coexist when their protobuf import paths are distinct. Major-version identities do not rename protobuf packages or disambiguate import paths; duplicates still fail, even for identical content. Checked tidy may repair consumer import declarations after a verified dependency namespace change, as described below. Same-identity maximum-minimum selection, exact SHA/tag agreement, immutable tags, local overlays and explicit frozen checks remain in force. +repo and repo/v2 are separate MVS, lock, cache and replacement identities. They may coexist when their protobuf import paths are distinct. Major-version identities do not rename protobuf packages or disambiguate import paths. Generation, lint, breaking and other module operations reject duplicate import paths; tidy uses the reachable exact-byte comparison described below. Checked tidy may repair consumer import declarations after a verified dependency namespace change. Same-identity maximum-minimum selection, exact SHA/tag agreement, immutable tags, local overlays and explicit frozen checks remain in force. The Go pre-module +incompatible exception is supported for an unsuffixed v2+ requirement whose exact Git revision has no native root protobuf.mod and no matching nested native manifest. easyp get repo@v2.0.0 adds the marker after verifying this boundary; a manually written manifest must include it explicitly. For example, repo v2.0.0+incompatible resolves the root tag v2.0.0, never a tag literally ending in +incompatible. Fetch, migration, cold installation and warm cache reads verify this metadata boundary. Update preserves the marker for eligible legacy releases and rejects a selected revision that has become native. Markers on v0/v1 or on suffixed identities are rejected. An unmarked unsuffixed v2+ requirement remains invalid. @@ -119,6 +144,33 @@ in internal/modules/operations_test.go. Sources: internal/modules/get.go, internal/modules/update.go, internal/modules/operations.go, internal/modules/manifest_requirements.go. +### Tidy import collisions + +mod tidy checks consumer targets and their reachable import closure. +Duplicate import paths that are unused by that closure do not fail tidy. For a +used name, every allowed candidate must contain exactly the same proto bytes; +identical copies resolve as one logical source, and different contents fail with +the import name and conflicting paths before project writes. Comments count as +bytes, so descriptor or package equality alone is insufficient. No physical +proto file is deleted or deduplicated in the installed snapshots. + +Consumer roots precede dependency roots; dependencies retain lock order and +each module's declared root order. Tidy validates all equal copies, including +noncanonical candidates, and rechecks their bytes, alias identities and used-name +membership before committing. Producer exclusions and module boundaries still +apply. Used-name rechecks bypass cached snapshot inventories so newly added +candidates remain observable after explicit cache verification. Verified +historical import repairs preserve the previous canonical owner +and physical source, so updating a noncanonical copy cannot rename another +producer's import. Local replacements use the same checks without writes. + +Conflicting copies in an old lock may have been unused. If the current selected +modules prove that the unchanged logical name now has byte-identical candidates, +tidy keeps that name without using the ambiguous history as rename evidence. +This includes removal of the conflicting provider. An absent or conflicting +current name still cannot be repaired from ambiguous history; equal historical +copies retain their previous canonical owner and physical-source binding. + ### Local effective graph Only replacements in the main (consuming) module apply, including to required transitive modules. Dependency manifests contribute their requirements, but their replacements are ignored. A replacement alone never introduces a dependency; unused replacements are not read or downloaded. All relative replacement paths are based on the main manifest directory, even at transitive depths. Absolute paths work directly. A native replacement manifest must declare the replaced identity; missing directories and identity mismatches are contextual errors. Legacy metadata uses the existing roots/requirements adapter without executing dependency plugins or loading dependency generation policy. @@ -224,9 +276,9 @@ The usual Git lock entry and verified h1: hash still pin the instal The backend is wired in [internal/api/mod_v1_cache.go](../../internal/api/mod_v1_cache.go) through gitmodules.NewWithBSRResolver. A future EasyP Service implementation replaces bsr.StaticResolver there and implements the same modules.BSRResolver; it need not change the dependency graph. The temporary backend has no BSR API requests, Buf executable dependency, HTTP client, implicit latest or universal BSR-to-Git inference. Other BSR revisions can require features absent from the compatibility snapshot. Separate manifests that require incompatible exact Git commits retain the existing conflict diagnostic. -A nested module root proto becomes <checkout>/<module-directory>/proto. A file below that root is imported without either physical prefix. modules.SourceRoots preserves module identity for managed selectors. Common import paths in different physical roots are rejected rather than silently selecting one. +A nested module root proto becomes <checkout>/<module-directory>/proto. A file below that root is imported without either physical prefix. modules.SourceRoots preserves module identity for managed selectors. Common import paths in different physical roots follow the strict collision check, except for tidy's reachable comparison above. -Generation, lint, breaking and module operations share root/collision rules. Source traversal excludes hidden directories, vendored output and nested-module boundaries via internal/core/path_helpers/v1_source.go. Config discovery and policy inheritance use their own traversal rules. +Generation, lint, breaking and module operations share source ownership and root rules. Tidy has its own reachable collision check; generation, lint, breaking, download, get/update, vendor and frozen checks retain strict namespace validation. Source traversal excludes hidden directories, vendored output and nested-module boundaries via internal/core/path_helpers/v1_source.go. Config discovery and policy inheritance use their own traversal rules. ## Persistence guarantees diff --git a/.spec/plans/2026-10-15-tidy-import-collisions.md b/.spec/plans/2026-10-15-tidy-import-collisions.md new file mode 100644 index 00000000..bf0d359a --- /dev/null +++ b/.spec/plans/2026-10-15-tidy-import-collisions.md @@ -0,0 +1,205 @@ +# Reachable duplicate imports during mod tidy + +> For agentic workers: execute inline using the test-driven-development and +> verification-before-completion skills. Review the plan and final changes. + +**Goal:** `mod tidy` ignores unused dependency import-path duplicates, resolves +byte-identical reachable copies as one logical proto, and rejects differing +reachable contracts before changing consumer sources, manifest or lock. + +**Architecture:** Keep the strict whole-source collision check for other module +operations. Tidy owns a bounded, observed resolver for its consumer targets and +reachable import closure. Enumerate every allowed candidate for a used name, +read its verified/proposed bytes through the existing `tidySourceView`, compare +exact bytes, and retain a deterministic canonical source. Every compared copy +must participate in transaction input rechecks. Preserve producer filters, +module boundaries, aliases, exact pins and root-inference guarantees. + +The plan review additionally requires used-name membership rechecks, including +candidate additions after an absent lookup; alias identity rechecks; an explicit +read-only overlay checkpoint with remote pin verification; the same canonical +root/owner in validation and compilation; and historical rename tests alongside +unused duplicate inventory. Root inference already indexes one producer at a +time, so its strict namespace checker needs no relaxation. + +**Tech stack:** Go 1.26.6, existing protocompile/sourceview/transaction machinery, +standard testing and testify. No dependency, toolchain or schema changes. + +## Contract and scope + +- A direct and transitive requirement on the same module remains one selection. +- Unused duplicates with equal or different contents do not invalidate tidy. +- Reachable equal bytes are deduplicated by import name, including transitive, + public/weak imports and copies reached through separate module roots. +- Reachable differing bytes remain an error identifying the import and paths. +- Consumer targets are used sources; preserve checks for conflicting target + import names rather than silently selecting a different consumer file. +- Local replacements use the same tidy import checks but retain their existing + validation-only behavior: no source/manifest/lock writes and no invented pins. +- Root inference and verified historical import renames retain ownership checks; + equality never justifies guessing roots or rebinding a renamed contract. +- Failed comparison/read/parse/cancellation/recheck leaves project files intact. +- Download, get/update, vendor, generation and frozen checks retain their current + collision contract. This task does not add consumer excludes. + +## Files and responsibilities + +- `internal/modules/imports.go`: extract bounded per-root candidate lookup so + tidy can enumerate matches without changing ordinary first-match resolution. +- `internal/modules/tidy_imports.go` (new): tidy-only used-name comparison and + deterministic logical deduplication, using observed verified source reads. +- `internal/modules/import_rewrites_validation.go`: route reachable validation + through the tidy resolver, preserve import owners and proposed-byte compile. +- `internal/modules/tidy_operation.go`: remove the eager whole-dependency + collision gate and route local overlays through tidy-specific validation. +- `internal/modules/tidy_overlay.go` (new if needed): capture consumer selection + and effective replacement roots without changing the shared lock. +- Root-rename namespace helpers: change only if a regression demonstrates that + unused duplicate inventory blocks an otherwise uniquely proven rename. +- `internal/modules/tidy_collisions_test.go`: real pinned Git repositories, + local replacements, preservation/idempotence and content/conflict behavior. +- `.spec/config/dependency.md`, `V1_RELEASE_NOTES.md`: document the tidy contract + and its exact-byte rule, distinguish it from other operations. + +## Execution + +- [x] Add real consumer contracts importing a dependency that imports another; + create a direct requirement on that common provider as well. +- [x] Add unused equal/differing duplicates, direct/transitive equal copies, + differing reachable contents and failed-transaction preservation assertions. +- [x] Cover pinned Git and local replace modes, multiple roots, aliases, producer + exclusions, missing/invalid imports and a second tidy run without changes. +- [x] Run the new tests with Go 1.26.6 and `-race`; record expected failures. +- [x] Implement observed candidate enumeration and exact-byte comparison. +- [x] Integrate it into both tidy paths and retain all equal-copy rechecks. +- [x] Add a mutation regression for a compared noncanonical copy: changing it + before commit must fail without project writes. +- [x] Run new and related module/root-rename/sourceview suites with `-race`. +- [x] Build fresh CLI/MCP and exercise normal CLI tidy on the real fixtures. +- [x] Run the full source race suite and Go linter with repository-pinned tools. +- [x] Review the diff against the fetched main, fix material findings and report + the checks actually run. Preserve unrelated and untracked files. + +## Decisions + +Exact raw-byte equality is intentional: comments, options or declarations that +differ do not qualify for automatic deduplication. No filesystem file is deleted +or rewritten to deduplicate imports. A logical import is parsed once after all +its allowed copies pass equality and source-ownership checks. + +## Review-driven checkpoints + +- Historical equal copies use previous lock order and declared root order; + an unchanged dependency or an owned consumer source retains its binding. +- Local overlay metadata is parsed from observed bytes before requirements or + remote acquisition. Compatibility adaptation also retains metadata absence + and inventories, including newly introduced nested metadata directories. +- Metadata-free local root inference retains its complete intrinsic proto + evidence and inventory; fixed metadata roots keep reachable-body validation. +- Optional absent metadata below a directory alias retains its link trace. + Missing import candidates below valid directory aliases are distinct from + dangling aliases. Authorized nested-repository absence produces an unresolved + import instead of dereferencing a missing file's nil metadata. +- Fresh CLI tests in easyp-test cover Git and replace modes, shared requirements, + preservation, idempotence and unchanged strict generation collision checks. + Existing v1 scenario arguments were updated to main's command-local --config; + assertions and v0 scenarios were not changed. + +Verification uses Go 1.26.6 and golangci-lint 2.14.0. The final source race run +limits package/test parallelism to 2/8 after earlier runs under simultaneous +builds encountered transient Git pipe/tree-read failures. No dependencies or +toolchain directives were changed. + +## Integration into dependency snapshot performance PR (2026-11-01) + +The existing design above is approved for integration into +`fix/dependency-snapshot-performance` (PR #247). Keep the command-scoped +snapshot verification, immutable file inventories, source ordering and shared +generation ownership index. Only tidy receives the reachable byte-equality +collision policy; generation and other module operations stay strict. + +- [x] Restore the collision regressions and verify the current branch rejects + the newly allowed unused and byte-identical cases (Go 1.26.6, `-race`). +- [x] Restore tidy's observed candidate resolver, canonical historical binding, + local replacement metadata checks and optional alias resolution. +- [x] Merge `effective_graph.go` explicitly: retain `cachedModuleSources` for + remote snapshots, inject the observed local readers and retain remote lock + evidence for independent tidy verification. Do not replace the PR's index + path with the stash's earlier filesystem traversal. +- [x] Reproduce additions of equal and differing copies to an indexed remote + snapshot after overlay verification. Probe live candidate membership in tidy + without modifying the graph/generation indexes, then pass all nine mutation + cases with `-race`. +- [x] Verify the combined collision/overlay/sourceview regressions and the PR's + snapshot inventory, immutable walk and generation regressions. +- [x] Run the complete source race suite with a timeout sufficient for builtin + WASM compilation, pinned lint, a fresh CLI build and `git diff --check`. +- [x] Review the final integration for correctness and verify the original + stash remains available. + +Use the repository's existing Taskfile/tool versions and Go 1.26.6 with +read-only module resolution. Do not change dependencies or toolchains. The +companion uncommitted `easyp-test` work remains a separate consumer fixture; +validate it with a fresh CLI if its harness is compatible with this branch. + +Integration verification completed with Go 1.26.6 and read-only modules: + +- The restored closure regression first reproduced the 12 missing-behavior + failures on PR #247. Combined module/sourceview/metadata package race tests + then passed. +- Review found that an indexed remote overlay could hide a new import copy + added immediately after verification. The added equal/differing-copy cases + failed before the live membership fix; all nine mutation cases passed after it. +- Final `go test -mod=readonly -race -count=1 -p=2 -parallel=8 -timeout=15m ./...` + passed all 3,352 test/subtest actions, including snapshot/index/generation and + API coverage. +- A fresh CLI passed all six `TestTidyReachableImportCollisions` e2e cases in + `easyp-test`, including unchanged strict generation rejection after tidy. +- Pinned golangci-lint 2.14.0 reported zero issues; formatting and + `git diff --check` passed. Independent follow-up review found no remaining + actionable issues. The original stash was retained unchanged. + +## Shared dependency version selection clarification (2026-11-01) + +The motivating report can describe a consumer and a transitive producer +requiring different versions of the same module. This is a version-selection +case before file collisions: select one highest semantic minimum and use only +that revision's proto contents. The existing resolver already implements this +policy; differing source bytes between its unselected and selected revisions +are allowed. + +- [x] Verify eight real-Git cases in + `internal/generation/mvs_integration_test.go`: direct/transitive minima in + both directions, both requirement orders, fresh and lower previous locks. + Confirm that one v1.5.0 is selected over v1.2.0, unrequired v1.9.0 is not + selected, and differing proto contents compile in warm/cold frozen generation. +- [x] Clarify MVS, final-graph file collisions and exact commit constraints in + the dependency specification without changing the established selection policy. +- [x] Review and rerun the affected generation/resolver tests and pinned lint. + Both package checks passed with Go 1.26.6 and `-race`; golangci-lint 2.14.0 + reported zero issues. Independent review approved the tests and documentation. + +## Historical collision false positive (2026-11-01) + +Code-path audit found a reproducible error in `tidyImportBindings`: an old lock +can legitimately contain unused differing copies, but after removing a provider +and importing the remaining current file, historical comparison rejects the +project. The exact current manifest/source succeeds without the old lock. + +- [x] Reproduce six failures across cross-module and same-module/root old + conflicts, both previous lock orders, and equal current copies. Confirm that + missing current names and conflicting current copies remain rejected. +- [x] Drop only ambiguous historical bindings when all selected revisions have + complete verified inspections and all candidates for the current name have + identical bytes. Keep the logical import unchanged. Preserve previous + canonical bindings for equal historical copies and reject guessed renames. +- [x] Pass the new regressions and existing canonical-owner, unverified identity + and deleted-old-export tests with Go 1.26.6 and `-race`. +- [x] Review the change, run the complete source race suite and pinned lint, + build a fresh CLI and exercise the current-versus-old-lock reproduction. + Final source race run passed 3,371 test/subtest actions; golangci-lint 2.14.0 + reported zero issues. The previous CLI reproduced the historical error, while + the fresh CLI passed tidy, idempotence and descriptor generation, producing + the same lock as the fresh current project. Independent review approved the + fix after its current-name proof was cached once per planning call; an + isolated 80-name allocation diagnostic fell from 174.4 MiB to 2.20 MiB. diff --git a/V1_RELEASE_NOTES.md b/V1_RELEASE_NOTES.md index dcfad810..a010a5d6 100644 --- a/V1_RELEASE_NOTES.md +++ b/V1_RELEASE_NOTES.md @@ -115,6 +115,8 @@ The native dependency format is documented in [`.spec/config/dependency.md`](.sp - `replace` is a main-module local development overlay. Local replacement state is not written into the shared `protobuf.lock`. - `--frozen` is explicit; it validates an existing manifest/lock graph and never infers frozen behavior from `CI=true`. - `mod tidy` does not discover arbitrary Git repositories from proto import strings. +- `mod tidy` ignores unused duplicate import paths and resolves reachable byte-identical proto copies as one logical source. Different bytes under a used import name remain an error before source/manifest/lock writes. All compared copies and their aliases are rechecked; local replacements keep their validation-only behavior. Generation and other commands retain their strict collision checks. +- `mod tidy` does not reject a valid current import solely because its old lock contained unused conflicting copies. It retains an unchanged name only after checking every selected current module's verified candidates; ambiguous historical renames and conflicting current contents remain errors. - Metadata-free declared dependencies can resolve missing import roots from an unambiguous pinned source/import map inside that same module. Consumer code only validates the resolved namespace and cannot select roots. Checked `get --import-root` and legacy Git input roots supply explicit hints; verified fallback roots are stored in the dependency's `protobuf.lock` entry and replayed in frozen/cold-cache operation. Existing native/Buf/legacy root metadata remains authoritative. - After a declared dependency revision changes its namespace, `mod tidy` repairs uniquely verified consumer import bindings, validates the proposed source and commits source/manifest/lock together. It preserves non-import bytes and reports applied renames; ambiguous or deleted imported bindings stop before writes. Tidy uses manifest versions without upgrading them; generation selectors and SDK output paths remain separate. - Known BSR dependencies encountered inside Git dependency metadata can be mapped through the current compatibility-snapshot resolver. Unknown BSR modules fail explicitly; no Git repository is guessed. diff --git a/internal/adapters/module_config/local_dependency.go b/internal/adapters/module_config/local_dependency.go index ba6242a1..40e5aea6 100644 --- a/internal/adapters/module_config/local_dependency.go +++ b/internal/adapters/module_config/local_dependency.go @@ -15,7 +15,19 @@ import ( // ReadLocalDependency adapts local metadata through a bounded logical view. // The regular temporary metadata tree keeps the Git compatibility reader's // strict historical checkout policy independent of local alias support. -func ReadLocalDependency(directory, source string) (_ v1.Module, resultErr error) { +func ReadLocalDependency(directory, source string) (v1.Module, error) { + return readLocalDependency(directory, source, func(name string) ([]byte, error) { + return sourceview.ReadLocal(context.Background(), directory, name) + }, false) +} + +// ReadLocalDependencyWithReader adapts captured metadata bytes while retaining +// the same bounded directory selection and regular temporary metadata tree. +func ReadLocalDependencyWithReader(directory, source string, readFile func(string) ([]byte, error)) (v1.Module, error) { + return readLocalDependency(directory, source, readFile, true) +} + +func readLocalDependency(directory, source string, readFile func(string) ([]byte, error), observeDirectories bool) (_ v1.Module, resultErr error) { stage, err := os.MkdirTemp("", "easyp-local-metadata-*") if err != nil { return v1.Module{}, fmt.Errorf("MkdirTemp: %w", err) @@ -33,6 +45,16 @@ func ReadLocalDependency(directory, source string) (_ v1.Module, resultErr error if logical != "." && (filepath.Base(logical) == ".git" || filepath.Base(logical) == "easyp_vendor" || filepath.Base(logical) == "node_modules") { return fs.SkipDir } + if observeDirectories { + // Mode selection also depends on absent metadata, including Buf + // workspace members. Capture that absence before parsing the tree. + for _, name := range []string{dependencyManifestFile, bufModuleConfigFile, bufWorkConfigFile, bufLockFile, legacyEasyPConfigFile} { + _, err := readFile(filepath.Join(logical, name)) + if err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("readFile: %w", err) + } + } + } err := os.MkdirAll(filepath.Join(stage, filepath.FromSlash(logical)), 0o755) if err != nil { return fmt.Errorf("MkdirAll: %w", err) @@ -42,9 +64,9 @@ func ReadLocalDependency(directory, source string) (_ v1.Module, resultErr error if !metadata { return nil } - raw, err := sourceview.ReadLocal(context.Background(), directory, logical) + raw, err := readFile(logical) if err != nil { - return fmt.Errorf("ReadLocal: %w", err) + return fmt.Errorf("readFile: %w", err) } err = os.WriteFile(filepath.Join(stage, filepath.FromSlash(logical)), raw, resolved.Info.Mode().Perm()) if err != nil { diff --git a/internal/generation/mvs_integration_test.go b/internal/generation/mvs_integration_test.go new file mode 100644 index 00000000..8ddbebee --- /dev/null +++ b/internal/generation/mvs_integration_test.go @@ -0,0 +1,132 @@ +package generation + +import ( + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/easyp-tech/easyp/internal/adapters/gitmodules" + v1 "github.com/easyp-tech/easyp/internal/config/v1" + "github.com/easyp-tech/easyp/internal/logger" + "github.com/easyp-tech/easyp/internal/modules" +) + +func TestTidyAndGenerationSelectSharedSemanticMinimum(t *testing.T) { + t.Parallel() + tests := []struct { + name string + direct string + transitive string + }{ + {name: "transitive raises direct minimum", direct: "v1.2.0", transitive: "v1.5.0"}, + {name: "direct raises transitive minimum", direct: "v1.5.0", transitive: "v1.2.0"}, + } + for _, tt := range tests { + for _, previousLock := range []bool{false, true} { + for _, reverse := range []bool{false, true} { + name := tt.name + "/fresh lock/common first" + if previousLock { + name = tt.name + "/existing lower lock/common first" + } + if reverse { + name = strings.TrimSuffix(name, "common first") + "producer first" + } + t.Run(name, func(t *testing.T) { + t.Parallel() + common := t.TempDir() + writeV1GenerateFixture(t, common, v1.ModuleFile, "module "+common+"\nroots proto\n") + runTestGit(t, common, "init", "-q") + const original = "syntax = \"proto3\"; package common.v1; message Common { string value = 1; }\n" + const selected = original + "message NewInMinor { string value = 1; }\n" + commits := make(map[string]string) + for _, revision := range []struct{ version, source string }{ + {version: "v1.2.0", source: original}, + {version: "v1.5.0", source: selected}, + {version: "v1.9.0", source: selected + "message NotRequired {}\n"}, + } { + writeV1GenerateFixture(t, common, "proto/common/v1/types.proto", revision.source) + runTestGit(t, common, "add", ".") + runTestGit(t, common, "-c", "user.name=EasyP Test", "-c", "user.email=test@example.com", "commit", "-qm", revision.version) + runTestGit(t, common, "tag", revision.version) + commits[revision.version] = strings.TrimSpace(runTestGit(t, common, "rev-parse", "HEAD")) + } + + producer := t.TempDir() + writeV1GenerateFixture(t, producer, v1.ModuleFile, "module "+producer+"\nroots proto\nrequire "+common+" "+tt.transitive+"\n") + writeV1GenerateFixture(t, producer, "proto/service/v1/service.proto", "syntax = \"proto3\"; package service.v1; import \"common/v1/types.proto\"; message Service { common.v1.Common value = 1; }\n") + runTestGit(t, producer, "init", "-q") + runTestGit(t, producer, "add", ".") + runTestGit(t, producer, "-c", "user.name=EasyP Test", "-c", "user.email=test@example.com", "commit", "-qm", "producer") + runTestGit(t, producer, "tag", "v1.0.0") + + root := t.TempDir() + cache := gitmodules.New(t.TempDir()) + const header = "module example.test/consumer\nroots proto\n" + if previousLock { + writeV1GenerateFixture(t, root, v1.ModuleFile, header+"require "+common+" v1.2.0\n") + writeV1GenerateFixture(t, root, "proto/request.proto", "syntax = \"proto3\"; import \"common/v1/types.proto\"; message Request { common.v1.Common value = 1; }\n") + require.NoError(t, modules.Tidy(t.Context(), root, cache)) + } + requirements := []string{"require " + common + " " + tt.direct, "require " + producer + " v1.0.0"} + if reverse { + slices.Reverse(requirements) + } + writeV1GenerateFixture(t, root, v1.ModuleFile, header+strings.Join(requirements, "\n")+"\n") + writeV1GenerateFixture(t, root, "proto/request.proto", "syntax = \"proto3\"; package consumer; import \"common/v1/types.proto\"; import \"service/v1/service.proto\"; message Request { common.v1.NewInMinor value = 1; service.v1.Service service = 2; }\n") + writeV1GenerateFixture(t, root, v1.GenerateFile, "version: v1\n") + + require.NoError(t, modules.Tidy(t.Context(), root, cache)) + + lock, err := modules.ReadLock(filepath.Join(root, v1.LockFile)) + require.NoError(t, err) + require.Len(t, lock.Modules, 2, "direct and transitive requirements share one selected module") + versions := make(map[string]string, len(lock.Modules)) + for _, entry := range lock.Modules { + versions[entry.Source] = entry.Version + if entry.Source == common { + assert.Equal(t, commits["v1.5.0"], entry.Commit) + } + } + assert.Equal(t, map[string]string{common: "v1.5.0", producer: "v1.0.0"}, versions) + manifestBefore, err := os.ReadFile(filepath.Join(root, v1.ModuleFile)) + require.NoError(t, err) + lockBefore, err := os.ReadFile(filepath.Join(root, v1.LockFile)) + require.NoError(t, err) + require.NoError(t, modules.Tidy(t.Context(), root, cache)) + + for _, cold := range []bool{false, true} { + generationCache := cache + output := filepath.Join(root, "warm.pb") + if cold { + generationCache = gitmodules.New(t.TempDir()) + output = filepath.Join(root, "cold.pb") + } + require.NoError(t, Run(t.Context(), logger.NewNop(), generationCache, Request{WorkDir: root, WorkspaceRoot: root, Frozen: true, DescriptorSetOut: output, IncludeImports: true})) + descriptors := readDescriptorSet(t, output) + assert.ElementsMatch(t, []string{"request.proto", "service/v1/service.proto", "common/v1/types.proto"}, descriptorNames(descriptors)) + for _, file := range descriptors.File { + if file.GetName() == "common/v1/types.proto" { + var names []string + for _, message := range file.MessageType { + names = append(names, message.GetName()) + } + assert.Equal(t, []string{"Common", "NewInMinor"}, names, "only the selected version supplies proto contents") + } + } + } + manifestAfter, err := os.ReadFile(filepath.Join(root, v1.ModuleFile)) + require.NoError(t, err) + lockAfter, err := os.ReadFile(filepath.Join(root, v1.LockFile)) + require.NoError(t, err) + assert.Equal(t, manifestBefore, manifestAfter) + assert.Equal(t, lockBefore, lockAfter) + }) + } + } + } +} diff --git a/internal/modules/aliases_test.go b/internal/modules/aliases_test.go index cd29105b..a4204354 100644 --- a/internal/modules/aliases_test.go +++ b/internal/modules/aliases_test.go @@ -56,6 +56,22 @@ func TestDeclaredDependencyAliasAcrossNestedRepository(t *testing.T) { require.NoError(t, Tidy(t.Context(), root, nil)) } +func TestTidyMissingImportThroughDeclaredNestedRepositoryAlias(t *testing.T) { + t.Parallel() + root := t.TempDir() + writeV1GenerateFixture(t, root, "protobuf.mod", "module example.test/app\nroots .\nrequire example.test/dep\nreplace example.test/dep => ./dep\n") + writeV1GenerateFixture(t, root, "dep/.git", "gitdir: /unused\n") + writeV1GenerateFixture(t, root, "dep/buf.yaml", "version: v2\nmodules:\n - path: .\n includes: [selected]\n") + writeV1GenerateFixture(t, root, "dep/selected/model.proto", "syntax = \"proto3\";\n") + writeV1GenerateFixture(t, root, "client.proto", "syntax = \"proto3\"; import \"alias/missing.proto\";\n") + require.NoError(t, os.Symlink("dep/selected", filepath.Join(root, "alias"))) + + err := Tidy(t.Context(), root, nil) + + require.ErrorContains(t, err, "cannot resolve imports") + require.NoFileExists(t, filepath.Join(root, v1.LockFile)) +} + func TestImportRootAliasPreservesPhysicalBufExclusions(t *testing.T) { t.Parallel() directory := t.TempDir() diff --git a/internal/modules/effective_graph.go b/internal/modules/effective_graph.go index 78c5a14e..eb201247 100644 --- a/internal/modules/effective_graph.go +++ b/internal/modules/effective_graph.go @@ -23,6 +23,8 @@ type EffectiveModule struct { type EffectiveGraph struct { Modules map[string]EffectiveModule Sources SourceRoots + // Remote pins remain internal validation evidence, never a replacement lock. + remoteLock v1.Lock } // EnsureEffectiveGraph resolves main-module replacements at every graph depth. @@ -37,6 +39,12 @@ func ensureEffectiveGraph(ctx context.Context, root string, module v1.Module, ca } func ensureEffectiveGraphWithRoots(ctx context.Context, root string, module v1.Module, cache Cache, localPath func(string) (string, error), refresh bool, hints map[string][]string) (EffectiveGraph, error) { + locals := newLocalReplacements(root, module, localPath) + locals.hints = hints + return resolveEffectiveGraph(ctx, root, module, cache, locals, refresh) +} + +func resolveEffectiveGraph(ctx context.Context, root string, module v1.Module, cache Cache, locals *localReplacements, refresh bool) (EffectiveGraph, error) { graph := EffectiveGraph{Modules: make(map[string]EffectiveModule)} existing, err := ReadLock(filepath.Join(root, v1.LockFile)) if err != nil && !errors.Is(err, os.ErrNotExist) { @@ -46,11 +54,9 @@ func ensureEffectiveGraphWithRoots(ctx context.Context, root string, module v1.M for _, entry := range existing.Modules { pins[entry.Source] = entry } - locals := newLocalReplacements(root, module, localPath) - locals.hints = hints - remote := &effectiveRemoteSource{cache: cache, pins: pins, refresh: refresh, historical: localPath != nil, modules: make(map[[4]string]cachedEffectiveModule)} + remote := &effectiveRemoteSource{cache: cache, pins: pins, refresh: refresh, historical: locals.localPath != nil, modules: make(map[[4]string]cachedEffectiveModule)} source, _ := cache.(Source) - remote.roots = newImportRootSource(source, existing, hints) + remote.roots = newImportRootSource(source, existing, locals.hints) loader := revisionLoader{source: remote, fetched: make(map[string]Fetched), pins: pins, local: locals.lookupRequirement} if refresh { loader.pins = nil @@ -63,6 +69,7 @@ func ensureEffectiveGraphWithRoots(ctx context.Context, root string, module v1.M if err != nil { return graph, fmt.Errorf("finalizeEffectiveImportRoots: %w", err) } + graph.remoteLock = pass.lock appendModule := func(name string, entry EffectiveModule, locked *v1.LockedModule) error { var roots SourceRoots var err error @@ -147,6 +154,10 @@ func (s *effectiveRemoteSource) installed(ctx context.Context, entry v1.LockedMo if err != nil { return EffectiveModule{}, fmt.Errorf("Cached: %w", err) } + dir, err = filepath.Abs(dir) + if err != nil { + return EffectiveModule{}, fmt.Errorf("Abs: %w", err) + } selected := EffectiveModule{Directory: dir, Module: module} s.modules[key] = cachedEffectiveModule{module: selected, bsr: slices.Clone(entry.BSR), roots: slices.Clone(entry.Roots)} return selected, nil diff --git a/internal/modules/import_rewrites_test.go b/internal/modules/import_rewrites_test.go index 52802697..32d6128e 100644 --- a/internal/modules/import_rewrites_test.go +++ b/internal/modules/import_rewrites_test.go @@ -37,7 +37,7 @@ func TestTidyBindingKeepsOriginalNameForAdditionalSourceAliases(t *testing.T) { previous := RootProtoFile{Path: "api/svc.proto", Identity: old.Source + "@" + old.Commit + ":physical/svc.proto"} file := RootProtoFile{Path: "api/svc.proto", Identity: current.Source + "@" + current.Commit + ":physical/svc.proto"} binding := tidyImportBinding{before: Fetched{Lock: old}, after: Fetched{Lock: current}, file: previous, - names: map[string]RootProtoFile{"svc.proto": file, "alias.proto": file}, + names: map[string][]RootProtoFile{"svc.proto": {file}, "alias.proto": {file}}, } name, err := binding.replacement("consumer.proto", "svc.proto") @@ -45,3 +45,31 @@ func TestTidyBindingKeepsOriginalNameForAdditionalSourceAliases(t *testing.T) { require.NoError(t, err) assert.Equal(t, "svc.proto", name) } + +func TestTidyBindingsPreserveRootOrderForIdenticalHistoricalCopies(t *testing.T) { + t.Parallel() + const name = "example.test/dep" + old := v1.LockedModule{Source: name, Version: "v0.4.0", Commit: versionlessCommitA, Hash: versionlessHash, Roots: []string{"schema", "api"}} + entry := v1.LockedModule{Source: name, Version: "v0.5.0", Commit: versionlessCommitANew, Hash: versionlessHash, Roots: []string{"schema/v1", "api"}} + files := func(commit string) []RootProtoFile { + var result []RootProtoFile + // Inspection order deliberately differs from root precedence. + for _, path := range []string{"api/v1/svc.proto", "schema/v1/svc.proto"} { + result = append(result, RootProtoFile{Path: path, Identity: name + "@" + commit + ":" + path, Content: []byte("syntax = \"proto3\"; message Service {}")}) + } + return result + } + previous := Fetched{Module: v1.Module{Name: name, Roots: old.Roots}, Lock: old, Inspection: &RootInspection{Files: files(old.Commit)}} + next := Fetched{Module: v1.Module{Name: name, Roots: entry.Roots}, Lock: entry, Inspection: &RootInspection{Files: files(entry.Commit)}} + repository := &rootEvidenceTestRepository{fakeRepository: &fakeRepository{}, previous: previous, current: next} + result := graphResolveResult{lock: v1.Lock{Version: 1, Modules: []v1.LockedModule{entry}}, revisions: map[rootRevisionKey]Fetched{rootRevision(name, entry.Commit): next}} + planner := newTidyImportPlanner(result, v1.Lock{Version: 1, Modules: []v1.LockedModule{old}}, repository) + + bindings, err := planner.tidyImportBindings(t.Context(), map[string]v1UnresolvedImport{"v1/svc.proto": {owner: "consumer.proto", path: "v1/svc.proto"}}, nil) + + require.NoError(t, err) + assert.Equal(t, "schema/v1/svc.proto", bindings["v1/svc.proto"].file.Path) + after, err := bindings["v1/svc.proto"].replacement("consumer.proto", "v1/svc.proto") + require.NoError(t, err) + assert.Equal(t, "svc.proto", after) +} diff --git a/internal/modules/import_rewrites_validation.go b/internal/modules/import_rewrites_validation.go index 4848b191..8d977d09 100644 --- a/internal/modules/import_rewrites_validation.go +++ b/internal/modules/import_rewrites_validation.go @@ -19,20 +19,24 @@ import ( // tidySourceView reads the proposed consumer bytes and captures unchanged // reachable inputs for the same pre-commit state check as the project writer. type tidySourceView struct { - observations *resolvedFileObservations - roots SourceRoots - proposed map[string][]byte - scopes map[string]*resolvedFileObservations - pinned map[string]map[string][]byte - old []tidyOldNamespace - mu sync.Mutex + observations *resolvedFileObservations + roots SourceRoots + proposed map[string][]byte + scopes map[string]*resolvedFileObservations + pinned map[string]map[string][]byte + old []tidyOldNamespace + imports map[string]tidyImportResolution + inferred []tidyLocalRoots + compatibility []tidyLocalMetadata + mu sync.Mutex } func newTidySourceView(observations *resolvedFileObservations, roots SourceRoots) *tidySourceView { return &tidySourceView{ observations: observations, roots: roots, proposed: make(map[string][]byte), scopes: make(map[string]*resolvedFileObservations), - pinned: make(map[string]map[string][]byte), + pinned: make(map[string]map[string][]byte), + imports: make(map[string]tidyImportResolution), } } @@ -57,8 +61,14 @@ func (view *tidySourceView) retainPinnedSources(resolved graphResolveResult) { func (view *tidySourceView) readSourceFile(path string) ([]byte, error) { view.mu.Lock() defer view.mu.Unlock() + allowed := view.roots.FileAllowed() + if !allowed(path) { + return nil, &os.PathError{Op: "open", Path: path, Err: os.ErrNotExist} + } for _, root := range view.roots { - if !sourcePathWithin(path, root.Path) || !root.allows(path, root.Path) { + // The logical root owns the spelling; the complete graph validates + // targets inside another declared module, including bounded aliases. + if !sourcePathWithin(path, root.Path) || !root.allowsPath(path, root.Path) { continue } tx := view.observations @@ -109,6 +119,10 @@ func (view *tidySourceView) openSourceFile(path string) (io.ReadCloser, error) { } func (view *tidySourceView) validateImports(ctx context.Context, files []string, report TidyResult) (map[string]bool, error) { + err := view.validateConsumerTargets(ctx, files) + if err != nil { + return nil, fmt.Errorf("validateConsumerTargets: %w", err) + } owners := make(map[string]bool) var queue []v1ImportSource seen := make(map[v1ImportSource]bool) @@ -120,7 +134,6 @@ func (view *tidySourceView) validateImports(ctx context.Context, files []string, seen[source] = true consumerFiles[path] = true } - allowed := view.roots.FileAllowed() var unresolved []v1UnresolvedImport for len(queue) > 0 { err := ctx.Err() @@ -146,21 +159,17 @@ func (view *tidySourceView) validateImports(ctx context.Context, files []string, if !ValidProtoImportPath(name) { return nil, fmt.Errorf("%s: invalid import %q", source.path, name) } - imported, err := resolveV1ImportSource(name, view.roots, allowed) + resolved, err := view.resolveImport(ctx, name) if errors.Is(err, os.ErrNotExist) { unresolved = append(unresolved, v1UnresolvedImport{owner: source.path, path: name}) continue } if err != nil { - return nil, fmt.Errorf("resolveV1ImportSource: %s imports %q: %w", source.path, name, err) + return nil, fmt.Errorf("resolveImport: %s imports %q: %w", source.path, name, err) } + imported := resolved.source if consumerFiles[source.path] && !imported.builtin { - for _, root := range view.roots { - if sourcePathWithin(imported.path, root.Path) && root.allows(imported.path, root.Path) { - owners[root.Module] = true - break - } - } + owners[resolved.root.Module] = true } if !seen[imported] { seen[imported] = true @@ -171,7 +180,7 @@ func (view *tidySourceView) validateImports(ctx context.Context, files []string, if len(unresolved) > 0 { return nil, fmt.Errorf("cannot resolve imports %v", unresolved) } - err := view.compileChanges(ctx, files, report) + err = view.compileChanges(ctx, files, report) if err != nil { return nil, fmt.Errorf("compileChanges: %w", err) } @@ -186,6 +195,7 @@ func (view *tidySourceView) compileChanges(ctx context.Context, files []string, } compiled[change.File] = true name := "" + physical := "" for _, file := range files { target, captured := view.observations.capturedTarget(file) if !captured { @@ -195,6 +205,7 @@ func (view *tidySourceView) compileChanges(ctx context.Context, files []string, continue } path := filepath.Join(view.observations.requestedRoot, file) + physical = path for _, root := range view.roots { if root.Module == view.roots[0].Module && sourcePathWithin(path, root.Path) { relative, err := filepath.Rel(root.Path, path) @@ -214,9 +225,7 @@ func (view *tidySourceView) compileChanges(ctx context.Context, files []string, } // Independent consumer targets (including owned build copies) are // compiled separately; only each target's reachable closure is linked. - compiler := protocompile.Compiler{Resolver: compilerimports.WithStandardImports(&protocompile.SourceResolver{ - ImportPaths: view.roots.Paths(), Accessor: view.openSourceFile, - })} + compiler := protocompile.Compiler{Resolver: compilerimports.WithStandardImports(view.compilerResolver(ctx, name, physical))} _, err := compiler.Compile(ctx, name) if err != nil { return fmt.Errorf("Compile: %s: %w", change.File, err) @@ -224,3 +233,30 @@ func (view *tidySourceView) compileChanges(ctx context.Context, files []string, } return nil } + +// Bind the changed target to its observed physical source. Its imports use the +// same canonical decision and equality proof as reachable validation. +func (view *tidySourceView) compilerResolver(ctx context.Context, targetName, targetPath string) protocompile.Resolver { + return protocompile.ResolverFunc(func(name string) (protocompile.SearchResult, error) { + path := targetPath + if name != targetName { + resolved, err := view.resolveImport(ctx, name) + if err != nil { + return protocompile.SearchResult{}, fmt.Errorf("resolveImport: %w", err) + } + if resolved.source.builtin { + data, err := wellknownimports.Content.ReadFile(resolved.source.path) + if err != nil { + return protocompile.SearchResult{}, fmt.Errorf("ReadFile: %w", err) + } + return protocompile.SearchResult{Source: bytes.NewReader(data)}, nil + } + path = resolved.source.path + } + source, err := view.openSourceFile(path) + if err != nil { + return protocompile.SearchResult{}, fmt.Errorf("openSourceFile: %w", err) + } + return protocompile.SearchResult{Source: source}, nil + }) +} diff --git a/internal/modules/import_roots_overlay.go b/internal/modules/import_roots_overlay.go index 8a9744c4..3b0cd6d5 100644 --- a/internal/modules/import_roots_overlay.go +++ b/internal/modules/import_roots_overlay.go @@ -12,7 +12,7 @@ func finalizeEffectiveImportRoots(ctx context.Context, pass resolutionPass, loca var candidates []importRootModule for _, name := range pass.locals { entry := locals.modules[name] - candidate, err := inspectLocalRootCandidates(ctx, entry.Directory, entry.Module, len(locals.hints[name]) > 0) + candidate, err := locals.inspect(ctx, entry.Directory, entry.Module, len(locals.hints[name]) > 0) if err != nil { return v1.Lock{}, fmt.Errorf("inspectLocalRootCandidates: %s: %w", name, err) } diff --git a/internal/modules/import_roots_tidy.go b/internal/modules/import_roots_tidy.go index 862fc168..3ba6d82c 100644 --- a/internal/modules/import_roots_tidy.go +++ b/internal/modules/import_roots_tidy.go @@ -1,6 +1,7 @@ package modules import ( + "bytes" "context" "fmt" "slices" @@ -25,11 +26,14 @@ type tidyImportBinding struct { before Fetched after Fetched file RootProtoFile - names map[string]RootProtoFile + names map[string][]RootProtoFile } func (planner tidyImportPlanner) tidyImportBindings(ctx context.Context, needed map[string]v1UnresolvedImport, view *tidySourceView) (tidyImportBindings, error) { bindings := make(tidyImportBindings) + keepCurrent := make(map[string]bool) + var currentImports map[string]bool + currentImportsChecked := false if len(needed) == 0 { return bindings, nil } @@ -39,12 +43,19 @@ func (planner tidyImportPlanner) tidyImportBindings(ctx context.Context, needed for _, entry := range after.Modules { current[entry.Source] = entry } + changed := slices.ContainsFunc(before.Modules, func(old v1.LockedModule) bool { + entry, retained := current[old.Source] + return !retained || !sameTidyImportScope(old, entry) + }) + if !changed { + return bindings, nil + } for _, old := range before.Modules { entry, retained := current[old.Source] - if retained && strings.EqualFold(old.Commit, entry.Commit) && slices.Equal(old.Roots, entry.Roots) { - continue - } if !supported { + if retained && sameTidyImportScope(old, entry) { + continue + } imported, used, err := planner.uninspectedTidyBinding(ctx, old, needed, view) if err != nil { return nil, fmt.Errorf("uninspectedTidyBinding: %w", err) @@ -69,7 +80,7 @@ func (planner tidyImportPlanner) tidyImportBindings(ctx context.Context, needed if previous.Inspection == nil || previous.Inspection.Provisional || (retained && (next.Inspection == nil || next.Inspection.Provisional || next.Lock.Hash == "" || next.Lock.Hash != entry.Hash)) { return nil, fmt.Errorf("module %s: cannot verify old %s commit %s roots %v and new %s commit %s roots %v from complete pinned inspections; restore the old pin or verify explicit producer roots before running easyp mod tidy", old.Source, old.Version, old.Commit, previous.Module.Roots, entry.Version, entry.Commit, next.Module.Roots) } - names := make(map[string]RootProtoFile) + names := make(map[string][]RootProtoFile) if retained { names, err = tidyRootNamespace(next) if err != nil { @@ -80,12 +91,46 @@ func (planner tidyImportPlanner) tidyImportBindings(ctx context.Context, needed if err != nil { return nil, fmt.Errorf("tidyRootNamespace: %w", err) } - for name, file := range previousNames { + for name, files := range previousNames { if _, used := needed[name]; !used { continue } - if prior, exists := bindings[name]; exists && prior.before.Lock.Source != old.Source { - return nil, fmt.Errorf("old locked namespace has duplicate import %q from %s and %s", name, prior.before.Lock.Source, old.Source) + if keepCurrent[name] { + continue + } + file := files[0] + var conflict error + for _, other := range files[1:] { + if !bytes.Equal(file.Content, other.Content) { + conflict = fmt.Errorf("module %s has conflicting historical import %q: %s and %s have different contents", old.Source, name, file.Path, other.Path) + break + } + } + prior, exists := bindings[name] + if conflict == nil && exists && !bytes.Equal(prior.file.Content, file.Content) { + conflict = fmt.Errorf("old locked namespace has conflicting import %q from %s and %s: different contents", name, prior.before.Lock.Source, old.Source) + } + if conflict != nil { + if !currentImportsChecked { + currentImports, err = planner.unambiguousCurrentImports(ctx) + if err != nil { + return nil, fmt.Errorf("unambiguousCurrentImports: %w", err) + } + currentImportsChecked = true + } + if !currentImports[name] { + return nil, conflict + } + // An unused old collision supplies no canonical rename evidence. + // Keep a name proved by every current selected module instead. + delete(bindings, name) + keepCurrent[name] = true + continue + } + if exists { + // Previous lock and root order chooses the owner of equal copies. + // A changed noncanonical producer cannot rename another's import. + continue } bindings[name] = tidyImportBinding{before: previous, after: next, file: file, names: names} } @@ -93,10 +138,51 @@ func (planner tidyImportPlanner) tidyImportBindings(ctx context.Context, needed return bindings, nil } +func (planner tidyImportPlanner) unambiguousCurrentImports(ctx context.Context) (map[string]bool, error) { + contents := make(map[string][]byte) + unambiguous := make(map[string]bool) + for _, entry := range planner.current.lockFile().Modules { + err := ctx.Err() + if err != nil { + return nil, fmt.Errorf("Err: %w", err) + } + fetched, exists := planner.current.currentRevision(entry) + if !exists || fetched.Inspection == nil || fetched.Inspection.Provisional || fetched.Lock.Hash == "" { + return nil, nil + } + names, err := tidyRootNamespace(fetched) + if err != nil { + return nil, fmt.Errorf("tidyRootNamespace: %w", err) + } + for name, candidates := range names { + for _, candidate := range candidates { + content, found := contents[name] + if !found { + contents[name] = candidate.Content + unambiguous[name] = true + continue + } + if !bytes.Equal(content, candidate.Content) { + unambiguous[name] = false + } + } + } + } + return unambiguous, nil +} + +func sameTidyImportScope(before, after v1.LockedModule) bool { + return strings.EqualFold(before.Commit, after.Commit) && slices.Equal(before.Roots, after.Roots) +} + func (planner tidyImportPlanner) lockedRootScope(ctx context.Context, selector rootSelectionSource, entry v1.LockedModule) (Fetched, error) { if previous, exists := planner.current.previousRootScope(entry); exists { return previous, nil } + if previous, exists := planner.current.currentRevision(entry); exists && sameTidyImportScope(entry, previous.Lock) && previous.Inspection != nil { + // An unchanged verified pin also proves its historical namespace. + return previous, nil + } previous, err := fetchLockedRootScope(ctx, selector, entry, entry.Roots) if err != nil { return Fetched{}, fmt.Errorf("fetchLockedRootScope: %w", err) @@ -126,11 +212,11 @@ func (planner tidyImportPlanner) uninspectedTidyBinding(ctx context.Context, ent return v1UnresolvedImport{}, false, nil } -func tidyRootNamespace(fetched Fetched) (map[string]RootProtoFile, error) { - names := make(map[string]RootProtoFile) +func tidyRootNamespace(fetched Fetched) (map[string][]RootProtoFile, error) { + names := make(map[string][]RootProtoFile) prefix := fetched.Lock.Source + "@" + fetched.Lock.Commit + ":" - for _, file := range fetched.Inspection.Files { - for _, root := range fetched.Module.Roots { + for _, root := range fetched.Module.Roots { + for _, file := range fetched.Inspection.Files { name, within := importRootRelative(root, file.Path) if !within { continue @@ -138,10 +224,7 @@ func tidyRootNamespace(fetched Fetched) (map[string]RootProtoFile, error) { if !strings.HasPrefix(file.Identity, prefix) || !ValidProtoImportPath(strings.TrimPrefix(file.Identity, prefix)) { return nil, fmt.Errorf("module %s has an unverified pinned source identity for %s at commit %s", fetched.Lock.Source, file.Path, fetched.Lock.Commit) } - if previous, exists := names[name]; exists && previous.Path != file.Path { - return nil, fmt.Errorf("module %s has duplicate import path %q: %s and %s", fetched.Lock.Source, name, previous.Path, file.Path) - } - names[name] = file + names[name] = append(names[name], file) } } return names, nil @@ -150,12 +233,15 @@ func tidyRootNamespace(fetched Fetched) (map[string]RootProtoFile, error) { func (binding tidyImportBinding) replacement(path, name string) (string, error) { physical := rootGitSourcePath(binding.file, binding.before.Lock) var names []string - for current, file := range binding.names { - if physical == rootGitSourcePath(file, binding.after.Lock) { - names = append(names, current) + for current, files := range binding.names { + for _, file := range files { + if physical == rootGitSourcePath(file, binding.after.Lock) { + names = append(names, current) + break + } } } - if current, retained := binding.names[name]; retained && (len(names) == 0 || rootGitSourcePath(current, binding.after.Lock) == physical) { + if current, retained := binding.names[name]; retained && (len(names) == 0 || slices.ContainsFunc(current, func(file RootProtoFile) bool { return rootGitSourcePath(file, binding.after.Lock) == physical })) { // A moved source retaining its public name remains ordinary producer // evolution. A surviving source under a new name has priority instead. return name, nil diff --git a/internal/modules/imports.go b/internal/modules/imports.go index 472715a0..c18bf5e0 100644 --- a/internal/modules/imports.go +++ b/internal/modules/imports.go @@ -179,35 +179,46 @@ func findUnresolvedV1ImportsWithSources(moduleDir string, roots []string, depend func resolveV1ImportSource(importPath string, roots SourceRoots, allowed func(string) bool) (v1ImportSource, error) { for _, root := range roots { - candidate := filepath.Join(root.Path, filepath.FromSlash(importPath)) - if allowed != nil { - // The shared filter checks both owners. This root still needs to own - // the import's spelling, but a selected target may belong to another root. - if !allowed(candidate) || !root.allowsPath(candidate, root.Path) { - continue - } - } else if !root.allows(candidate, root.Path) { - continue - } - relative, err := filepath.Rel(root.boundary(), candidate) - if err != nil { - return v1ImportSource{}, fmt.Errorf("Rel: %w", err) - } - resolved, err := sourceview.ResolveLocal(context.Background(), root.boundary(), relative) - info := resolved.Info - if errors.Is(err, sourceview.ErrNestedRepository) && roots.ownsSelectedPhysical(filepath.Join(physicalSourcePath(root.boundary()), filepath.FromSlash(resolved.Path))) { - err = nil - } + source, err := resolveV1ImportInRoot(importPath, root, roots, allowed) if errors.Is(err, os.ErrNotExist) { continue } if err != nil { - return v1ImportSource{}, fmt.Errorf("Stat: %w", err) + return v1ImportSource{}, fmt.Errorf("resolveV1ImportInRoot: %w", err) } - if info.Mode().IsRegular() { - return v1ImportSource{path: candidate}, nil + return source, nil + } + return resolveV1BuiltinImport(importPath) +} + +// Each candidate retains the complete graph's ownership and alias checks. +func resolveV1ImportInRoot(importPath string, root SourceRoot, roots SourceRoots, allowed func(string) bool) (v1ImportSource, error) { + candidate := filepath.Join(root.Path, filepath.FromSlash(importPath)) + if allowed != nil { + if !allowed(candidate) || !root.allowsPath(candidate, root.Path) { + return v1ImportSource{}, os.ErrNotExist } + } else if !root.allows(candidate, root.Path) { + return v1ImportSource{}, os.ErrNotExist + } + relative, err := filepath.Rel(root.boundary(), candidate) + if err != nil { + return v1ImportSource{}, fmt.Errorf("Rel: %w", err) + } + resolved, err := sourceview.ResolveLocalOptional(context.Background(), root.boundary(), relative) + if errors.Is(err, sourceview.ErrNestedRepository) && roots.ownsSelectedPhysical(filepath.Join(physicalSourcePath(root.boundary()), filepath.FromSlash(resolved.Path))) { + err = nil + } + if err != nil { + return v1ImportSource{}, fmt.Errorf("ResolveLocalOptional: %w", err) + } + if resolved.Info == nil || !resolved.Info.Mode().IsRegular() { + return v1ImportSource{}, os.ErrNotExist } + return v1ImportSource{path: candidate}, nil +} + +func resolveV1BuiltinImport(importPath string) (v1ImportSource, error) { info, err := fs.Stat(wellknownimports.Content, importPath) if err != nil { return v1ImportSource{}, fmt.Errorf("Stat: %w", err) diff --git a/internal/modules/local_sources.go b/internal/modules/local_sources.go index 53873b58..9a94c79f 100644 --- a/internal/modules/local_sources.go +++ b/internal/modules/local_sources.go @@ -21,6 +21,8 @@ type localReplacements struct { paths map[string]string localPath func(string) (string, error) hints map[string][]string + read func(string, string) (EffectiveModule, error) + inspect func(context.Context, string, v1.Module, bool) (importRootModule, error) } func newLocalReplacements(directory string, module v1.Module, localPath func(string) (string, error)) *localReplacements { @@ -28,7 +30,10 @@ func newLocalReplacements(directory string, module v1.Module, localPath func(str for _, replacement := range module.Replaces { targets[replacement.Module] = replacement.Target } - return &localReplacements{directory: directory, targets: targets, modules: make(map[string]EffectiveModule), paths: make(map[string]string), localPath: localPath} + return &localReplacements{ + directory: directory, targets: targets, modules: make(map[string]EffectiveModule), paths: make(map[string]string), localPath: localPath, + read: readLocalReplacement, inspect: inspectLocalRootCandidates, + } } func (l *localReplacements) lookupRequirement(name, version string) (v1.Module, bool, error) { @@ -57,7 +62,7 @@ func (l *localReplacements) lookup(name string) (v1.Module, bool, error) { return v1.Module{}, true, fmt.Errorf("local replacement %s => %s: %w", name, target, err) } } - entry, err := readLocalReplacement(directory, name) + entry, err := l.read(directory, name) if err != nil { return v1.Module{}, true, fmt.Errorf("local replacement %s => %s: %w", name, target, err) } @@ -78,6 +83,16 @@ func (l *localReplacements) lookup(name string) (v1.Module, bool, error) { } func readLocalReplacement(directory, name string) (EffectiveModule, error) { + return readLocalReplacementWithReader(directory, name, nil) +} + +func readLocalReplacementWithReader(directory, name string, readFile func(string) ([]byte, error)) (EffectiveModule, error) { + observed := readFile != nil + if !observed { + readFile = func(file string) ([]byte, error) { + return sourceview.ReadLocal(context.Background(), directory, file) + } + } info, err := os.Stat(directory) if err != nil { return EffectiveModule{}, fmt.Errorf("Stat: %w", err) @@ -85,9 +100,9 @@ func readLocalReplacement(directory, name string) (EffectiveModule, error) { if !info.IsDir() { return EffectiveModule{}, fmt.Errorf("%s is not a directory", directory) } - raw, err := sourceview.ReadLocal(context.Background(), directory, v1.ModuleFile) + raw, err := readFile(v1.ModuleFile) if err != nil && !errors.Is(err, os.ErrNotExist) { - return EffectiveModule{}, fmt.Errorf("ReadFile: %w", err) + return EffectiveModule{}, fmt.Errorf("readFile: %w", err) } var module v1.Module if v1.IsModuleManifest(raw) { @@ -100,9 +115,16 @@ func readLocalReplacement(directory, name string) (EffectiveModule, error) { } } else { // Compatibility reads only roots/requirements, never dependency plugins. - module, err = moduleconfig.ReadLocalDependency(directory, name) - if err != nil { - return EffectiveModule{}, fmt.Errorf("ReadGitDependency: %w", err) + if observed { + module, err = moduleconfig.ReadLocalDependencyWithReader(directory, name, readFile) + if err != nil { + return EffectiveModule{}, fmt.Errorf("ReadLocalDependencyWithReader: %w", err) + } + } else { + module, err = moduleconfig.ReadLocalDependency(directory, name) + if err != nil { + return EffectiveModule{}, fmt.Errorf("ReadLocalDependency: %w", err) + } } } if _, err := ModuleSources(directory, module); err != nil { diff --git a/internal/modules/resolved_file_observations.go b/internal/modules/resolved_file_observations.go index b38607a1..ef7cecc9 100644 --- a/internal/modules/resolved_file_observations.go +++ b/internal/modules/resolved_file_observations.go @@ -138,6 +138,15 @@ func (tx *resolvedFileObservations) readState(name string, allowed SourceRoots) if errors.Is(err, os.ErrNotExist) && len(resolved.Links) == 0 { return resolvedFileState{name: name, resolution: sourceview.Resolution{Path: filepath.ToSlash(name)}, allowed: slices.Clone(allowed)}, nil } + if errors.Is(err, os.ErrNotExist) { + missing, checkErr := tx.missingFileThroughDirectoryAlias(view, name, resolved) + if checkErr != nil { + return resolvedFileState{}, fmt.Errorf("missingFileThroughDirectoryAlias: %w", checkErr) + } + if missing { + return resolvedFileState{name: name, resolution: resolved, allowed: slices.Clone(allowed)}, nil + } + } if err != nil { return resolvedFileState{}, fmt.Errorf("Resolve: %w", err) } @@ -186,16 +195,41 @@ func (tx *resolvedFileObservations) readState(name string, allowed SourceRoots) return resolvedFileState{name: name, data: data, exists: true, resolution: resolved, allowed: slices.Clone(allowed)}, nil } +// Optional metadata can be absent beneath an existing directory alias. Resolve +// the complete parent to distinguish that case from a dangling alias target. +// Keep the link trace so retargeting an equally empty directory is still caught. +func (tx *resolvedFileObservations) missingFileThroughDirectoryAlias(view *sourceview.View, name string, resolved sourceview.Resolution) (bool, error) { + parent, err := view.Resolve(context.Background(), filepath.ToSlash(filepath.Dir(name))) + if errors.Is(err, os.ErrNotExist) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("Resolve: %w", err) + } + if !parent.Info.IsDir() || resolved.Path != filepath.ToSlash(filepath.Join(parent.Path, filepath.Base(name))) || !sameResolvedLinks(resolved.Links, parent.Links) { + return false, nil + } + err = sourceview.CheckLocalResolution(tx.root, parent) + if err != nil { + return false, fmt.Errorf("CheckLocalResolution: %w", err) + } + return true, nil +} + func sameResolvedInfo(before, after os.FileInfo) bool { return before != nil && after != nil && os.SameFile(before, after) && before.Mode() == after.Mode() && before.Size() == after.Size() && before.ModTime().Equal(after.ModTime()) } func sameResolvedPath(before, after sourceview.Resolution) bool { - if before.Path != after.Path || len(before.Links) != len(after.Links) || !sameResolvedInfo(before.Info, after.Info) { + return before.Path == after.Path && sameResolvedInfo(before.Info, after.Info) && sameResolvedLinks(before.Links, after.Links) +} + +func sameResolvedLinks(before, after []sourceview.Link) bool { + if len(before) != len(after) { return false } - for i, link := range before.Links { - other := after.Links[i] + for i, link := range before { + other := after[i] if link.Path != other.Path || link.Target != other.Target || !sameResolvedInfo(link.Info, other.Info) { return false } @@ -204,7 +238,10 @@ func sameResolvedPath(before, after sourceview.Resolution) bool { } func sameResolvedFile(before, after resolvedFileState) bool { - return before.exists == after.exists && (!before.exists || (sameResolvedPath(before.resolution, after.resolution) && bytes.Equal(before.data, after.data))) + if before.exists != after.exists || before.resolution.Path != after.resolution.Path || !sameResolvedLinks(before.resolution.Links, after.resolution.Links) { + return false + } + return !before.exists || (sameResolvedInfo(before.resolution.Info, after.resolution.Info) && bytes.Equal(before.data, after.data)) } func (tx *resolvedFileObservations) captureParents(name string) error { diff --git a/internal/modules/tidy_collisions_metadata_test.go b/internal/modules/tidy_collisions_metadata_test.go new file mode 100644 index 00000000..0bdfedad --- /dev/null +++ b/internal/modules/tidy_collisions_metadata_test.go @@ -0,0 +1,379 @@ +package modules_test + +import ( + "context" + "os" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + "github.com/easyp-tech/easyp/internal/adapters/gitmodules" + v1 "github.com/easyp-tech/easyp/internal/config/v1" + "github.com/easyp-tech/easyp/internal/modules" + "github.com/easyp-tech/easyp/internal/sourceview" +) + +func TestTidyUnusedDuplicatesKeepIntrinsicRootInference(t *testing.T) { + t.Parallel() + fixture := newTidyCollisionFixture(t, "pinned git", "", map[string]string{"api/unused.proto": "syntax = \"proto3\"; package first;"}, map[string]string{"proto/unused.proto": "syntax = \"proto3\"; package second;"}) + provider, commit := importRootsRepository(t, map[string]string{ + "api/request.proto": "syntax = \"proto3\"; import \"extra.proto\";", + "api/extra.proto": "syntax = \"proto3\"; package extra; message Extra {}", + }, nil) + manifest := string(importRootsRead(t, fixture.consumer, v1.ModuleFile)) + "require " + provider + " " + commit + "\n" + importRootsWrite(t, fixture.consumer, v1.ModuleFile, manifest) + importRootsWrite(t, fixture.consumer, "api/extra_client.proto", "syntax = \"proto3\"; import \"extra.proto\"; message ExtraClient { extra.Extra value = 1; }") + + err := modules.Tidy(t.Context(), fixture.consumer, fixture.cache) + + require.NoError(t, err) + lock, err := modules.ReadLock(filepath.Join(fixture.consumer, v1.LockFile)) + require.NoError(t, err) + require.Len(t, lock.Modules, 3) + found := false + for _, entry := range lock.Modules { + if entry.Source == provider { + found = true + assert.Equal(t, []string{"api"}, entry.Roots) + assert.Equal(t, commit, entry.Commit) + } + } + assert.True(t, found) +} + +func TestTidyDuplicateCandidatesRespectBufExclusions(t *testing.T) { + t.Parallel() + for _, mode := range []string{"pinned git", "local replace"} { + t.Run(mode, func(t *testing.T) { + t.Parallel() + fixture := newTidyCollisionFixture(t, mode, "", map[string]string{"api/common/v1/types.proto": strings.Replace(tidyCommonProto, "string value", "bool value", 1)}, nil) + require.NoError(t, os.Remove(filepath.Join(fixture.producer, v1.ModuleFile))) + importRootsWrite(t, fixture.producer, "buf.yaml", "version: v2\nmodules:\n - path: api\n excludes: [api/common]\n") + importRootsWrite(t, fixture.producer, "easyp.yaml", "deps: ["+fixture.common+"@"+fixture.commits[fixture.common]+"]\ngenerate:\n inputs: [{directory: {root: api}}]\n") + commit := tidyCollisionCommit(t, fixture.producer) + manifest := strings.Replace(string(importRootsRead(t, fixture.consumer, v1.ModuleFile)), fixture.commits[fixture.producer], commit, 1) + importRootsWrite(t, fixture.consumer, v1.ModuleFile, manifest) + + err := modules.Tidy(t.Context(), fixture.consumer, fixture.cache) + + require.NoError(t, err) + assert.Equal(t, tidyCommonProto, string(importRootsRead(t, fixture.common, "proto/common/v1/types.proto"))) + }) + } +} + +func TestTidyDuplicateCandidatesRecheckAliasIdentity(t *testing.T) { + t.Parallel() + fixture := newTidyCollisionFixture(t, "local replace", "", map[string]string{"api/materialized/one.proto": tidyCommonProto, "api/materialized/two.proto": tidyCommonProto}, nil) + alias := filepath.Join(fixture.producer, "api/common/v1/types.proto") + require.NoError(t, os.MkdirAll(filepath.Dir(alias), 0o755)) + require.NoError(t, os.Symlink("../../materialized/one.proto", alias)) + manifest := string(importRootsRead(t, fixture.consumer, v1.ModuleFile)) + manifest = strings.Replace(manifest, "replace "+fixture.common+" => "+fixture.common+"\n", "", 1) + importRootsWrite(t, fixture.consumer, v1.ModuleFile, manifest) + importRootsWrite(t, fixture.consumer, v1.LockFile, "version: 1\nmodules: []\n") + require.NoError(t, modules.Tidy(t.Context(), fixture.consumer, fixture.cache)) + before := importRootsRead(t, fixture.consumer, v1.LockFile) + spy := &tidyCollisionMutation{Cache: fixture.cache, at: 1, mutate: func(v1.Lock) { + require.NoError(t, os.Remove(alias)) + require.NoError(t, os.Symlink("../../materialized/two.proto", alias)) + }} + + err := modules.Tidy(t.Context(), fixture.consumer, spy) + + require.ErrorIs(t, err, sourceview.ErrChanged) + assert.Equal(t, manifest, string(importRootsRead(t, fixture.consumer, v1.ModuleFile))) + assert.Equal(t, before, importRootsRead(t, fixture.consumer, v1.LockFile)) +} + +func TestTidyConflictingConsumerTargets(t *testing.T) { + t.Parallel() + for _, different := range []bool{false, true} { + t.Run(map[bool]string{false: "equal", true: "different"}[different], func(t *testing.T) { + t.Parallel() + fixture := newTidyCollisionFixture(t, "pinned git", "", nil, nil) + manifest := strings.Replace(string(importRootsRead(t, fixture.consumer, v1.ModuleFile)), "roots api\n", "roots api other\n", 1) + importRootsWrite(t, fixture.consumer, v1.ModuleFile, manifest) + copy := string(importRootsRead(t, fixture.consumer, "api/consumer/request.proto")) + if different { + copy += "// distinct consumer contract\n" + } + importRootsWrite(t, fixture.consumer, "other/consumer/request.proto", copy) + + err := modules.Tidy(t.Context(), fixture.consumer, fixture.cache) + + if different { + require.ErrorContains(t, err, "duplicate import path") + assert.ErrorContains(t, err, "consumer/request.proto") + assert.Equal(t, manifest, string(importRootsRead(t, fixture.consumer, v1.ModuleFile))) + assert.NoFileExists(t, filepath.Join(fixture.consumer, v1.LockFile)) + return + } + require.NoError(t, err) + }) + } +} + +func TestTidyIdenticalCopiesKeepPinsOnColdCache(t *testing.T) { + t.Parallel() + fixture := newTidyCollisionFixture(t, "pinned git", "", map[string]string{"api/common/v1/types.proto": tidyCommonProto}, nil) + require.NoError(t, modules.Tidy(t.Context(), fixture.consumer, fixture.cache)) + manifest, lock := importRootsRead(t, fixture.consumer, v1.ModuleFile), importRootsRead(t, fixture.consumer, v1.LockFile) + + err := modules.Tidy(t.Context(), fixture.consumer, gitmodules.New(t.TempDir())) + + require.NoError(t, err) + assert.Equal(t, manifest, importRootsRead(t, fixture.consumer, v1.ModuleFile)) + assert.Equal(t, lock, importRootsRead(t, fixture.consumer, v1.LockFile)) +} + +func TestTidyEqualCopiesPromoteOnlyCanonicalProvider(t *testing.T) { + t.Parallel() + fixture := newTidyCollisionFixture(t, "pinned git", "", map[string]string{"api/common/v1/types.proto": tidyCommonProto}, nil) + manifest := string(importRootsRead(t, fixture.consumer, v1.ModuleFile)) + for _, commit := range fixture.commits { + manifest = strings.Replace(manifest, commit+"\n", commit+" // indirect\n", 1) + } + importRootsWrite(t, fixture.consumer, v1.ModuleFile, manifest) + importRootsWrite(t, fixture.consumer, "api/consumer/request.proto", "syntax = \"proto3\"; import \"common/v1/types.proto\"; message Request { common.v1.Common value = 1; }") + + err := modules.Tidy(t.Context(), fixture.consumer, fixture.cache) + + require.NoError(t, err) + updated := string(importRootsRead(t, fixture.consumer, v1.ModuleFile)) + assert.Contains(t, updated, "require "+fixture.producer+" "+fixture.commits[fixture.producer]+" // indirect\n") + assert.Contains(t, updated, "require "+fixture.common+" "+fixture.commits[fixture.common]+"\n") +} + +func TestTidyRenamePreservesCanonicalSharedProvider(t *testing.T) { + t.Parallel() + tests := []struct { + name string + consumerCopy bool + producerFirst bool + wantRewrites int + }{ + {name: "unchanged dependency owns the import", wantRewrites: 1}, + {name: "consumer owns the import", consumerCopy: true, wantRewrites: 1}, + {name: "updated dependency owned the import", producerFirst: true, wantRewrites: 2}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + fixture := newTidyCollisionFixture(t, "pinned git", "", map[string]string{"api/common/v1/types.proto": tidyCommonProto}, nil) + if tt.consumerCopy { + importRootsWrite(t, fixture.consumer, "api/common/v1/types.proto", tidyCommonProto) + } + require.NoError(t, modules.Tidy(t.Context(), fixture.consumer, fixture.cache)) + if tt.producerFirst { + // A valid lock's module order is the previous source precedence. + // Put the producer's copy first to exercise the opposite owner. + lock, err := modules.ReadLock(filepath.Join(fixture.consumer, v1.LockFile)) + require.NoError(t, err) + slices.Reverse(lock.Modules) + raw, err := yaml.Marshal(lock) + require.NoError(t, err) + importRootsWrite(t, fixture.consumer, v1.LockFile, string(raw)) + service := string(importRootsRead(t, fixture.producer, "api/service/v1/service.proto")) + importRootsWrite(t, fixture.producer, "api/service/v1/service.proto", strings.Replace(service, "common/v1/types.proto", "v1/types.proto", 1)) + } + consumer := string(importRootsRead(t, fixture.consumer, "api/consumer/request.proto")) + manifest := string(importRootsRead(t, fixture.producer, v1.ModuleFile)) + manifest = strings.Replace(manifest, "roots api\n", "roots api/service api/common\n", 1) + importRootsWrite(t, fixture.producer, v1.ModuleFile, manifest) + commit := tidyCollisionCommit(t, fixture.producer) + manifest = strings.Replace(string(importRootsRead(t, fixture.consumer, v1.ModuleFile)), fixture.commits[fixture.producer], commit, 1) + importRootsWrite(t, fixture.consumer, v1.ModuleFile, manifest) + + report, err := modules.TidyWithReport(t.Context(), fixture.consumer, fixture.cache) + + require.NoError(t, err) + require.Len(t, report.Imports, tt.wantRewrites) + want := strings.Replace(consumer, "service/v1/service.proto", "v1/service.proto", 1) + if tt.producerFirst { + want = strings.Replace(want, "common/v1/types.proto", "v1/types.proto", 1) + } + assert.Equal(t, want, string(importRootsRead(t, fixture.consumer, "api/consumer/request.proto"))) + }) + } +} + +func TestTidyOverlayCapturesMetadataBeforeRemoteAcquisition(t *testing.T) { + t.Parallel() + tests := []struct { + name string + initial map[string]string + change map[string]string + }{ + {name: "native roots", change: map[string]string{v1.ModuleFile: "roots other"}}, + {name: "Buf workspace member", initial: map[string]string{ + "buf.work.yaml": "version: v1\ndirectories: [api]\n", + "api/buf.yaml": "version: v1\n", + }, change: map[string]string{"api/buf.yaml": "version: v1\nbuild:\n excludes: [common]\n"}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + fixture := newTidyCollisionFixture(t, "local replace", "", map[string]string{"api/common/v1/types.proto": tidyCommonProto}, nil) + manifest := string(importRootsRead(t, fixture.consumer, v1.ModuleFile)) + manifest = strings.Replace(manifest, "replace "+fixture.common+" => "+fixture.common+"\n", "", 1) + importRootsWrite(t, fixture.consumer, v1.ModuleFile, manifest) + if len(tt.initial) > 0 { + require.NoError(t, os.Remove(filepath.Join(fixture.producer, v1.ModuleFile))) + importRootsWrite(t, fixture.producer, "easyp.yaml", "deps: ["+fixture.common+"@"+fixture.commits[fixture.common]+"]\n") + for name, raw := range tt.initial { + importRootsWrite(t, fixture.producer, name, raw) + } + } else { + importRootsWrite(t, fixture.producer, "other/service/v1/service.proto", string(importRootsRead(t, fixture.producer, "api/service/v1/service.proto"))) + } + consumer := importRootsRead(t, fixture.consumer, "api/consumer/request.proto") + spy := &tidyOverlayAcquisitionMutation{Cache: fixture.cache, mutate: func() { + for name, raw := range tt.change { + if name == v1.ModuleFile { + raw = strings.Replace(string(importRootsRead(t, fixture.producer, name)), "roots api", raw, 1) + } + importRootsWrite(t, fixture.producer, name, raw) + } + }} + + err := modules.Tidy(t.Context(), fixture.consumer, spy) + + require.True(t, spy.changed, "mutation must happen during remote acquisition") + require.ErrorIs(t, err, sourceview.ErrChanged) + assert.Equal(t, manifest, string(importRootsRead(t, fixture.consumer, v1.ModuleFile))) + assert.Equal(t, consumer, importRootsRead(t, fixture.consumer, "api/consumer/request.proto")) + assert.NoFileExists(t, filepath.Join(fixture.consumer, v1.LockFile)) + }) + } +} + +type tidyOverlayAcquisitionMutation struct { + *gitmodules.Cache + mutate func() + changed bool +} + +func (repository *tidyOverlayAcquisitionMutation) Install(ctx context.Context, lock v1.Lock) error { + if !repository.changed && len(lock.Modules) > 0 { + repository.changed = true + repository.mutate() + } + return repository.Cache.Install(ctx, lock) +} + +func TestTidyOverlayWithRelativeCacheDirectory(t *testing.T) { + // t.Chdir excludes parallel tests while the relative cache is in use. + t.Chdir(t.TempDir()) + fixture := newTidyCollisionFixture(t, "local replace", "", map[string]string{"api/common/v1/types.proto": tidyCommonProto}, nil) + manifest := string(importRootsRead(t, fixture.consumer, v1.ModuleFile)) + manifest = strings.Replace(manifest, "replace "+fixture.common+" => "+fixture.common+"\n", "", 1) + importRootsWrite(t, fixture.consumer, v1.ModuleFile, manifest) + + err := modules.Tidy(t.Context(), fixture.consumer, gitmodules.New("cache")) + + require.NoError(t, err) + assert.Equal(t, manifest, string(importRootsRead(t, fixture.consumer, v1.ModuleFile))) + assert.NoFileExists(t, filepath.Join(fixture.consumer, v1.LockFile)) +} + +func TestTidyOverlayRechecksIntrinsicRootEvidence(t *testing.T) { + t.Parallel() + for _, mutation := range []string{"change unused inference source", "add new intrinsic root", "add nested producer metadata"} { + t.Run(mutation, func(t *testing.T) { + t.Parallel() + fixture := newTidyCollisionFixture(t, "local replace", "", map[string]string{"api/common/v1/types.proto": tidyCommonProto}, nil) + manifest := string(importRootsRead(t, fixture.consumer, v1.ModuleFile)) + manifest = strings.Replace(manifest, "replace "+fixture.common+" => "+fixture.common+"\n", "", 1) + importRootsWrite(t, fixture.consumer, v1.ModuleFile, manifest) + require.NoError(t, os.Remove(filepath.Join(fixture.producer, v1.ModuleFile))) + importRootsWrite(t, fixture.producer, "api/request.proto", "syntax = \"proto3\"; import \"svc.proto\";") + importRootsWrite(t, fixture.producer, "api/svc.proto", "syntax = \"proto3\"; package service; message Service {}") + const consumer = "syntax = \"proto3\"; import \"svc.proto\"; import \"common/v1/types.proto\"; message Request { service.Service service = 1; common.v1.Common common = 2; }" + importRootsWrite(t, fixture.consumer, "api/consumer/request.proto", consumer) + spy := &tidyCollisionMutation{Cache: fixture.cache, at: 1, mutate: func(v1.Lock) { + if mutation == "change unused inference source" { + importRootsWrite(t, fixture.producer, "api/request.proto", "syntax = \"proto3\"; import \"api/svc.proto\";") + return + } + if mutation == "add nested producer metadata" { + importRootsWrite(t, fixture.producer, "newmodule/protobuf.mod", "module "+fixture.producer+"\nroots .\n") + return + } + importRootsWrite(t, fixture.producer, "schema/request.proto", "syntax = \"proto3\"; import \"svc.proto\";") + importRootsWrite(t, fixture.producer, "schema/svc.proto", "syntax = \"proto3\"; package other; message Other {}") + }} + + err := modules.Tidy(t.Context(), fixture.consumer, spy) + + require.Equal(t, 1, spy.checks, "mutation must happen after local root inference") + require.ErrorIs(t, err, sourceview.ErrChanged) + assert.Equal(t, manifest, string(importRootsRead(t, fixture.consumer, v1.ModuleFile))) + assert.Equal(t, consumer, string(importRootsRead(t, fixture.consumer, "api/consumer/request.proto"))) + assert.NoFileExists(t, filepath.Join(fixture.consumer, v1.LockFile)) + }) + } +} + +func TestTidyOverlayAcceptsImportRootDirectoryAlias(t *testing.T) { + t.Parallel() + for _, mode := range []string{"native", "Buf"} { + t.Run(mode, func(t *testing.T) { + t.Parallel() + fixture := newTidyCollisionFixture(t, "local replace", "", nil, nil) + require.NoError(t, os.Symlink("api", filepath.Join(fixture.producer, "proto"))) + if mode == "native" { + manifest := string(importRootsRead(t, fixture.producer, v1.ModuleFile)) + importRootsWrite(t, fixture.producer, v1.ModuleFile, strings.Replace(manifest, "roots api", "roots proto", 1)) + } else { + require.NoError(t, os.Remove(filepath.Join(fixture.producer, v1.ModuleFile))) + importRootsWrite(t, fixture.producer, "buf.yaml", "version: v2\nmodules: [{path: proto}]\n") + importRootsWrite(t, fixture.producer, "easyp.yaml", "deps: ["+fixture.common+"@"+fixture.commits[fixture.common]+"]\n") + } + + err := modules.Tidy(t.Context(), fixture.consumer, fixture.cache) + + require.NoError(t, err) + assert.NoFileExists(t, filepath.Join(fixture.consumer, v1.LockFile)) + }) + } +} + +func TestTidyOverlayRechecksAbsentMetadataThroughDirectoryAlias(t *testing.T) { + t.Parallel() + fixture := newTidyCollisionFixture(t, "local replace", "", nil, nil) + manifest := strings.Replace(string(importRootsRead(t, fixture.consumer, v1.ModuleFile)), "replace "+fixture.common+" => "+fixture.common+"\n", "", 1) + importRootsWrite(t, fixture.consumer, v1.ModuleFile, manifest) + producer := string(importRootsRead(t, fixture.producer, v1.ModuleFile)) + importRootsWrite(t, fixture.producer, v1.ModuleFile, strings.Replace(producer, "roots api", "roots proto", 1)) + require.NoError(t, os.Symlink("api", filepath.Join(fixture.producer, "proto"))) + importRootsWrite(t, fixture.producer, "other/service/v1/service.proto", string(importRootsRead(t, fixture.producer, "api/service/v1/service.proto"))) + spy := &tidyCollisionMutation{Cache: fixture.cache, at: 1, mutate: func(v1.Lock) { + require.NoError(t, os.Remove(filepath.Join(fixture.producer, "proto"))) + require.NoError(t, os.Symlink("other", filepath.Join(fixture.producer, "proto"))) + }} + + err := modules.Tidy(t.Context(), fixture.consumer, spy) + + require.Equal(t, 1, spy.checks, "mutation must happen after absent metadata capture") + require.ErrorIs(t, err, sourceview.ErrChanged) + assert.Equal(t, manifest, string(importRootsRead(t, fixture.consumer, v1.ModuleFile))) + assert.NoFileExists(t, filepath.Join(fixture.consumer, v1.LockFile)) +} + +func TestTidyOverlayRejectsDanglingMetadataFileAlias(t *testing.T) { + t.Parallel() + fixture := newTidyCollisionFixture(t, "local replace", "", nil, nil) + require.NoError(t, os.Symlink("missing.yaml", filepath.Join(fixture.producer, "api/easyp.yaml"))) + + err := modules.Tidy(t.Context(), fixture.consumer, fixture.cache) + + require.Error(t, err) + assert.Contains(t, err.Error(), "api/easyp.yaml") + assert.NoFileExists(t, filepath.Join(fixture.consumer, v1.LockFile)) +} diff --git a/internal/modules/tidy_collisions_test.go b/internal/modules/tidy_collisions_test.go new file mode 100644 index 00000000..cd154fb3 --- /dev/null +++ b/internal/modules/tidy_collisions_test.go @@ -0,0 +1,312 @@ +package modules_test + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/easyp-tech/easyp/internal/adapters/gitmodules" + v1 "github.com/easyp-tech/easyp/internal/config/v1" + "github.com/easyp-tech/easyp/internal/modules" + "github.com/easyp-tech/easyp/internal/sourceview" +) + +const tidyCommonProto = "syntax = \"proto3\"; package common.v1; message Common { string value = 1; }\n" + +func TestTidyDuplicateImportClosure(t *testing.T) { + t.Parallel() + tests := []struct { + name string + copies map[string]string + common map[string]string + consumerCopy string + roots string + wantConflict string + }{ + {name: "one direct and transitive provider"}, + {name: "unused identical duplicates", copies: map[string]string{"api/tests/unused.proto": "syntax = \"proto3\"; package unused;"}, common: map[string]string{"proto/tests/unused.proto": "syntax = \"proto3\"; package unused;"}}, + {name: "unused differing duplicates", copies: map[string]string{"api/tests/unused.proto": "syntax = \"proto3\"; package first;"}, common: map[string]string{"proto/tests/unused.proto": "syntax = \"proto3\"; package second;"}}, + {name: "unused malformed duplicate", copies: map[string]string{"api/tests/unused.proto": "this source is never imported"}, common: map[string]string{"proto/tests/unused.proto": "syntax = \"proto3\"; package unused;"}}, + {name: "reachable identical copies", copies: map[string]string{"api/common/v1/types.proto": tidyCommonProto}}, + {name: "reachable differing copies", copies: map[string]string{"api/common/v1/types.proto": strings.Replace(tidyCommonProto, "string value", "int64 value", 1)}, wantConflict: "common/v1/types.proto"}, + {name: "reachable comment difference", copies: map[string]string{"api/common/v1/types.proto": tidyCommonProto + "// another contract source\n"}, wantConflict: "common/v1/types.proto"}, + {name: "identical copies in another root", roots: "api extra", copies: map[string]string{"extra/common/v1/types.proto": tidyCommonProto}}, + {name: "conflicting copies in another root", roots: "api extra", copies: map[string]string{"extra/common/v1/types.proto": strings.Replace(tidyCommonProto, "string value", "bytes value", 1)}, wantConflict: "common/v1/types.proto"}, + {name: "identical consumer copy", consumerCopy: tidyCommonProto}, + {name: "conflicting consumer copy", consumerCopy: strings.Replace(tidyCommonProto, "string value", "bool value", 1), wantConflict: "common/v1/types.proto"}, + } + for _, mode := range []string{"pinned git", "local replace"} { + for _, tt := range tests { + t.Run(mode+"/"+tt.name, func(t *testing.T) { + t.Parallel() + fixture := newTidyCollisionFixture(t, mode, tt.roots, tt.copies, tt.common) + if tt.consumerCopy != "" { + importRootsWrite(t, fixture.consumer, "api/common/v1/types.proto", tt.consumerCopy) + } + beforeManifest := importRootsRead(t, fixture.consumer, v1.ModuleFile) + beforeSource := importRootsRead(t, fixture.consumer, "api/consumer/request.proto") + beforeProducer := importRootsRead(t, fixture.producer, "api/service/v1/service.proto") + beforeCommon := importRootsRead(t, fixture.common, "proto/common/v1/types.proto") + // Even invalid/old lock bytes must remain untouched in replacement mode. + const localLock = "version: 1\nmodules: []\n# local developer lock\n" + if mode == "local replace" { + importRootsWrite(t, fixture.consumer, v1.LockFile, localLock) + } + + err := modules.Tidy(t.Context(), fixture.consumer, fixture.cache) + + assert.Equal(t, beforeSource, importRootsRead(t, fixture.consumer, "api/consumer/request.proto")) + assert.Equal(t, beforeProducer, importRootsRead(t, fixture.producer, "api/service/v1/service.proto")) + assert.Equal(t, beforeCommon, importRootsRead(t, fixture.common, "proto/common/v1/types.proto")) + if tt.wantConflict != "" { + require.ErrorContains(t, err, "duplicate import path") + assert.ErrorContains(t, err, tt.wantConflict) + assert.Equal(t, beforeManifest, importRootsRead(t, fixture.consumer, v1.ModuleFile)) + if mode == "pinned git" { + assert.NoFileExists(t, filepath.Join(fixture.consumer, v1.LockFile)) + } + } else { + require.NoError(t, err) + if mode == "pinned git" { + lock, err := modules.ReadLock(filepath.Join(fixture.consumer, v1.LockFile)) + require.NoError(t, err) + require.Len(t, lock.Modules, 2) + for _, entry := range lock.Modules { + assert.Equal(t, fixture.commits[entry.Source], entry.Commit) + assert.Equal(t, entry.Commit, entry.Version) + } + } + manifest, lock := importRootsRead(t, fixture.consumer, v1.ModuleFile), importRootsRead(t, fixture.consumer, v1.LockFile) + require.NoError(t, modules.Tidy(t.Context(), fixture.consumer, fixture.cache)) + assert.Equal(t, manifest, importRootsRead(t, fixture.consumer, v1.ModuleFile)) + assert.Equal(t, lock, importRootsRead(t, fixture.consumer, v1.LockFile)) + } + if mode == "local replace" { + assert.Equal(t, beforeManifest, importRootsRead(t, fixture.consumer, v1.ModuleFile)) + assert.Equal(t, localLock, string(importRootsRead(t, fixture.consumer, v1.LockFile))) + } + }) + } + } +} + +type tidyCollisionFixture struct { + consumer string + producer string + common string + commits map[string]string + cache *gitmodules.Cache +} + +func newTidyCollisionFixture(t *testing.T, mode, roots string, copies, commonFiles map[string]string) tidyCollisionFixture { + t.Helper() + common := map[string]string{"proto/common/v1/types.proto": tidyCommonProto} + for name, content := range commonFiles { + common[name] = content + } + commonRepo, _ := importRootsRepository(t, common, nil) + importRootsWrite(t, commonRepo, v1.ModuleFile, "module "+commonRepo+"\nroots proto\n") + commonCommit := tidyCollisionCommit(t, commonRepo) + service := "syntax = \"proto3\"; package service.v1; import public \"common/v1/types.proto\"; message Service { common.v1.Common common = 1; }\n" + files := map[string]string{"api/service/v1/service.proto": service} + for name, content := range copies { + files[name] = content + } + producer, _ := importRootsRepository(t, files, nil) + if roots == "" { + roots = "api" + } + importRootsWrite(t, producer, v1.ModuleFile, "module "+producer+"\nroots "+roots+"\nrequire "+commonRepo+" "+commonCommit+"\n") + producerCommit := tidyCollisionCommit(t, producer) + root := t.TempDir() + manifest := "module example.test/consumer\nroots api\nrequire " + producer + " " + producerCommit + "\nrequire " + commonRepo + " " + commonCommit + "\n" + if mode == "local replace" { + manifest += "replace " + producer + " => " + producer + "\nreplace " + commonRepo + " => " + commonRepo + "\n" + } + importRootsWrite(t, root, v1.ModuleFile, manifest) + importRootsWrite(t, root, "api/consumer/request.proto", "syntax = \"proto3\"; package consumer; import \"service/v1/service.proto\"; import \"common/v1/types.proto\"; message Request { service.v1.Service service = 1; common.v1.Common common = 2; }\n") + importRootsWrite(t, root, v1.GenerateFile, "version: v1\nplugins: [{name: python, out: gen/python}]\n") + return tidyCollisionFixture{consumer: root, producer: producer, common: commonRepo, commits: map[string]string{producer: producerCommit, commonRepo: commonCommit}, cache: gitmodules.New(t.TempDir())} +} + +func tidyCollisionCommit(t *testing.T, directory string) string { + t.Helper() + importRootsGit(t, directory, "add", ".") + importRootsGit(t, directory, "commit", "--quiet", "-m", "dependency fixture metadata") + return importRootsGit(t, directory, "rev-parse", "HEAD") +} + +func TestTidyComparedProtoClosure(t *testing.T) { + t.Parallel() + for _, conflict := range []bool{false, true} { + t.Run(map[bool]string{false: "identical transitive copies", true: "different transitive copies"}[conflict], func(t *testing.T) { + t.Parallel() + parent := "syntax = \"proto3\"; package common.v1; import weak \"common/v1/detail.proto\"; message Common { Detail value = 1; }\n" + detail := "syntax = \"proto3\"; package common.v1; message Detail { string value = 1; }\n" + copiedDetail := detail + if conflict { + copiedDetail = strings.Replace(detail, "string value", "bytes value", 1) + } + fixture := newTidyCollisionFixture(t, "pinned git", "", map[string]string{"api/common/v1/types.proto": parent, "api/common/v1/detail.proto": copiedDetail}, map[string]string{"proto/common/v1/types.proto": parent, "proto/common/v1/detail.proto": detail}) + before := importRootsRead(t, fixture.consumer, v1.ModuleFile) + + err := modules.Tidy(t.Context(), fixture.consumer, fixture.cache) + + if conflict { + require.ErrorContains(t, err, "duplicate import path") + assert.ErrorContains(t, err, "common/v1/detail.proto") + assert.Equal(t, before, importRootsRead(t, fixture.consumer, v1.ModuleFile)) + assert.NoFileExists(t, filepath.Join(fixture.consumer, v1.LockFile)) + return + } + require.NoError(t, err) + lock, err := modules.ReadLock(filepath.Join(fixture.consumer, v1.LockFile)) + require.NoError(t, err) + require.Len(t, lock.Modules, 2) + }) + } +} + +func TestTidyDuplicateConflictPreservesExistingLock(t *testing.T) { + t.Parallel() + fixture := newTidyCollisionFixture(t, "pinned git", "", nil, nil) + require.NoError(t, modules.Tidy(t.Context(), fixture.consumer, fixture.cache)) + importRootsWrite(t, fixture.producer, "api/common/v1/types.proto", strings.Replace(tidyCommonProto, "string value", "int32 value", 1)) + commit := tidyCollisionCommit(t, fixture.producer) + manifest := string(importRootsRead(t, fixture.consumer, v1.ModuleFile)) + manifest = strings.Replace(manifest, fixture.commits[fixture.producer], commit, 1) + importRootsWrite(t, fixture.consumer, v1.ModuleFile, manifest) + lock, source := importRootsRead(t, fixture.consumer, v1.LockFile), importRootsRead(t, fixture.consumer, "api/consumer/request.proto") + + err := modules.Tidy(t.Context(), fixture.consumer, fixture.cache) + + require.ErrorContains(t, err, "duplicate import path") + assert.Equal(t, manifest, string(importRootsRead(t, fixture.consumer, v1.ModuleFile))) + assert.Equal(t, lock, importRootsRead(t, fixture.consumer, v1.LockFile)) + assert.Equal(t, source, importRootsRead(t, fixture.consumer, "api/consumer/request.proto")) + _, err = os.Stat(filepath.Join(fixture.consumer, v1.LockFile)) + require.NoError(t, err) +} + +func TestTidyRenamesImportWithUnusedDuplicateInventory(t *testing.T) { + t.Parallel() + for _, inventory := range []string{"old", "new"} { + t.Run(inventory, func(t *testing.T) { + t.Parallel() + roots, nextRoots := "api extra", "api/service extra" + if inventory == "new" { + roots, nextRoots = nextRoots, roots + } + fixture := newTidyCollisionFixture(t, "pinned git", roots, map[string]string{ + "api/tests/unused.proto": "syntax = \"proto3\"; package first;", + "extra/tests/unused.proto": "syntax = \"proto3\"; package second;", + }, nil) + consumer := string(importRootsRead(t, fixture.consumer, "api/consumer/request.proto")) + beforeName, afterName := "service/v1/service.proto", "v1/service.proto" + if inventory == "new" { + beforeName, afterName = afterName, beforeName + } + consumer = strings.Replace(consumer, "service/v1/service.proto", beforeName, 1) + importRootsWrite(t, fixture.consumer, "api/consumer/request.proto", consumer) + require.NoError(t, modules.Tidy(t.Context(), fixture.consumer, fixture.cache)) + manifest := string(importRootsRead(t, fixture.producer, v1.ModuleFile)) + manifest = strings.Replace(manifest, "roots "+roots+"\n", "roots "+nextRoots+"\n", 1) + importRootsWrite(t, fixture.producer, v1.ModuleFile, manifest) + commit := tidyCollisionCommit(t, fixture.producer) + manifest = strings.Replace(string(importRootsRead(t, fixture.consumer, v1.ModuleFile)), fixture.commits[fixture.producer], commit, 1) + importRootsWrite(t, fixture.consumer, v1.ModuleFile, manifest) + + report, err := modules.TidyWithReport(t.Context(), fixture.consumer, fixture.cache) + + require.NoError(t, err) + require.Len(t, report.Imports, 1) + assert.Equal(t, beforeName, report.Imports[0].From) + assert.Equal(t, afterName, report.Imports[0].To) + assert.Equal(t, strings.Replace(consumer, beforeName, afterName, 1), string(importRootsRead(t, fixture.consumer, "api/consumer/request.proto"))) + assert.Equal(t, manifest, string(importRootsRead(t, fixture.consumer, v1.ModuleFile))) + }) + } +} + +type tidyCollisionMutation struct { + *gitmodules.Cache + checks int + at int + mutate func(v1.Lock) +} + +func (cache *tidyCollisionMutation) VerifyCached(ctx context.Context, lock v1.Lock) error { + cache.checks++ + err := cache.Cache.VerifyCached(ctx, lock) + if err == nil && cache.checks == cache.at { + cache.mutate(lock) + } + return err +} + +func TestTidyDuplicateComparisonRechecksInputs(t *testing.T) { + t.Parallel() + for _, mode := range []string{"pinned git", "mixed overlay", "mixed overlay remote copy"} { + for _, mutation := range []string{"changed copy", "added copy", "added differing copy"} { + t.Run(mode+"/"+mutation, func(t *testing.T) { + t.Parallel() + copies := map[string]string{} + if mutation == "changed copy" { + copies["api/common/v1/types.proto"] = tidyCommonProto + } + fixtureMode := mode + if mode != "pinned git" { + fixtureMode = "local replace" + } + fixture := newTidyCollisionFixture(t, fixtureMode, "", copies, nil) + if mode != "pinned git" { + manifest := string(importRootsRead(t, fixture.consumer, v1.ModuleFile)) + remote := fixture.common + if mode == "mixed overlay remote copy" { + remote = fixture.producer + } + manifest = strings.Replace(manifest, "replace "+remote+" => "+remote+"\n", "", 1) + importRootsWrite(t, fixture.consumer, v1.ModuleFile, manifest) + importRootsWrite(t, fixture.consumer, v1.LockFile, "version: 1\nmodules: []\n") + } + require.NoError(t, modules.Tidy(t.Context(), fixture.consumer, fixture.cache)) + manifest, lock := importRootsRead(t, fixture.consumer, v1.ModuleFile), importRootsRead(t, fixture.consumer, v1.LockFile) + source := importRootsRead(t, fixture.consumer, "api/consumer/request.proto") + at := 2 + if mode != "pinned git" { + at = 1 + } + spy := &tidyCollisionMutation{Cache: fixture.cache, at: at, mutate: func(current v1.Lock) { + directory := fixture.producer + if mode != "mixed overlay" { + for _, entry := range current.Modules { + if entry.Source == fixture.producer { + var err error + directory, _, err = fixture.cache.Cached(entry) + require.NoError(t, err) + } + } + } + data := tidyCommonProto + if mutation != "added copy" { + data = strings.Replace(data, "string value", "bool value", 1) + } + importRootsWrite(t, directory, "api/common/v1/types.proto", data) + }} + + err := modules.Tidy(t.Context(), fixture.consumer, spy) + + require.ErrorIs(t, err, sourceview.ErrChanged) + assert.GreaterOrEqual(t, spy.checks, at) + assert.Equal(t, manifest, importRootsRead(t, fixture.consumer, v1.ModuleFile)) + assert.Equal(t, lock, importRootsRead(t, fixture.consumer, v1.LockFile)) + assert.Equal(t, source, importRootsRead(t, fixture.consumer, "api/consumer/request.proto")) + }) + } + } +} diff --git a/internal/modules/tidy_historical_collisions_test.go b/internal/modules/tidy_historical_collisions_test.go new file mode 100644 index 00000000..523f3592 --- /dev/null +++ b/internal/modules/tidy_historical_collisions_test.go @@ -0,0 +1,125 @@ +package modules_test + +import ( + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" + "github.com/easyp-tech/easyp/internal/modules" +) + +const tidyHistoricalCurrentProto = "syntax = \"proto3\"; package model; message New { string value = 1; }\n" +const tidyHistoricalConsumerProto = "syntax = \"proto3\"; package consumer; import \"new/v1/model.proto\"; message Request { model.New value = 1; }\n" + +func TestTidyHistoricalCollisionsRetainUnambiguousCurrentImport(t *testing.T) { + t.Parallel() + tests := []struct { + name string + conflictingRoots bool + retainProducer bool + }{ + {name: "remove conflicting provider"}, + {name: "remove provider with conflicting roots", conflictingRoots: true}, + {name: "current copies have identical bytes", retainProducer: true}, + } + for _, tt := range tests { + for _, reverse := range []bool{false, true} { + name := tt.name + "/original lock order" + if reverse { + name = tt.name + "/reversed lock order" + } + t.Run(name, func(t *testing.T) { + t.Parallel() + fixture := newTidyHistoricalCollisionFixture(t, tt.conflictingRoots, reverse) + manifest := "module example.test/consumer\nroots api\nrequire " + fixture.common + " " + fixture.commits[fixture.common] + "\n" + if tt.retainProducer { + importRootsWrite(t, fixture.producer, "api/new/v1/model.proto", tidyHistoricalCurrentProto) + commit := tidyCollisionCommit(t, fixture.producer) + manifest += "require " + fixture.producer + " " + commit + "\n" + } + importRootsWrite(t, fixture.consumer, v1.ModuleFile, manifest) + importRootsWrite(t, fixture.consumer, "api/consumer/request.proto", tidyHistoricalConsumerProto) + + // The same current project is valid without historical ambiguity. + fresh := t.TempDir() + importRootsWrite(t, fresh, v1.ModuleFile, manifest) + importRootsWrite(t, fresh, "api/consumer/request.proto", tidyHistoricalConsumerProto) + require.NoError(t, modules.Tidy(t.Context(), fresh, fixture.cache)) + + report, err := modules.TidyWithReport(t.Context(), fixture.consumer, fixture.cache) + + require.NoError(t, err) + assert.Empty(t, report.Imports, "an unambiguous current name does not need a historical rename") + assert.Equal(t, tidyHistoricalConsumerProto, string(importRootsRead(t, fixture.consumer, "api/consumer/request.proto"))) + assert.Equal(t, manifest, string(importRootsRead(t, fixture.consumer, v1.ModuleFile))) + lock := importRootsRead(t, fixture.consumer, v1.LockFile) + assert.Equal(t, importRootsRead(t, fresh, v1.LockFile), lock) + require.NoError(t, modules.Tidy(t.Context(), fixture.consumer, fixture.cache)) + assert.Equal(t, lock, importRootsRead(t, fixture.consumer, v1.LockFile)) + }) + } + } +} + +func TestTidyHistoricalCollisionsCannotGuessCurrentBinding(t *testing.T) { + t.Parallel() + for _, mode := range []string{"missing current name", "conflicting current copies"} { + t.Run(mode, func(t *testing.T) { + t.Parallel() + fixture := newTidyHistoricalCollisionFixture(t, false, false) + manifest := "module example.test/consumer\nroots api\n" + commonCommit := fixture.commits[fixture.common] + if mode == "missing current name" { + module := string(importRootsRead(t, fixture.common, v1.ModuleFile)) + importRootsWrite(t, fixture.common, v1.ModuleFile, strings.Replace(module, "roots proto", "roots proto/common proto/new", 1)) + commonCommit = tidyCollisionCommit(t, fixture.common) + } + manifest += "require " + fixture.common + " " + commonCommit + "\n" + if mode == "conflicting current copies" { + importRootsWrite(t, fixture.producer, "api/unrelated.proto", "syntax = \"proto3\"; package unrelated; message Added {}\n") + producerCommit := tidyCollisionCommit(t, fixture.producer) + manifest += "require " + fixture.producer + " " + producerCommit + "\n" + } + importRootsWrite(t, fixture.consumer, v1.ModuleFile, manifest) + importRootsWrite(t, fixture.consumer, "api/consumer/request.proto", tidyHistoricalConsumerProto) + lock := importRootsRead(t, fixture.consumer, v1.LockFile) + + report, err := modules.TidyWithReport(t.Context(), fixture.consumer, fixture.cache) + + require.ErrorContains(t, err, "old locked namespace has conflicting import") + assert.Contains(t, err.Error(), "new/v1/model.proto") + assert.Empty(t, report.Imports) + assert.Equal(t, tidyHistoricalConsumerProto, string(importRootsRead(t, fixture.consumer, "api/consumer/request.proto"))) + assert.Equal(t, manifest, string(importRootsRead(t, fixture.consumer, v1.ModuleFile))) + assert.Equal(t, lock, importRootsRead(t, fixture.consumer, v1.LockFile)) + }) + } +} + +func newTidyHistoricalCollisionFixture(t *testing.T, conflictingRoots, reverse bool) tidyCollisionFixture { + t.Helper() + const previous = "syntax = \"proto3\"; package previous; message Old { int32 value = 1; }\n" + copies := map[string]string{"api/new/v1/model.proto": previous} + roots := "" + if conflictingRoots { + roots = "api extra" + copies["extra/new/v1/model.proto"] = "syntax = \"proto3\"; package other; message Old { bool value = 1; }\n" + } + fixture := newTidyCollisionFixture(t, "pinned git", roots, copies, map[string]string{"proto/new/v1/model.proto": tidyHistoricalCurrentProto}) + require.NoError(t, modules.Tidy(t.Context(), fixture.consumer, fixture.cache), "differing copies are initially unused") + if reverse { + lock, err := modules.ReadLock(filepath.Join(fixture.consumer, v1.LockFile)) + require.NoError(t, err) + slices.Reverse(lock.Modules) + raw, err := yaml.Marshal(lock) + require.NoError(t, err) + importRootsWrite(t, fixture.consumer, v1.LockFile, string(raw)) + } + return fixture +} diff --git a/internal/modules/tidy_imports.go b/internal/modules/tidy_imports.go new file mode 100644 index 00000000..9cb057a8 --- /dev/null +++ b/internal/modules/tidy_imports.go @@ -0,0 +1,139 @@ +package modules + +import ( + "bytes" + "context" + "errors" + "fmt" + "os" + "path/filepath" + "slices" + + "github.com/easyp-tech/easyp/internal/sourceview" +) + +// Resolution belongs to a logical import, not to a particular cached directory. +// Equal copies keep the ordinary root-order precedence and canonical owner. +type tidyImportResolution struct { + source v1ImportSource + root SourceRoot + candidates []v1ImportSource +} + +func (view *tidySourceView) importCandidates(ctx context.Context, name string) (tidyImportResolution, error) { + resolution := tidyImportResolution{} + // Re-probe used names against the filesystem after cache verification. + // An immutable inventory must not hide a newly added import candidate. + roots := slices.Clone(view.roots) + for index := range roots { + roots[index].index = nil + } + allowed := roots.FileAllowed() + seen := make(map[string]bool) + for _, root := range roots { + err := ctx.Err() + if err != nil { + return tidyImportResolution{}, fmt.Errorf("Err: %w", err) + } + candidate, err := resolveV1ImportInRoot(name, root, roots, allowed) + if errors.Is(err, os.ErrNotExist) { + continue + } + if err != nil { + return tidyImportResolution{}, fmt.Errorf("resolveV1ImportInRoot: %w", err) + } + if seen[candidate.path] { + continue + } + seen[candidate.path] = true + if len(resolution.candidates) == 0 { + resolution.source, resolution.root = candidate, root + } + resolution.candidates = append(resolution.candidates, candidate) + } + if len(resolution.candidates) > 0 { + return resolution, nil + } + source, err := resolveV1BuiltinImport(name) + if err != nil { + return tidyImportResolution{}, fmt.Errorf("resolveV1BuiltinImport: %w", err) + } + resolution.source = source + return resolution, nil +} + +func (view *tidySourceView) resolveImport(ctx context.Context, name string) (tidyImportResolution, error) { + resolution, err := view.importCandidates(ctx, name) + if err != nil { + return tidyImportResolution{}, fmt.Errorf("importCandidates: %w", err) + } + var canonical []byte + for offset, candidate := range resolution.candidates { + data, err := view.readSourceFile(candidate.path) + if err != nil { + return tidyImportResolution{}, fmt.Errorf("readSourceFile: %w", err) + } + if offset == 0 { + canonical = data + continue + } + if !bytes.Equal(canonical, data) { + return tidyImportResolution{}, fmt.Errorf("duplicate import path %q: %s and %s have different contents", name, resolution.source.path, candidate.path) + } + } + view.mu.Lock() + defer view.mu.Unlock() + if previous, exists := view.imports[name]; exists && !slices.Equal(previous.candidates, resolution.candidates) { + return tidyImportResolution{}, fmt.Errorf("import candidates for %q changed: %w", name, sourceview.ErrChanged) + } + view.imports[name] = resolution + return resolution, nil +} + +// Re-probe used names so adding, removing or excluding another candidate cannot +// invalidate deduplication after comparison. File observations check all bytes +// and alias identities, including noncanonical copies, separately. +func (view *tidySourceView) verifyImportCandidates(ctx context.Context) error { + view.mu.Lock() + expected := make(map[string]tidyImportResolution, len(view.imports)) + for name, resolution := range view.imports { + expected[name] = resolution + } + view.mu.Unlock() + names := make([]string, 0, len(expected)) + for name := range expected { + names = append(names, name) + } + slices.Sort(names) + for _, name := range names { + current, err := view.importCandidates(ctx, name) + if err != nil { + return fmt.Errorf("importCandidates: %s: %w", name, err) + } + if !slices.Equal(expected[name].candidates, current.candidates) { + return fmt.Errorf("import candidates for %q changed: %w", name, sourceview.ErrChanged) + } + } + return nil +} + +func (view *tidySourceView) validateConsumerTargets(ctx context.Context, files []string) error { + consumer := view.roots[0].Module + for _, name := range files { + path := filepath.Join(view.observations.requestedRoot, name) + for _, root := range view.roots { + if root.Module != consumer || !sourcePathWithin(path, root.Path) || !root.allows(path, root.Path) { + continue + } + importPath, err := filepath.Rel(root.Path, path) + if err != nil { + return fmt.Errorf("Rel: %w", err) + } + _, err = view.resolveImport(ctx, filepath.ToSlash(importPath)) + if err != nil { + return fmt.Errorf("resolveImport: %s: %w", name, err) + } + } + } + return nil +} diff --git a/internal/modules/tidy_inputs.go b/internal/modules/tidy_inputs.go index 483be19e..be3f34de 100644 --- a/internal/modules/tidy_inputs.go +++ b/internal/modules/tidy_inputs.go @@ -169,6 +169,10 @@ func tidySourceFiles(tx *resolvedFileObservations, roots SourceRoots) ([]string, } func (inputs tidyInputs) capturedImports() (map[string]v1UnresolvedImport, error) { + owned, err := inputs.consumerImportNames() + if err != nil { + return nil, fmt.Errorf("consumerImportNames: %w", err) + } needed := make(map[string]v1UnresolvedImport) for _, file := range inputs.files { observed, captured := inputs.observations.capturedFile(file) @@ -180,6 +184,11 @@ func (inputs tidyInputs) capturedImports() (map[string]v1UnresolvedImport, error return nil, fmt.Errorf("ParseProtoImports: %w", err) } for _, name := range imports { + // Consumer roots precede dependencies. A producer update must not + // rename an import supplied by an observed consumer source. + if owned[name] { + continue + } if _, exists := needed[name]; !exists { needed[name] = v1UnresolvedImport{owner: file, path: name} } @@ -187,3 +196,21 @@ func (inputs tidyInputs) capturedImports() (map[string]v1UnresolvedImport, error } return needed, nil } + +func (inputs tidyInputs) consumerImportNames() (map[string]bool, error) { + names := make(map[string]bool) + for _, file := range inputs.files { + path := filepath.Join(inputs.root, file) + for _, root := range inputs.roots { + if !sourcePathWithin(path, root.Path) || !root.allowsPath(path, root.Path) { + continue + } + name, err := filepath.Rel(root.Path, path) + if err != nil { + return nil, fmt.Errorf("Rel: %w", err) + } + names[filepath.ToSlash(name)] = true + } + } + return names, nil +} diff --git a/internal/modules/tidy_local_sources.go b/internal/modules/tidy_local_sources.go new file mode 100644 index 00000000..62214996 --- /dev/null +++ b/internal/modules/tidy_local_sources.go @@ -0,0 +1,151 @@ +package modules + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "slices" + + moduleconfig "github.com/easyp-tech/easyp/internal/adapters/module_config" + v1 "github.com/easyp-tech/easyp/internal/config/v1" + "github.com/easyp-tech/easyp/internal/sourceview" +) + +// Metadata-free root selection depends on the producer's full intrinsic import +// map, including sources the consumer does not import. Retain that evidence +// separately from tidy's reachable namespace and its fixed-root checks. +type tidyLocalRoots struct { + directory string + module v1.Module + candidate importRootModule +} + +type tidyLocalMetadata struct { + directory string + name string + files map[string]bool +} + +func (view *tidySourceView) readLocalReplacement(directory, name string) (EffectiveModule, error) { + directory, err := filepath.Abs(directory) + if err != nil { + return EffectiveModule{}, fmt.Errorf("Abs: %w", err) + } + err = view.observeModuleMetadata(directory, v1.Module{}) + if err != nil { + return EffectiveModule{}, fmt.Errorf("observeModuleMetadata: %w", err) + } + scope, err := view.dependencyScope(directory) + if err != nil { + return EffectiveModule{}, fmt.Errorf("dependencyScope: %w", err) + } + files := make(map[string]bool) + entry, err := readLocalReplacementWithReader(directory, name, func(file string) ([]byte, error) { + state, err := scope.capture(file, nil) + if err != nil { + return nil, fmt.Errorf("capture: %w", err) + } + if !state.exists { + return nil, &os.PathError{Op: "open", Path: filepath.Join(directory, file), Err: os.ErrNotExist} + } + files[file] = true + return state.data, nil + }) + if err != nil { + return EffectiveModule{}, fmt.Errorf("readLocalReplacementWithReader: %w", err) + } + err = view.observeModuleMetadata(directory, entry.Module) + if err != nil { + return EffectiveModule{}, fmt.Errorf("observeModuleMetadata: %w", err) + } + manifest, _ := scope.capturedFile(v1.ModuleFile) + if !v1.IsModuleManifest(manifest.data) { + view.compatibility = append(view.compatibility, tidyLocalMetadata{directory: directory, name: name, files: files}) + } + return entry, nil +} + +// Fallback metadata adaptation scans nested configs too. Revisit the same +// bounded metadata tree to catch new directories that could change module +// identity, roots or requirements without changing the proto inventory. +func (view *tidySourceView) verifyLocalMetadata(ctx context.Context) error { + for _, evidence := range view.compatibility { + scope, err := view.dependencyScope(evidence.directory) + if err != nil { + return fmt.Errorf("dependencyScope: %w", err) + } + seen := make(map[string]bool) + _, err = moduleconfig.ReadLocalDependencyWithReader(evidence.directory, evidence.name, func(file string) ([]byte, error) { + err := ctx.Err() + if err != nil { + return nil, fmt.Errorf("Err: %w", err) + } + state, err := scope.capture(file, nil) + if err != nil { + return nil, fmt.Errorf("capture: %w", err) + } + if !state.exists { + return nil, &os.PathError{Op: "open", Path: filepath.Join(evidence.directory, file), Err: os.ErrNotExist} + } + if !evidence.files[file] { + return nil, fmt.Errorf("local metadata inventory for %s changed: %w", evidence.name, sourceview.ErrChanged) + } + seen[file] = true + return state.data, nil + }) + if err != nil { + return fmt.Errorf("ReadLocalDependencyWithReader: %w", err) + } + if len(seen) != len(evidence.files) { + return fmt.Errorf("local metadata inventory for %s changed: %w", evidence.name, sourceview.ErrChanged) + } + } + return nil +} + +func (view *tidySourceView) inspectLocalRoots(ctx context.Context, directory string, module v1.Module, hinted bool) (importRootModule, error) { + candidate, err := inspectLocalRootCandidates(ctx, directory, module, hinted) + if err != nil { + return importRootModule{}, fmt.Errorf("inspectLocalRootCandidates: %w", err) + } + if candidate.fixed { + return candidate, nil + } + scope, err := view.dependencyScope(directory) + if err != nil { + return importRootModule{}, fmt.Errorf("dependencyScope: %w", err) + } + for _, file := range candidate.inspection.Files { + state, err := scope.capture(filepath.FromSlash(file.Path), nil) + if err != nil { + return importRootModule{}, fmt.Errorf("capture: %w", err) + } + physical := filepath.Join(scope.canonicalRoot, filepath.FromSlash(state.path)) + if !state.exists || physical != file.Identity || !bytes.Equal(state.data, file.Content) { + return importRootModule{}, fmt.Errorf("local root evidence %s changed: %w", file.Path, sourceview.ErrChanged) + } + } + view.inferred = append(view.inferred, tidyLocalRoots{directory: directory, module: module, candidate: candidate}) + return candidate, nil +} + +func (view *tidySourceView) verifyLocalRoots(ctx context.Context) error { + for _, evidence := range view.inferred { + current, err := inspectLocalRootCandidates(ctx, evidence.directory, evidence.module, false) + if err != nil { + return fmt.Errorf("inspectLocalRootCandidates: %w", err) + } + sameFiles := slices.EqualFunc(evidence.candidate.inspection.Files, current.inspection.Files, func(before, after RootProtoFile) bool { + return before.Path == after.Path && before.Identity == after.Identity && bytes.Equal(before.Content, after.Content) + }) + sameProblems := slices.EqualFunc(evidence.candidate.inspection.Problems, current.inspection.Problems, func(before, after RootPathProblem) bool { + return before.Path == after.Path + }) + if !sameFiles || !sameProblems { + return fmt.Errorf("local root evidence for %s changed: %w", evidence.module.Name, sourceview.ErrChanged) + } + } + return nil +} diff --git a/internal/modules/tidy_operation.go b/internal/modules/tidy_operation.go index 432e145e..1ccb72f2 100644 --- a/internal/modules/tidy_operation.go +++ b/internal/modules/tidy_operation.go @@ -41,7 +41,7 @@ func TidyWithReport(ctx context.Context, root string, repository Repository) (_ return TidyResult{}, fmt.Errorf("captureTidyInputs: %w", err) } if len(inputs.module.Replaces) > 0 { - err = validateLocalOverlay(ctx, inputs.root, inputs.module, repository, false) + err = validateTidyOverlay(ctx, &inputs, repository) return TidyResult{}, err } if inputs.completeCacheOwnership { @@ -88,13 +88,13 @@ func TidyWithReport(ctx context.Context, root string, repository Repository) (_ return TidyResult{}, fmt.Errorf("excludePreviousPinSources: %w", err) } roots := append(slices.Clone(inputs.roots), dependencies...) - err = CheckSourceCollisions(roots) - if err != nil { - return TidyResult{}, fmt.Errorf("CheckSourceCollisions: %w", err) - } view.roots = roots tx.validateInputs = func() error { - err := checkTidySourceSelection(inputs.observations, inputs.roots, inputs.files) + err := view.verifyImportCandidates(ctx) + if err != nil { + return fmt.Errorf("verifyImportCandidates: %w", err) + } + err = checkTidySourceSelection(inputs.observations, inputs.roots, inputs.files) if err != nil { return fmt.Errorf("checkTidySourceSelection: %w", err) } @@ -102,7 +102,11 @@ func TidyWithReport(ctx context.Context, root string, repository Repository) (_ if err != nil { return fmt.Errorf("verifyOldNamespaces: %w", err) } - return verifyTidyCache(ctx, lock, repository) + err = verifyTidyCache(ctx, lock, repository) + if err != nil { + return fmt.Errorf("verifyTidyCache: %w", err) + } + return view.verifyImportCandidates(ctx) } report, err := bindings.planRewrites(tx, inputs.files, view) if err != nil { diff --git a/internal/modules/tidy_overlay.go b/internal/modules/tidy_overlay.go new file mode 100644 index 00000000..8a2a4135 --- /dev/null +++ b/internal/modules/tidy_overlay.go @@ -0,0 +1,93 @@ +package modules + +import ( + "bytes" + "context" + "fmt" + "slices" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" + "github.com/easyp-tech/easyp/internal/sourceview" +) + +// Replacement validation shares tidy's reachable resolver without proposing +// consumer edits or serializing the effective graph into the published lock. +func validateTidyOverlay(ctx context.Context, inputs *tidyInputs, repository Repository) error { + manifest, err := inputs.observations.capture(v1.ModuleFile, nil) + if err != nil { + return fmt.Errorf("capture: %w", err) + } + if !bytes.Equal(manifest.data, inputs.original) { + return fmt.Errorf("protobuf.mod changed while reading: %w", sourceview.ErrChanged) + } + _, err = inputs.observations.capture(v1.LockFile, nil) + if err != nil { + return fmt.Errorf("capture: %w", err) + } + view := newTidySourceView(inputs.observations, nil) + locals := newLocalReplacements(inputs.root, inputs.module, nil) + locals.read = view.readLocalReplacement + locals.inspect = view.inspectLocalRoots + graph, err := resolveEffectiveGraph(ctx, inputs.root, inputs.module, repository, locals, false) + if err != nil { + return fmt.Errorf("resolveEffectiveGraph: %w", err) + } + inputs.roots, err = ModuleSources(inputs.root, inputs.module) + if err != nil { + return fmt.Errorf("ModuleSources: %w", err) + } + err = inputs.captureResolvedSources(repository, graph.remoteLock) + if err != nil { + return fmt.Errorf("captureResolvedSources: %w", err) + } + if inputs.completeCacheOwnership { + err = inputs.captureConsumerSources() + if err != nil { + return fmt.Errorf("captureConsumerSources: %w", err) + } + } + view.roots = append(slices.Clone(inputs.roots), graph.Sources...) + names := make([]string, 0, len(graph.Modules)) + for name := range graph.Modules { + names = append(names, name) + } + slices.Sort(names) + for _, name := range names { + module := graph.Modules[name] + err = view.observeModuleMetadata(module.Directory, module.Module) + if err != nil { + return fmt.Errorf("observeModuleMetadata: %w", err) + } + } + _, err = view.validateImports(ctx, inputs.files, TidyResult{}) + if err != nil { + return fmt.Errorf("validateImports: %w", err) + } + if len(graph.remoteLock.Modules) > 0 { + err = verifyTidyCache(ctx, graph.remoteLock, repository) + if err != nil { + return fmt.Errorf("verifyTidyCache: %w", err) + } + } + err = view.verifyImportCandidates(ctx) + if err != nil { + return fmt.Errorf("verifyImportCandidates: %w", err) + } + err = checkTidySourceSelection(inputs.observations, inputs.roots, inputs.files) + if err != nil { + return fmt.Errorf("checkTidySourceSelection: %w", err) + } + err = view.verifyLocalRoots(ctx) + if err != nil { + return fmt.Errorf("verifyLocalRoots: %w", err) + } + err = view.verifyLocalMetadata(ctx) + if err != nil { + return fmt.Errorf("verifyLocalMetadata: %w", err) + } + err = inputs.observations.verifyCapturedInputs() + if err != nil { + return fmt.Errorf("verifyCapturedInputs: %w", err) + } + return nil +} diff --git a/internal/sourceview/local.go b/internal/sourceview/local.go index bc18e523..56815878 100644 --- a/internal/sourceview/local.go +++ b/internal/sourceview/local.go @@ -118,6 +118,24 @@ func ResolveLocal(ctx context.Context, rootPath, logical string) (_ Resolution, return resolved, localResolutionError(logical, resolved, err) } +// ResolveLocalOptional supports absent import candidates beneath directory +// aliases while preserving the local boundary and nested-repository checks. +func ResolveLocalOptional(ctx context.Context, rootPath, logical string) (_ Resolution, resultErr error) { + root, view, err := localView(rootPath) + if err != nil { + return Resolution{}, fmt.Errorf("localView: %w", err) + } + defer func() { resultErr = errors.Join(resultErr, root.Close()) }() + resolved, err := view.ResolveOptional(ctx, filepath.ToSlash(logical)) + if err == nil || errors.Is(err, fs.ErrNotExist) { + checkErr := CheckLocalResolution(root, resolved) + if checkErr != nil { + return resolved, checkErr + } + } + return resolved, err +} + func localView(rootPath string) (*os.Root, *View, error) { canonical, err := filepath.EvalSymlinks(rootPath) if err != nil { diff --git a/internal/sourceview/view.go b/internal/sourceview/view.go index c9721187..d32df7e6 100644 --- a/internal/sourceview/view.go +++ b/internal/sourceview/view.go @@ -112,6 +112,26 @@ func (v *View) Resolve(ctx context.Context, logical string) (Resolution, error) return resolved, nil } +// ResolveOptional distinguishes ordinary absence beneath valid directory +// aliases from dangling aliases. Missing candidates can be skipped by import +// lookup; a present alias whose own target is missing must still fail. +func (v *View) ResolveOptional(ctx context.Context, logical string) (Resolution, error) { + resolved, err := v.Resolve(ctx, logical) + if !errors.Is(err, fs.ErrNotExist) { + return resolved, err + } + for _, link := range resolved.Links { + target, linkErr := v.Resolve(ctx, link.Path) + if errors.Is(linkErr, fs.ErrNotExist) || (linkErr == nil && !target.Info.IsDir()) { + return resolved, localResolutionError(logical, resolved, err) + } + if linkErr != nil { + return resolved, fmt.Errorf("Resolve: %w", linkErr) + } + } + return resolved, err +} + func (v *View) resolveComponents(ctx context.Context, base string, components []string, active map[string]bool, resolved *Resolution) error { err := ctx.Err() if err != nil { diff --git a/internal/sourceview/view_test.go b/internal/sourceview/view_test.go index ffbbae79..0131936e 100644 --- a/internal/sourceview/view_test.go +++ b/internal/sourceview/view_test.go @@ -62,6 +62,38 @@ func TestResolvePreservesComponentOrderAndLinkTrace(t *testing.T) { assert.Equal(t, "message File {}", string(data)) } +func TestResolveOptionalDistinguishesMissingCandidateFromDanglingAlias(t *testing.T) { + t.Parallel() + view := New(fstest.MapFS{ + "sources/file.proto": {Data: []byte("message File {}")}, + "proto": {Mode: fs.ModeSymlink, Data: []byte("sources")}, + "dangling.proto": {Mode: fs.ModeSymlink, Data: []byte("missing.proto")}, + "dangling_dir": {Mode: fs.ModeSymlink, Data: []byte("missing_dir")}, + }) + tests := []struct { + name string + path string + wantErr error + }{ + {name: "missing leaf below directory alias", path: "proto/missing.proto", wantErr: fs.ErrNotExist}, + {name: "missing subtree below directory alias", path: "proto/other/file.proto", wantErr: fs.ErrNotExist}, + {name: "dangling file alias", path: "dangling.proto", wantErr: ErrUnsupported}, + {name: "dangling directory alias", path: "dangling_dir/file.proto", wantErr: ErrUnsupported}, + {name: "present file below directory alias", path: "proto/file.proto"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + resolved, err := view.ResolveOptional(t.Context(), tt.path) + require.ErrorIs(t, err, tt.wantErr) + assert.NotEmpty(t, resolved.Links) + if tt.wantErr == ErrUnsupported { + assert.NotErrorIs(t, err, fs.ErrNotExist, "dangling aliases must not be skipped as absent imports") + } + }) + } +} + func TestResolveLinkCyclesUsePhysicalPaths(t *testing.T) { t.Parallel() view := New(fstest.MapFS{ From 965a2344117ff3653e1528f943c5465480788b84 Mon Sep 17 00:00:00 2001 From: Khasbulat Abdullin Date: Mon, 2 Nov 2026 00:40:04 +0300 Subject: [PATCH 5/5] perf: parallelize dependency acquisition and plugin execution Reuse bounded ordered workers for module acquisition and plugins, share builtin WASM compilation across each run, index workspace declarations once, and skip permanent repository fetch fallbacks. --- .spec/ARCHITECTURE.md | 8 + .spec/config/dependency.md | 2 + .../2026-11-02-generation-parallelism.md | 65 +++ internal/adapters/gitmodules/download.go | 15 +- .../gitmodules/download_parallel_test.go | 182 +++++++ .../adapters/gitmodules/fetch_preparation.go | 34 ++ .../gitmodules/fetch_preparation_test.go | 113 +++++ internal/adapters/gitmodules/object_cache.go | 16 +- .../gitmodules/object_cache_fallback_test.go | 116 +++++ .../adapters/gitmodules/snapshot_checkout.go | 26 +- internal/adapters/plugin/builtin_impl.go | 82 +++- internal/adapters/plugin/builtin_impl_test.go | 109 +++++ internal/adapters/plugin/builtin_run.go | 182 +++++++ internal/adapters/plugin/builtin_run_test.go | 452 ++++++++++++++++++ internal/core/generate.go | 121 ++--- internal/core/generate_bucket.go | 21 +- internal/core/generate_bucket_test.go | 35 ++ .../core/generate_compiler_version_test.go | 16 +- internal/core/generate_descriptors_test.go | 7 +- internal/core/generate_parallel_test.go | 191 ++++++++ internal/core/generate_plugins.go | 119 +++++ internal/core/generate_selection_test.go | 8 +- internal/generation/descriptor_set.go | 3 +- internal/generation/frozen.go | 6 +- internal/generation/generate.go | 13 +- internal/generation/selection.go | 49 +- internal/generation/workspace_index.go | 84 ++++ internal/generation/workspace_index_test.go | 303 ++++++++++++ internal/modules/fetch_preparation.go | 129 +++++ internal/modules/fetch_preparation_test.go | 246 ++++++++++ internal/modules/import_roots_preparation.go | 51 ++ internal/modules/import_roots_resolution.go | 5 + internal/modules/resolve.go | 127 ++--- internal/parallel/parallel.go | 59 +++ internal/parallel/parallel_test.go | 112 +++++ 35 files changed, 2869 insertions(+), 238 deletions(-) create mode 100644 .spec/plans/2026-11-02-generation-parallelism.md create mode 100644 internal/adapters/gitmodules/download_parallel_test.go create mode 100644 internal/adapters/gitmodules/fetch_preparation.go create mode 100644 internal/adapters/gitmodules/fetch_preparation_test.go create mode 100644 internal/adapters/gitmodules/object_cache_fallback_test.go create mode 100644 internal/adapters/plugin/builtin_impl_test.go create mode 100644 internal/adapters/plugin/builtin_run.go create mode 100644 internal/adapters/plugin/builtin_run_test.go create mode 100644 internal/core/generate_bucket_test.go create mode 100644 internal/core/generate_parallel_test.go create mode 100644 internal/core/generate_plugins.go create mode 100644 internal/generation/workspace_index.go create mode 100644 internal/generation/workspace_index_test.go create mode 100644 internal/modules/fetch_preparation.go create mode 100644 internal/modules/fetch_preparation_test.go create mode 100644 internal/modules/import_roots_preparation.go create mode 100644 internal/parallel/parallel.go create mode 100644 internal/parallel/parallel_test.go diff --git a/.spec/ARCHITECTURE.md b/.spec/ARCHITECTURE.md index afe87faf..95048b14 100644 --- a/.spec/ARCHITECTURE.md +++ b/.spec/ARCHITECTURE.md @@ -95,6 +95,8 @@ Nested protobuf module roots are rebased by module_config from the The Git cache lives beneath the CLI-supplied EasyP directory in v1/git; internal/adapters/gitmodules/object_cache.go maintains reusable bare object repositories with OS locks. Cache.Cached reads installed metadata; Install supplies content verification before cached metadata is trusted. Successful verification and the inventory are reused within one cache handle, keyed by source, commit, content hash and selected roots. A new handle rechecks the snapshot; explicit VerifyCached bypasses the in-memory result. Revision resolution atomically retains completed snapshots so installation does not materialize them a second time. +Independent locked entries are installed with at most eight workers, retaining snapshot and repository locks. Sources that explicitly support independent fetch preparation also acquire known revisions concurrently during normal resolution. The solver consumes prepared metadata in FIFO request order; versionless selection, import-root validation, BSR expansion and mutable graph bookkeeping remain serial. Generic sources and local-overlay resolution keep their existing serial fetch contract. A permanently unavailable repository skips the same-remote full-history fallback; distinct parent candidates remain available for nested-module discovery. + Local replacements use an ephemeral main-module overlay graph and never alter the shared lock. Tidy validates the overlay; get/update may edit explicit manifest requirements. Vendor copies the effective local graph without touching the lock. Explicit frozen operation verifies existing native manifests and locked graphs; automatic unknown-import-to-module discovery is intentionally not implemented. ## Generation @@ -117,6 +119,12 @@ api.Generate.Action: flags + cwd + cache Request.WorkDir controls discovery, relative descriptor output and local plugin execution. Plugin output remains relative to the generator config. Default discovery chooses the nearest ancestor generator within the workspace; repeatable --project selects consumers explicitly and --all enables recursive discovery. Generation builds no unrelated lint rules. Only generation options inherit; plugins and module selections remain local to the consumer. +Each generation run lazily indexes workspace module declarations once for normal and frozen target lookup. The index stores declaration locations and lookup errors, never consumer dependency graphs or selected versions. A later run reads the workspace again. + +Within each prepared target, at most four plugin invocations run concurrently with independent requests. Workers store indexed responses; the caller applies responses, insertion points and conflicts in configured plugin order, and targets retain their original order. The completed bucket validates and writes paths in sorted order. internal/parallel supplies bounded atomic-index scheduling and input-ordered errors; cancellation stops new tasks and joins active work. + +generation.Run owns one lazy builtin WASM compilation scope across all target cores. It compiles each distinct embedded binary once with a shared wazero cache, then creates independent runtime, memory, module instances and I/O per invocation. Standalone core execution owns a temporary scope when no command scope exists. Closing a scope cancels and joins active work before releasing compiled code and the cache; cleanup failures remain command errors. + ## Policy and validation commands Lint/breaking CLI adapters select producer policy and prepare shared module import roots before constructing the engine. buildCore in internal/api/runtime.go constructs only lint/breaking engines. core.Options carries AllowCommentIgnores and KnownLintRules per engine; comment directives are parsed and applied to findings by internal/core/comment_suppressions.go. Breaking comparisons use historical Git trees and the baseline-specific roots prepared by the API layer. diff --git a/.spec/config/dependency.md b/.spec/config/dependency.md index 12bca6df..4ee0b783 100644 --- a/.spec/config/dependency.md +++ b/.spec/config/dependency.md @@ -198,6 +198,8 @@ for the distinction between such errors and CLI exit classification. The resolver accepts Source.Fetch, independent of Git/cache. It selects the highest required semantic version, treats versionless requirements as weak constraints, and rejects incompatible exact commits (including tag/commit disagreement). It caches revision fetches within a resolution, rebuilds provisional HEAD edges when stronger requirements appear, and sorts resulting entries. Update additionally needs version enumeration; vendor and published download only need the cache contract; local-overlay download additionally uses Source when a new remote requirement needs resolution. +The Git adapter installs independent entries of a validated lock with at most eight workers. Normal resolution can also prepare independent known revisions concurrently through an optional source capability, then consume their metadata in the original request order. Weak versionless requests wait for stronger constraints; graph selection, root validation and BSR metadata callbacks remain serial. Generic sources and local replacement overlays retain serial fetch semantics. Snapshot and object-store locks still protect shared cache publication. Definite missing-repository or authentication failures skip the redundant full-history fetch of the same remote; other candidate repository URLs and valid historical-commit fallback remain supported. + ## Frozen graph validation Metadata-free module resolution may record canonical repository-snapshot-relative `roots` in diff --git a/.spec/plans/2026-11-02-generation-parallelism.md b/.spec/plans/2026-11-02-generation-parallelism.md new file mode 100644 index 00000000..30f0d1a4 --- /dev/null +++ b/.spec/plans/2026-11-02-generation-parallelism.md @@ -0,0 +1,65 @@ +# Generation Performance Implementation Plan + +> **For agentic workers:** Use superpowers:subagent-driven-development with isolated file ownership and superpowers:dispatching-parallel-agents for independent adapters. No commits or pushes are required by this task. + +**Goal:** Implement the five optimizations requested in issue #245 while preserving module selection, generated bytes, output application order, and deterministic errors. + +**Architecture:** Reuse the test compiler's bounded workers, atomic next-task index, exclusive indexed result slots, and join-before-ordered-merge pattern. Command-owned state holds compiled WASM and a workspace declaration index; independent consumers retain independent MVS graphs. No compiler replacement, new dependencies, CLI/schema changes, or persistent cache formats. + +**Tech Stack:** Go 1.26.6, sync/atomic, context, wazero 1.9.0, existing testify/local Git fixtures. + +## Contract + +- Results and errors are consumed by task index, never completion order. Cancellation stops scheduling and waits for active tasks to finish. +- Keep output application in configured target/plugin order, including insertion points and conflicting-output validation. Independent plugin requests and memory are mandatory. +- Bound Git/network and plugin concurrency. Keep per-snapshot and per-repository locks; solver maps and graph selection are merged serially. +- A missing repository must not trigger full-history fallback for the same remote. Nested-module discovery still tries distinct parent repository candidates; shallow-fetch compatibility fallback remains for supported cases. +- Compile builtin universal WASM and its memory module once per generation command. Use independent runtime/module instances and I/O, one shared compilation cache, and explicit lifetime/close error handling. +- Workspace indexing reads declarations once per command, retains existing skip/boundary/alias/duplicate-identity behavior, and never reuses dependency versions between consumers. A new command sees changed declarations. + +## Task 1: Bounded ordered workers + +Files: `internal/parallel/parallel.go`, `parallel_test.go`. +- [x] Add meaningful tests for actual overlap, strict worker bound, once-only execution, input-ordered errors, and cancellation/join. +- [x] Run tests red before implementing `Run(ctx, count, limit, func(index int) error) error` using atomic work claiming and indexed errors; no first-completed-error selection. +- [x] Run package tests with `-race`. + +## Task 2: Git module acquisition and permanent failures + +Files: `internal/adapters/gitmodules/download.go`, `object_cache.go`, related cache helpers/tests; `internal/modules/resolve.go` and import-root fetch helpers/tests when needed for cold resolve prefetch. +- [x] Demonstrate sequential acquisition and redundant missing-repository fallback with controlled local fixtures. +- [x] Use bounded workers for independent installs. For cold tidy, prepare independent revision fetches concurrently and consume metadata in original solver order; preserve provisional/versionless/roots behavior and cache source safety. +- [x] Classify permanent missing-repository failures and bypass useless same-remote fallback without suppressing parent candidates or valid exact-SHA history fetches. +- [x] Verify race safety, stable lock/errors, missing repo call counts, cancellation, same-repository locking, and existing MVS/roots fixtures. + +## Task 3: Command-owned WASM compilation + +Files: `internal/adapters/plugin/builtin_impl.go`, new builtin run/cache helpers and tests. Integration in `internal/generation/generate.go` and standalone core execution belongs to orchestrator after adapter API is ready. +- [x] Test shared compilation and isolated invocation state, including overlapping calls and close/cancellation. +- [x] Provide an explicit command context/lifetime API, lazily compile each distinct builtin binary once, and instantiate fresh memory/runtime/I/O per invocation with identical engine features/imports. +- [x] Handle failed initialization and cleanup deterministically; share cache across all target cores in a command, not only one executor. +- [x] Run existing builtin fixtures and new concurrency tests under `-race`. + +## Task 4: Ordered parallel plugin execution + +Files: `internal/core/generate.go`, extracted plugin preparation/application helpers and tests; `internal/generation/generate.go` for shared run scope. +- [x] Prove independent calls overlap and responses/errors remain ordered with controlled executor doubles; cover insertion points, duplicate outputs, request isolation, cancellation and no output on failure. +- [x] Split independent plugin invocation from ordered response application. Store indexed responses, join workers, then apply in configuration order to the existing bucket. +- [x] Keep command invocations process-local (independent I/O and `cmd.Dir`); the existing console adapters are stateless and support concurrent calls without changing global cwd. +- [x] Run core/generation tests under `-race`. + +## Task 5: Workspace declaration index + +Files: `internal/generation/selection.go`, `frozen.go`, new workspace index helper/tests, caller plumbing in generation. +- [x] Test one traversal across repeated selections, unchanged duplicate/invalid declaration behavior and boundaries, new-run freshness and independent consumer versions. +- [x] Construct a lazy per-command declaration index and pass it through normal/frozen selection. Preserve explicit external-directory contexts. +- [x] Run generation tests under `-race`. + +## Task 6: Integration and regress + +- [x] Review task results for contract compliance, then code quality; fix substantive findings and recheck. +- [x] Update architecture/dependency documentation with actual lifetimes and ordering. +- [x] Run full source suite: `GOTOOLCHAIN=go1.26.6 go test -mod=readonly -race -count=1 -p=2 -parallel=8 -timeout=15m -json ./...`; run pinned Go linter and `git diff --check`. +- [x] Build fresh CLI and MCP into a temporary artifact directory, record revision/diff identity. +- [x] Run easyp-test regress using those binaries: `go test -mod=readonly -tags=v1 -race -count=1 -p=2 -parallel=4 -timeout=20m -json ./tests/e2e/v1` with its existing local changes preserved. +- [x] Report exact tests/results, any remaining limitation, and working-tree status. diff --git a/internal/adapters/gitmodules/download.go b/internal/adapters/gitmodules/download.go index 88ca3dbe..94530211 100644 --- a/internal/adapters/gitmodules/download.go +++ b/internal/adapters/gitmodules/download.go @@ -10,8 +10,11 @@ import ( "github.com/easyp-tech/easyp/internal/adapters/gitcommand" moduleconfig "github.com/easyp-tech/easyp/internal/adapters/module_config" v1 "github.com/easyp-tech/easyp/internal/config/v1" + "github.com/easyp-tech/easyp/internal/parallel" ) +const moduleAcquisitionLimit = 8 + func v1ModuleCachePath(cacheRoot string, entry v1.LockedModule) string { identity := entry.Commit + ":" + entry.Hash if roots := v1RootSelectionKey(entry.Roots); roots != "" { @@ -41,13 +44,11 @@ func (c *Cache) install(ctx context.Context, lock v1.Lock, acquire bool) error { if err := lock.Validate(); err != nil { return fmt.Errorf("Validate: %w", err) } - for _, entry := range lock.Modules { - if err := ctx.Err(); err != nil { - return fmt.Errorf("Err: %w", err) - } - if err := c.installEntry(ctx, entry, acquire); err != nil { - return err - } + err := parallel.Run(ctx, len(lock.Modules), moduleAcquisitionLimit, func(index int) error { + return c.installEntry(ctx, lock.Modules[index], acquire) + }) + if err != nil { + return fmt.Errorf("Run: %w", err) } return nil } diff --git a/internal/adapters/gitmodules/download_parallel_test.go b/internal/adapters/gitmodules/download_parallel_test.go new file mode 100644 index 00000000..22a1a3f2 --- /dev/null +++ b/internal/adapters/gitmodules/download_parallel_test.go @@ -0,0 +1,182 @@ +package gitmodules + +import ( + "context" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "testing/synctest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/yaml.v3" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" + "github.com/easyp-tech/easyp/internal/logger" + "github.com/easyp-tech/easyp/internal/modules" +) + +func TestInstallBoundsAcquisitionAndJoinsCancellation(t *testing.T) { + t.Parallel() + for _, test := range []struct { + name string + download bool + }{ + {name: "cache_install"}, + {name: "mod_download", download: true}, + } { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + testInstallAcquisitionBound(t, test.download) + }) + } +} + +func testInstallAcquisitionBound(t *testing.T, download bool) { + t.Helper() + storage := t.TempDir() + project := t.TempDir() + synctest.Test(t, func(t *testing.T) { + const count = 12 + started := make(chan string, count) + release := make(chan struct{}) + log := installBarrierLogger{started: started, release: release} + cache := New(storage).WithLogger(log) + lock := v1.Lock{Version: 1} + for index := range count { + lock.Modules = append(lock.Modules, v1.LockedModule{ + Source: "example.test/team/" + string(rune('a'+index)), + Version: strings.Repeat("a", 40), Commit: strings.Repeat("a", 40), + Hash: "h1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + }) + } + var manifest strings.Builder + manifest.WriteString("module example.test/app\n") + for _, entry := range lock.Modules { + manifest.WriteString("require " + entry.Source + " " + entry.Version + "\n") + } + require.NoError(t, os.WriteFile(filepath.Join(project, v1.ModuleFile), []byte(manifest.String()), 0o600)) + raw, err := yaml.Marshal(lock) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(project, v1.LockFile), raw, 0o600)) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { + if download { + done <- modules.Download(ctx, project, cache) + return + } + done <- cache.Install(ctx, lock) + }() + synctest.Wait() + assert.Len(t, started, 8, "independent installs should fill the bounded worker pool") + cancel() + synctest.Wait() + select { + case <-done: + t.Error("Install returned while acquisition workers were active") + default: + } + close(release) + require.ErrorIs(t, <-done, context.Canceled) + assert.Len(t, started, 8, "cancellation must stop scheduling more installs") + }) +} + +func TestVerifyCachedReturnsFirstLockErrorAfterOutOfOrderCompletion(t *testing.T) { + t.Parallel() + storage := t.TempDir() + synctest.Test(t, func(t *testing.T) { + log := installOrderLogger{secondDone: make(chan struct{})} + cache := New(storage).WithLogger(log) + lock := v1.Lock{Version: 1, Modules: []v1.LockedModule{ + {Source: "a", Version: versionlessTestPin, Commit: versionlessTestPin, Hash: "h1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="}, + {Source: "b", Version: versionlessTestPin, Commit: versionlessTestPin, Hash: "h1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="}, + }} + err := cache.VerifyCached(t.Context(), lock) + require.ErrorContains(t, err, "policy module a@") + _, statErr := os.Stat(cache.root) + require.ErrorIs(t, statErr, os.ErrNotExist, "read-only verification must not create cache directories") + }) +} + +func TestConcurrentInstallAndVerificationRespectSnapshotGate(t *testing.T) { + t.Parallel() + cache := New(t.TempDir()) + entry := v1.LockedModule{Source: filepath.Join(t.TempDir(), "missing"), Version: versionlessTestPin, Commit: versionlessTestPin, Hash: "h1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA="} + lock := v1.Lock{Version: 1, Modules: []v1.LockedModule{entry}} + synctest.Test(t, func(t *testing.T) { + snapshot := cache.snapshotState(v1ModuleCachePath(cache.root, entry)) + require.NoError(t, snapshot.lock(t.Context())) + defer snapshot.unlock() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 2) + go func() { done <- cache.Install(ctx, lock) }() + go func() { done <- cache.VerifyCached(ctx, lock) }() + synctest.Wait() + assert.Empty(t, done, "both commands must wait for the shared snapshot gate") + cancel() + require.ErrorIs(t, <-done, context.Canceled) + require.ErrorIs(t, <-done, context.Canceled) + _, err := os.Stat(cache.root) + require.ErrorIs(t, err, os.ErrNotExist) + }) +} + +const versionlessTestPin = "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + +type installOrderLogger struct { + installBarrierLogger + secondDone chan struct{} +} + +func (log installOrderLogger) Debug(_ context.Context, message string, _ ...slog.Attr) { + if log.phase != "install module" { + return + } + if log.source == "a" && message == "Dependency operation started" { + <-log.secondDone + } + if log.source == "b" && message == "Dependency operation finished" { + close(log.secondDone) + } +} + +func (log installOrderLogger) With(attrs ...slog.Attr) logger.Logger { + log.installBarrierLogger = log.installBarrierLogger.With(attrs...).(installBarrierLogger) + return log +} + +type installBarrierLogger struct { + started chan<- string + release <-chan struct{} + source string + phase string +} + +func (log installBarrierLogger) Debug(_ context.Context, message string, _ ...slog.Attr) { + if message == "Dependency operation started" && log.phase == "install module" { + log.started <- log.source + <-log.release + } +} + +func (installBarrierLogger) Info(context.Context, string, ...slog.Attr) {} +func (installBarrierLogger) Warn(context.Context, string, ...slog.Attr) {} +func (installBarrierLogger) Error(context.Context, string, ...slog.Attr) {} + +func (log installBarrierLogger) With(attrs ...slog.Attr) logger.Logger { + for _, attr := range attrs { + switch attr.Key { + case "source": + log.source = attr.Value.String() + case "phase": + log.phase = attr.Value.String() + } + } + return log +} diff --git a/internal/adapters/gitmodules/fetch_preparation.go b/internal/adapters/gitmodules/fetch_preparation.go new file mode 100644 index 00000000..b6842a77 --- /dev/null +++ b/internal/adapters/gitmodules/fetch_preparation.go @@ -0,0 +1,34 @@ +package gitmodules + +import ( + "context" + "fmt" + + "github.com/easyp-tech/easyp/internal/modules" +) + +// PreparationSource advertises preparation for this cache receiver. Embedding +// wrappers that override fetch behavior must explicitly opt in for themselves. +func (c *Cache) PreparationSource() modules.Source { return c } + +// PrepareFetch acquires an independent immutable revision before the solver +// consumes it. Cache publication keeps the existing snapshot/repository locks. +// The callback defers metadata backends, which need not support concurrent use. +// No temporary checkout survives preparation, including discarded graph edges. +func (c *Cache) PrepareFetch(ctx context.Context, request modules.FetchRequest) (modules.PreparedFetch, error) { + fetched, err := c.fetchSnapshot(ctx, request.Source, request.Version, request.Roots, request.Inspect || len(request.Roots) > 0, true) + if err != nil { + return nil, fmt.Errorf("fetchSnapshot: %w", err) + } + return func(ctx context.Context) (modules.Fetched, error) { + if err := ctx.Err(); err != nil { + return modules.Fetched{}, fmt.Errorf("Err: %w", err) + } + module, bindings, err := c.resolveBSRDependencies(ctx, fetched.Module) + if err != nil { + return modules.Fetched{}, fmt.Errorf("resolveBSRDependencies: %w", err) + } + fetched.Module, fetched.Lock.BSR = module, bindings + return fetched, nil + }, nil +} diff --git a/internal/adapters/gitmodules/fetch_preparation_test.go b/internal/adapters/gitmodules/fetch_preparation_test.go new file mode 100644 index 00000000..98ade2b6 --- /dev/null +++ b/internal/adapters/gitmodules/fetch_preparation_test.go @@ -0,0 +1,113 @@ +package gitmodules + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" + "github.com/easyp-tech/easyp/internal/modules" +) + +func TestPrepareFetchDefersBSRBackendAndCleansTemporaryWork(t *testing.T) { + t.Parallel() + remote := bsrTestRepository(t, "version: v1\ndeps: [buf.build/googleapis/googleapis]\n") + backend := &fakeBSRResolver{git: v1.Requirement{Module: "github.com/acme/provider", Version: "v1.2.3"}} + cache := NewWithBSRResolver(t.TempDir(), backend) + preparer, supported := any(cache).(interface { + PrepareFetch(context.Context, modules.FetchRequest) (modules.PreparedFetch, error) + }) + require.True(t, supported, "Git source must support independent fetch preparation") + complete, err := preparer.PrepareFetch(t.Context(), modules.FetchRequest{Source: remote, Version: "v1.0.0", Inspect: true}) + require.NoError(t, err) + assert.Empty(t, backend.calls, "metadata backends are consumed in serial request order") + entries, err := os.ReadDir(cache.root) + require.NoError(t, err) + for _, entry := range entries { + assert.Contains(t, []string{"objects", "modules", "sources"}, entry.Name(), "preparation must release temporary work even if its callback is discarded") + } + fetched, err := complete(t.Context()) + require.NoError(t, err) + require.Len(t, backend.calls, 1) + assert.Equal(t, "v1.0.0", fetched.Lock.Version) + assert.Equal(t, []v1.Requirement{backend.git}, fetched.Module.Requires) + assert.NotNil(t, fetched.Inspection) + assert.Len(t, fetched.Lock.BSR, 1) + _, _, err = cache.Cached(fetched.Lock) + require.NoError(t, err) + assert.FileExists(t, filepath.Join(cache.root, "sources", v1CacheSourceKey(remote), fetched.Lock.Commit+".json")) +} + +func TestPreparedGitResolutionPreservesVersionlessSemanticAndCommitConstraints(t *testing.T) { + t.Parallel() + dependency := preparedGitRepository(t, "a", nil) + require.NoError(t, os.WriteFile(filepath.Join(dependency, "a.proto"), []byte("syntax = \"proto3\"; package a; message Selected {}\n"), 0o600)) + runTestGit(t, dependency, "add", ".") + runTestGit(t, dependency, "-c", "user.name=Test", "-c", "user.email=test@example.test", "commit", "-qm", "selected version") + runTestGit(t, dependency, "tag", "v1.2.0") + selectedCommit := strings.TrimSpace(runTestGit(t, dependency, "rev-parse", "HEAD")) + // HEAD is deliberately unusable. The weak root requirement must wait for + // the semantic and exact requirements discovered from independent modules. + require.NoError(t, os.WriteFile(filepath.Join(dependency, v1.ModuleFile), []byte("module "+dependency+"\nrequire example.test/unavailable v1.0.0\n"), 0o600)) + runTestGit(t, dependency, "add", ".") + runTestGit(t, dependency, "-c", "user.name=Test", "-c", "user.email=test@example.test", "commit", "-qm", "unselected HEAD") + semantic := preparedGitRepository(t, "b", []v1.Requirement{{Module: dependency, Version: "v1.2.0"}}) + pinned := preparedGitRepository(t, "c", []v1.Requirement{{Module: dependency, Version: selectedCommit}}) + root := v1.Module{Requires: []v1.Requirement{{Module: dependency}, {Module: semantic, Version: "v1.0.0"}, {Module: pinned, Version: "v1.0.0"}}} + cache := New(t.TempDir()) + prepared, err := modules.Resolve(t.Context(), root, cache, nil) + require.NoError(t, err) + serial, err := modules.Resolve(t.Context(), root, struct{ modules.Source }{New(t.TempDir())}, nil) + require.NoError(t, err) + assert.Equal(t, serial, prepared) + require.Len(t, prepared.Modules, 3) + for _, entry := range prepared.Modules { + if entry.Source == dependency { + assert.Equal(t, "v1.2.0", entry.Version) + assert.Equal(t, selectedCommit, entry.Commit) + } + } + require.NoError(t, cache.Install(t.Context(), prepared)) + require.NoError(t, cache.VerifyCached(t.Context(), prepared)) +} + +func TestPreparedProvisionalRevisionFinalizesWithoutAnotherNetworkFetch(t *testing.T) { + t.Parallel() + repository, commit := migrationTestRepository(t, map[string]string{"api/a.proto": "syntax = \"proto3\"; package a;\n"}) + runTestGit(t, repository, "tag", "v1.0.0") + cache := New(t.TempDir()) + complete, err := cache.PrepareFetch(t.Context(), modules.FetchRequest{Source: repository, Version: "v1.0.0", Inspect: true}) + require.NoError(t, err) + fetched, err := complete(t.Context()) + require.NoError(t, err) + require.NotNil(t, fetched.Inspection) + assert.True(t, fetched.Inspection.Provisional) + assert.Empty(t, fetched.Lock.Hash) + require.NoError(t, os.Rename(repository, repository+"-offline")) + finalized, err := cache.FetchWithRoots(t.Context(), repository, commit, []string{"api"}) + require.NoError(t, err) + assert.Equal(t, commit, finalized.Lock.Commit) + assert.Equal(t, []string{"api"}, finalized.Lock.Roots) + assert.NotEmpty(t, finalized.Lock.Hash) +} + +func preparedGitRepository(t *testing.T, name string, requirements []v1.Requirement) string { + t.Helper() + remote := t.TempDir() + manifest := "module " + remote + "\n" + for _, requirement := range requirements { + manifest += "require " + requirement.Module + " " + requirement.Version + "\n" + } + require.NoError(t, os.WriteFile(filepath.Join(remote, v1.ModuleFile), []byte(manifest), 0o600)) + require.NoError(t, os.WriteFile(filepath.Join(remote, name+".proto"), []byte("syntax = \"proto3\"; package "+name+";\n"), 0o600)) + runTestGit(t, remote, "init", "-q") + runTestGit(t, remote, "add", ".") + runTestGit(t, remote, "-c", "user.name=Test", "-c", "user.email=test@example.test", "commit", "-qm", "initial") + runTestGit(t, remote, "tag", "v1.0.0") + return remote +} diff --git a/internal/adapters/gitmodules/object_cache.go b/internal/adapters/gitmodules/object_cache.go index 9b29b658..10335255 100644 --- a/internal/adapters/gitmodules/object_cache.go +++ b/internal/adapters/gitmodules/object_cache.go @@ -51,7 +51,7 @@ func checkoutCachedCommit(ctx context.Context, checkout string, entry v1.LockedM // requesting its object ID. Do not silently replace an unavailable pin by HEAD. _, fetchErr := gitV1(ctx, repository, "-c", "fetch.fsckObjects=true", "fetch", "--quiet", "--no-tags", "--no-write-fetch-head", "--depth=1", "--", candidate.remote, commit) if fetchErr != nil { - if gitcommand.Interrupted(fetchErr) { + if gitcommand.Interrupted(fetchErr) || unavailableObjectRepository(fetchErr) { return false, fmt.Errorf("gitV1: %w", fetchErr) } // Some servers only expose advertised refs. Fetch reachable history as a @@ -92,6 +92,20 @@ func checkoutCachedCommit(ctx context.Context, checkout string, entry v1.LockedM return true, nil } +// unavailableObjectRepository recognizes diagnostics that fetching more history +// from the same remote cannot fix. Unknown and transport failures retain the +// advertised-ref fallback; callers still try distinct parent repositories. +func unavailableObjectRepository(err error) bool { + diagnostic := strings.ToLower(err.Error()) + return strings.Contains(diagnostic, "repository not found") || + (strings.Contains(diagnostic, "fatal: repository '") && strings.Contains(diagnostic, "' not found")) || + strings.Contains(diagnostic, "does not appear to be a git repository") || + strings.Contains(diagnostic, "authentication failed for") || + (strings.Contains(diagnostic, "could not read username for") && strings.Contains(diagnostic, "terminal prompts disabled")) || + (strings.Contains(diagnostic, "could not read password for") && strings.Contains(diagnostic, "terminal prompts disabled")) || + strings.Contains(diagnostic, "permission denied (publickey") +} + // OS locks are released on process exit and shared across concurrent CLI runs. func lockObjectRepository(ctx context.Context, path string) (unlock func(), err error) { ctx, cancel, err := gitcommand.Operation(ctx) diff --git a/internal/adapters/gitmodules/object_cache_fallback_test.go b/internal/adapters/gitmodules/object_cache_fallback_test.go new file mode 100644 index 00000000..aa2f1bc3 --- /dev/null +++ b/internal/adapters/gitmodules/object_cache_fallback_test.go @@ -0,0 +1,116 @@ +package gitmodules + +import ( + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" +) + +func TestPinnedFetchSkipsHistoryForUnavailableRepository(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("SSH fixture uses a POSIX shell") + } + tests := []struct { + name string + diagnostic string + attempts int + }{ + {name: "missing_repository", diagnostic: "ERROR: Repository not found.", attempts: 1}, + {name: "http_missing_repository", diagnostic: "fatal: repository 'https://example.test/team/repo' not found", attempts: 1}, + {name: "authentication_failed", diagnostic: "fatal: Authentication failed for 'https://example.test/team/repo'", attempts: 1}, + {name: "noninteractive_credentials", diagnostic: "fatal: could not read Username for 'https://example.test': terminal prompts disabled", attempts: 1}, + {name: "ssh_credentials", diagnostic: "git@example.test: Permission denied (publickey).", attempts: 1}, + {name: "transient_failure_keeps_fallback", diagnostic: "fatal: connection reset by peer", attempts: 2}, + {name: "unadvertised_commit_keeps_fallback", diagnostic: "error: Server does not allow request for unadvertised object", attempts: 2}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + directory := t.TempDir() + attempts := filepath.Join(directory, "attempts") + ssh := filepath.Join(directory, "ssh") + script := "#!/bin/sh\nprintf 'attempt\\n' >> " + quoteTestShell(attempts) + "\nprintf '%s\\n' " + quoteTestShell(tt.diagnostic) + " >&2\nexit 128\n" + require.NoError(t, os.WriteFile(ssh, []byte(script), 0o755)) + t.Setenv("GIT_SSH_COMMAND", ssh) + t.Setenv("GIT_SSH_VARIANT", "ssh") + _, err := checkoutCachedCommit(t.Context(), filepath.Join(directory, "checkout"), + v1.LockedModule{Commit: strings.Repeat("a", 40)}, + v1GitModuleCandidate{remote: "ssh://example.test/team/repo"}) + require.Error(t, err) + data, err := os.ReadFile(attempts) + require.NoError(t, err) + assert.Equal(t, tt.attempts, strings.Count(string(data), "attempt\n")) + }) + } +} + +func TestPinnedFetchHistoryFallbackFindsAdvertisedAncestor(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("SSH fixture uses a POSIX shell") + } + directory := t.TempDir() + remote := filepath.Join(directory, "remote") + require.NoError(t, os.Mkdir(remote, 0o755)) + require.NoError(t, os.WriteFile(filepath.Join(remote, "api.proto"), []byte("first"), 0o644)) + runTestGit(t, remote, "init", "-q") + runTestGit(t, remote, "add", ".") + runTestGit(t, remote, "-c", "user.name=Test", "-c", "user.email=test@example.test", "commit", "-qm", "first") + commit := strings.TrimSpace(runTestGit(t, remote, "rev-parse", "HEAD")) + require.NoError(t, os.WriteFile(filepath.Join(remote, "api.proto"), []byte("second"), 0o644)) + runTestGit(t, remote, "add", ".") + runTestGit(t, remote, "-c", "user.name=Test", "-c", "user.email=test@example.test", "commit", "-qm", "second") + attempts := filepath.Join(directory, "attempts") + ssh := filepath.Join(directory, "ssh") + script := "#!/bin/sh\nif test ! -e " + quoteTestShell(attempts) + "; then\n printf 'attempt\\n' >> " + quoteTestShell(attempts) + "\n printf '%s\\n' 'error: Server does not allow request for unadvertised object' >&2\n exit 128\nfi\nprintf 'attempt\\n' >> " + quoteTestShell(attempts) + "\nexec git-upload-pack " + quoteTestShell(remote) + "\n" + require.NoError(t, os.WriteFile(ssh, []byte(script), 0o755)) + t.Setenv("GIT_SSH_COMMAND", ssh) + t.Setenv("GIT_SSH_VARIANT", "ssh") + checkout := filepath.Join(directory, "checkout") + found, err := checkoutCachedCommit(t.Context(), checkout, v1.LockedModule{Commit: commit}, v1GitModuleCandidate{remote: "ssh://example.test/team/repo"}) + require.NoError(t, err) + assert.True(t, found) + assert.Equal(t, commit, strings.TrimSpace(runTestGit(t, checkout, "rev-parse", "HEAD"))) + data, err := os.ReadFile(attempts) + require.NoError(t, err) + assert.Equal(t, 2, strings.Count(string(data), "attempt\n")) +} + +func TestPinnedMissingNestedRepositoryStillSearchesParent(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("SSH fixture uses a POSIX shell") + } + directory := t.TempDir() + remote := filepath.Join(directory, "remote") + require.NoError(t, os.MkdirAll(filepath.Join(remote, "nested"), 0o755)) + const source = "ssh://example.test/team/repo/nested" + require.NoError(t, os.WriteFile(filepath.Join(remote, "nested", "protobuf.mod"), []byte("module "+source+"\n"), 0o644)) + require.NoError(t, os.WriteFile(filepath.Join(remote, "nested", "api.proto"), []byte("syntax = \"proto3\"; package api;\n"), 0o644)) + runTestGit(t, remote, "init", "-q") + runTestGit(t, remote, "add", ".") + runTestGit(t, remote, "-c", "user.name=Test", "-c", "user.email=test@example.test", "commit", "-qm", "initial") + commit := strings.TrimSpace(runTestGit(t, remote, "rev-parse", "HEAD")) + attempts := filepath.Join(directory, "attempts") + ssh := filepath.Join(directory, "ssh") + script := "#!/bin/sh\nprintf '%s\\n' \"$*\" >> " + quoteTestShell(attempts) + "\ncase \"$*\" in\n */repo/nested*) printf '%s\\n' 'ERROR: Repository not found.' >&2; exit 128;;\nesac\nexec git-upload-pack " + quoteTestShell(remote) + "\n" + require.NoError(t, os.WriteFile(ssh, []byte(script), 0o755)) + t.Setenv("GIT_SSH_COMMAND", ssh) + t.Setenv("GIT_SSH_VARIANT", "ssh") + fetched, err := New(filepath.Join(directory, "cache")).Fetch(t.Context(), source, commit) + require.NoError(t, err) + assert.Equal(t, source, fetched.Module.Name) + assert.Equal(t, commit, fetched.Lock.Commit) + data, err := os.ReadFile(attempts) + require.NoError(t, err) + assert.Len(t, strings.Split(strings.TrimSpace(string(data)), "\n"), 2) + assert.Equal(t, 1, strings.Count(string(data), "/repo/nested")) +} + +func quoteTestShell(value string) string { + return "'" + strings.ReplaceAll(value, "'", "'\\''") + "'" +} diff --git a/internal/adapters/gitmodules/snapshot_checkout.go b/internal/adapters/gitmodules/snapshot_checkout.go index 6daf5173..6c64cb38 100644 --- a/internal/adapters/gitmodules/snapshot_checkout.go +++ b/internal/adapters/gitmodules/snapshot_checkout.go @@ -43,7 +43,15 @@ func (c *Cache) FetchForRootResolution(ctx context.Context, source, version stri return fetched, nil } -func (c *Cache) fetchWithRootSelection(ctx context.Context, source, version string, roots []string, inspect bool) (fetched modules.Fetched, err error) { +func (c *Cache) fetchWithRootSelection(ctx context.Context, source, version string, roots []string, inspect bool) (modules.Fetched, error) { + fetched, err := c.fetchSnapshot(ctx, source, version, roots, inspect, false) + if err != nil { + return modules.Fetched{}, fmt.Errorf("fetchSnapshot: %w", err) + } + return fetched, nil +} + +func (c *Cache) fetchSnapshot(ctx context.Context, source, version string, roots []string, inspect, deferBSR bool) (fetched modules.Fetched, err error) { ctx = c.operationContext(ctx, source, version) finish := gitcommand.Start(ctx, "resolve module") defer func() { finish(err) }() @@ -78,15 +86,25 @@ func (c *Cache) fetchWithRootSelection(ctx context.Context, source, version stri if version == "" { version = checkout.commit } - module, bindings, err := c.resolveBSRDependencies(ctx, checkout.module) - if err != nil { - return modules.Fetched{}, fmt.Errorf("resolveBSRDependencies: %w", err) + module := checkout.module + var bindings []v1.BSRResolution + if !deferBSR { + module, bindings, err = c.resolveBSRDependencies(ctx, module) + if err != nil { + return modules.Fetched{}, fmt.Errorf("resolveBSRDependencies: %w", err) + } } entry := v1.LockedModule{Source: source, Version: version, Commit: checkout.commit, Hash: hash, Roots: lockedV1ModuleRoots(module, roots), BSR: bindings} if hash != "" { if err := c.retainFetchedSnapshot(ctx, checkout, entry, files); err != nil { return modules.Fetched{}, fmt.Errorf("retainFetchedSnapshot: %w", err) } + } else if deferBSR { + // Root inference consumes this same exact revision later. Retain its + // immutable objects even though no complete snapshot can be installed. + if err := retainFetchedSource(ctx, c.root, checkout.dir, entry); err != nil { + return modules.Fetched{}, fmt.Errorf("retainFetchedSource: %w", err) + } } return modules.Fetched{Module: module, Lock: entry, Inspection: checkout.inspection}, nil } diff --git a/internal/adapters/plugin/builtin_impl.go b/internal/adapters/plugin/builtin_impl.go index ed09aeb5..7db71649 100644 --- a/internal/adapters/plugin/builtin_impl.go +++ b/internal/adapters/plugin/builtin_impl.go @@ -10,10 +10,8 @@ import ( "log/slog" "github.com/tetratelabs/wazero" - "github.com/tetratelabs/wazero/api" "github.com/tetratelabs/wazero/experimental" "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" - "github.com/tetratelabs/wazero/sys" "github.com/wasilibs/wazero-helpers/allocator" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/pluginpb" @@ -94,40 +92,82 @@ func (e *BuiltinPluginExecutor) Execute(ctx context.Context, plugin Info, reques // runWasmPlugin runs WASM module with custom stdin/stdout func (e *BuiltinPluginExecutor) runWasmPlugin(ctx context.Context, pluginName string, stdin io.Reader) (output []byte, runErr error) { - // Get WASM module and arguments for the plugin wasmBin, args, err := getWasmModule(pluginName) if err != nil { - return nil, fmt.Errorf("get wasm module for plugin %s: %w", pluginName, err) + return nil, fmt.Errorf("getWasmModule: %w", err) } + return runBuiltinModule(ctx, wasmBin, args, stdin) +} - // Create context with allocator - ctx = experimental.WithMemoryAllocator(ctx, allocator.NewNonMoving()) +func runBuiltinModule(ctx context.Context, binary []byte, args []string, stdin io.Reader) (output []byte, runErr error) { + if err := ctx.Err(); err != nil { + return nil, err + } + ctx, owner := EnsureBuiltinRun(ctx) + if owner != nil { + defer func() { + err := owner.Close(context.WithoutCancel(ctx)) + if err != nil { + output = nil + runErr = errors.Join(runErr, fmt.Errorf("Close: %w", err)) + } + }() + } + run := ctx.Value(builtinRunContextKey{}).(*BuiltinRun) + err := run.acquire(ctx) + if err != nil { + return nil, fmt.Errorf("acquire: %w", err) + } + defer run.work.Done() + + ctx, cancel := context.WithCancel(ctx) + stop := context.AfterFunc(run.ctx, cancel) + defer func() { + stop() + cancel() + }() + + var memory wazero.CompiledModule + if len(wasmMemory) > 0 { + memory, err = run.compiledModule(ctx, wasmMemory) + if err != nil { + return nil, fmt.Errorf("compiledModule: %w", err) + } + } + module, err := run.compiledModule(ctx, binary) + if err != nil { + return nil, fmt.Errorf("compiledModule: %w", err) + } - // Create wazero runtime - rt := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig().WithCoreFeatures(api.CoreFeaturesV2|experimental.CoreFeaturesThreads)) + ctx = experimental.WithMemoryAllocator(ctx, allocator.NewNonMoving()) + rt := run.runtimeFactory(ctx, run.config) // A failed runtime close must not turn into successful generated output. defer func() { - if err := rt.Close(ctx); err != nil { + err := rt.Close(context.WithoutCancel(ctx)) + if err != nil { output = nil runErr = errors.Join(runErr, fmt.Errorf("Close: %w", err)) } }() - // Instantiate WASI - wasi_snapshot_preview1.MustInstantiate(ctx, rt) + _, err = wasi_snapshot_preview1.Instantiate(ctx, rt) + if err != nil { + return nil, fmt.Errorf("Instantiate: %w", err) + } - // Instantiate memory module - if len(wasmMemory) > 0 { - if _, err = rt.InstantiateWithConfig(ctx, wasmMemory, wazero.NewModuleConfig().WithName("env")); err != nil { - return nil, fmt.Errorf("failed to instantiate memory module: %w", err) + if memory != nil { + _, err = rt.InstantiateModule(ctx, memory, wazero.NewModuleConfig().WithName("env")) + if err != nil { + return nil, fmt.Errorf("InstantiateModule: %w", err) } } // Read stdin into buffer var stdinBuf bytes.Buffer - if _, err = io.Copy(&stdinBuf, stdin); err != nil { - return nil, fmt.Errorf("failed to read stdin: %w", err) + _, err = io.Copy(&stdinBuf, stdin) + if err != nil { + return nil, fmt.Errorf("Copy: %w", err) } // Create buffer for stdout @@ -145,12 +185,12 @@ func (e *BuiltinPluginExecutor) runWasmPlugin(ctx context.Context, pluginName st WithArgs(args...) // Instantiate WASM module - _, err = rt.InstantiateWithConfig(ctx, wasmBin, cfg) + _, err = rt.InstantiateModule(ctx, module, cfg) if err != nil { - if sErr, ok := err.(*sys.ExitError); ok { //nolint:errorlint - return nil, fmt.Errorf("wasm plugin exited with code %d", sErr.ExitCode()) + if ctx.Err() != nil { + err = errors.Join(err, ctx.Err()) } - return nil, fmt.Errorf("failed to instantiate wasm module: %w", err) + return nil, fmt.Errorf("InstantiateModule: %w", err) } return stdoutBuf.Bytes(), nil diff --git a/internal/adapters/plugin/builtin_impl_test.go b/internal/adapters/plugin/builtin_impl_test.go new file mode 100644 index 00000000..36001336 --- /dev/null +++ b/internal/adapters/plugin/builtin_impl_test.go @@ -0,0 +1,109 @@ +package plugin + +import ( + "bytes" + "context" + "crypto/rand" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/experimental" + "github.com/tetratelabs/wazero/imports/wasi_snapshot_preview1" + "github.com/wasilibs/wazero-helpers/allocator" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/descriptorpb" + "google.golang.org/protobuf/types/pluginpb" + + "github.com/easyp-tech/easyp/internal/logger" +) + +func TestBuiltinCanceledBeforeExecution(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(t.Context()) + cancel() + executor := NewBuiltinPluginExecutor(logger.NewNop()) + + output, err := executor.runWasmPlugin(ctx, "memory", strings.NewReader("")) + + require.ErrorIs(t, err, context.Canceled) + assert.Nil(t, output) +} + +func TestBuiltinUniversalOutputCompatibility(t *testing.T) { + t.Parallel() + + request, err := proto.Marshal(&pluginpb.CodeGeneratorRequest{ + FileToGenerate: []string{"example.proto"}, + ProtoFile: []*descriptorpb.FileDescriptorProto{{ + Name: proto.String("example.proto"), Package: proto.String("example"), Syntax: proto.String("proto3"), + MessageType: []*descriptorpb.DescriptorProto{{ + Name: proto.String("Greeting"), + Field: []*descriptorpb.FieldDescriptorProto{{ + Name: proto.String("text"), Number: proto.Int32(1), + Label: descriptorpb.FieldDescriptorProto_LABEL_OPTIONAL.Enum(), + Type: descriptorpb.FieldDescriptorProto_TYPE_STRING.Enum(), + }}, + }}, + Service: []*descriptorpb.ServiceDescriptorProto{{ + Name: proto.String("Greeter"), + Method: []*descriptorpb.MethodDescriptorProto{{ + Name: proto.String("Hello"), InputType: proto.String(".example.Greeting"), OutputType: proto.String(".example.Greeting"), + }}, + }}, + }}, + }) + require.NoError(t, err) + ctx, run := EnsureBuiltinRun(t.Context()) + t.Cleanup(func() { assert.NoError(t, run.Close(context.Background())) }) + for _, name := range []string{"python", "grpc_python"} { + t.Run(name, func(t *testing.T) { + t.Parallel() + want := builtinDirectOutput(t, name, request) + executor := NewBuiltinPluginExecutor(logger.NewNop()) + + got, err := executor.runWasmPlugin(ctx, name, bytes.NewReader(request)) + + require.NoError(t, err) + assert.Equal(t, want, got, "compiled execution must preserve the original runtime's generated bytes") + var response pluginpb.CodeGeneratorResponse + require.NoError(t, proto.Unmarshal(got, &response)) + require.Empty(t, response.GetError()) + require.NotEmpty(t, response.File) + }) + } +} + +func TestBuiltinStandaloneModule(t *testing.T) { + t.Parallel() + + for range 2 { + output, err := runBuiltinModule(t.Context(), builtinEchoModule(t), []string{"echo"}, strings.NewReader("standalone")) + require.NoError(t, err) + assert.Equal(t, "standalone", string(output)) + } +} + +// builtinDirectOutput reproduces the previous per-invocation compilation path. +// It intentionally has no shared cache or context-termination instrumentation. +func builtinDirectOutput(t *testing.T, pluginName string, request []byte) []byte { + t.Helper() + ctx := experimental.WithMemoryAllocator(t.Context(), allocator.NewNonMoving()) + runtime := wazero.NewRuntimeWithConfig(ctx, wazero.NewRuntimeConfig().WithCoreFeatures(api.CoreFeaturesV2|experimental.CoreFeaturesThreads)) + defer func() { assert.NoError(t, runtime.Close(context.WithoutCancel(ctx))) }() + _, err := wasi_snapshot_preview1.Instantiate(ctx, runtime) + require.NoError(t, err) + _, err = runtime.InstantiateWithConfig(ctx, wasmMemory, wazero.NewModuleConfig().WithName("env")) + require.NoError(t, err) + var stdout, stderr bytes.Buffer + _, err = runtime.InstantiateWithConfig(ctx, protocGenUniversal, wazero.NewModuleConfig(). + WithSysNanosleep().WithSysNanotime().WithSysWalltime(). + WithStdin(bytes.NewReader(request)).WithStdout(&stdout).WithStderr(&stderr). + WithRandSource(rand.Reader).WithArgs("protoc-gen-universal", pluginName)) + require.NoError(t, err, stderr.String()) + return stdout.Bytes() +} diff --git a/internal/adapters/plugin/builtin_run.go b/internal/adapters/plugin/builtin_run.go new file mode 100644 index 00000000..2074923f --- /dev/null +++ b/internal/adapters/plugin/builtin_run.go @@ -0,0 +1,182 @@ +package plugin + +import ( + "bytes" + "context" + "crypto/sha256" + "errors" + "fmt" + "slices" + "sync" + + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/experimental" +) + +type builtinRunContextKey struct{} + +// BuiltinRun owns compiled builtin modules for one command. Each invocation +// uses its own runtime, module instances, memory, and I/O. +type BuiltinRun struct { + ctx context.Context + cancel context.CancelFunc + cache wazero.CompilationCache + config wazero.RuntimeConfig + runtimeFactory func(context.Context, wazero.RuntimeConfig) wazero.Runtime + + mu sync.Mutex + closed bool + compiler wazero.Runtime + modules map[[sha256.Size]byte]*builtinCompilation + work sync.WaitGroup + close sync.Once + closeErr error +} + +type builtinCompilation struct { + ready chan struct{} + module wazero.CompiledModule + err error +} + +// EnsureBuiltinRun reuses the command's builtin compilation scope, or creates +// one when absent. A non-nil owner must be closed after all callers finish. +// Nested callers receive a nil owner and must leave the existing run open. +func EnsureBuiltinRun(ctx context.Context) (context.Context, *BuiltinRun) { + if _, ok := ctx.Value(builtinRunContextKey{}).(*BuiltinRun); ok { + return ctx, nil + } + + commandCtx, cancel := context.WithCancel(ctx) + cache := wazero.NewCompilationCache() + run := &BuiltinRun{ + ctx: commandCtx, cancel: cancel, cache: cache, + config: wazero.NewRuntimeConfig(). + WithCoreFeatures(api.CoreFeaturesV2 | experimental.CoreFeaturesThreads). + WithCloseOnContextDone(true). + WithCompilationCache(cache), + runtimeFactory: wazero.NewRuntimeWithConfig, + modules: make(map[[sha256.Size]byte]*builtinCompilation), + } + return context.WithValue(ctx, builtinRunContextKey{}, run), run +} + +// Close stops new invocations, cancels active work, joins preparation and +// invocations, then releases compiled modules and their shared engine. +// Cleanup continues even when ctx is canceled. Repeated calls return the same +// cleanup error. Callers should normally join their invocations before Close. +func (r *BuiltinRun) Close(ctx context.Context) error { + r.close.Do(func() { + r.mu.Lock() + r.closed = true + r.cancel() + r.mu.Unlock() + r.work.Wait() + + cleanupCtx := context.WithoutCancel(ctx) + keys := make([][sha256.Size]byte, 0, len(r.modules)) + for key := range r.modules { + keys = append(keys, key) + } + slices.SortFunc(keys, func(left, right [sha256.Size]byte) int { + return bytes.Compare(left[:], right[:]) + }) + for _, key := range keys { + module := r.modules[key].module + if module == nil { + continue + } + err := module.Close(cleanupCtx) + if err != nil { + r.closeErr = errors.Join(r.closeErr, fmt.Errorf("Close: %w", err)) + } + } + if r.compiler != nil { + err := r.compiler.Close(cleanupCtx) + if err != nil { + r.closeErr = errors.Join(r.closeErr, fmt.Errorf("Close: %w", err)) + } + } + err := r.cache.Close(cleanupCtx) + if err != nil { + r.closeErr = errors.Join(r.closeErr, fmt.Errorf("Close: %w", err)) + } + }) + return r.closeErr +} + +func (r *BuiltinRun) acquire(ctx context.Context) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.closed { + return fmt.Errorf("builtin run is closed") + } + if err := ctx.Err(); err != nil { + return err + } + if err := r.ctx.Err(); err != nil { + return err + } + r.work.Add(1) + return nil +} + +func (r *BuiltinRun) compiledModule(ctx context.Context, binary []byte) (wazero.CompiledModule, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + key := sha256.Sum256(binary) + r.mu.Lock() + if r.closed { + r.mu.Unlock() + return nil, fmt.Errorf("builtin run is closed") + } + if err := r.ctx.Err(); err != nil { + r.mu.Unlock() + return nil, err + } + compilation, ok := r.modules[key] + if !ok { + compilation = &builtinCompilation{ready: make(chan struct{})} + r.modules[key] = compilation + if r.compiler == nil { + r.compiler = r.runtimeFactory(r.ctx, r.config) + } + r.work.Add(1) + go r.compile(binary, compilation) + } + r.mu.Unlock() + + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-r.ctx.Done(): + return nil, r.ctx.Err() + case <-compilation.ready: + if err := ctx.Err(); err != nil { + return nil, err + } + if err := r.ctx.Err(); err != nil { + return nil, err + } + return compilation.module, compilation.err + } +} + +func (r *BuiltinRun) compile(binary []byte, compilation *builtinCompilation) { + defer r.work.Done() + defer close(compilation.ready) + + // Preparation belongs to the command. Canceling the first waiting target + // must not poison the compiled module for the other targets. + if err := r.ctx.Err(); err != nil { + compilation.err = err + return + } + module, err := r.compiler.CompileModule(r.ctx, binary) + compilation.module = module + if err != nil { + compilation.err = fmt.Errorf("CompileModule: %w", err) + } +} diff --git a/internal/adapters/plugin/builtin_run_test.go b/internal/adapters/plugin/builtin_run_test.go new file mode 100644 index 00000000..3cb26b2b --- /dev/null +++ b/internal/adapters/plugin/builtin_run_test.go @@ -0,0 +1,452 @@ +package plugin + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + "io" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "testing/synctest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tetratelabs/wazero" + "github.com/tetratelabs/wazero/api" + "github.com/tetratelabs/wazero/experimental" +) + +func TestBuiltinRunCompilesOnceAndIsolatesInvocations(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + const count = 4 + var compilations atomic.Int64 + started := make(chan api.Memory, count) + release := make(chan struct{}) + releaseAll := sync.OnceFunc(func() { close(release) }) + ctx := experimental.WithFunctionListenerFactory(t.Context(), experimental.FunctionListenerFactoryFunc(func(def api.FunctionDefinition) experimental.FunctionListener { + if !slices.Contains(def.ExportNames(), "_start") { + return nil + } + compilations.Add(1) + return experimental.FunctionListenerFunc(func(_ context.Context, mod api.Module, _ api.FunctionDefinition, _ []uint64, _ experimental.StackIterator) { + started <- mod.Memory() + <-release + }) + })) + ctx, run := EnsureBuiltinRun(ctx) + require.NotNil(t, run) + defer func() { + releaseAll() + assert.NoError(t, run.Close(context.Background())) + }() + + module := builtinEchoModule(t) + type result struct { + input string + output []byte + err error + } + results := make(chan result, count) + for i := range count { + input := strings.Repeat(string(rune('a'+i)), i+1) + go func() { + output, err := runBuiltinModule(ctx, module, []string{"echo"}, strings.NewReader(input)) + results <- result{input: input, output: output, err: err} + }() + } + + synctest.Wait() + require.Len(t, started, count, "all invocations must overlap") + assert.EqualValues(t, 1, compilations.Load()) + memories := make([]api.Memory, 0, count) + for range count { + memory := <-started + require.NotNil(t, memory) + for _, previous := range memories { + assert.NotSame(t, previous, memory) + } + memories = append(memories, memory) + } + + releaseAll() + synctest.Wait() + require.Len(t, results, count) + for range count { + result := <-results + require.NoError(t, result.err) + assert.Equal(t, result.input, string(result.output)) + } + + // Reuse after the previous runtimes closed must keep the compiled code alive. + output, err := runBuiltinModule(ctx, module, []string{"echo"}, strings.NewReader("later")) + require.NoError(t, err) + assert.Equal(t, "later", string(output)) + assert.EqualValues(t, 1, compilations.Load()) + + first, err := run.compiledModule(ctx, wasmMemory) + require.NoError(t, err) + second, err := run.compiledModule(ctx, bytes.Clone(wasmMemory)) + require.NoError(t, err) + assert.Same(t, first, second, "equal binary contents share one compiled module") + }) +} + +func TestEnsureBuiltinRunReusesScope(t *testing.T) { + t.Parallel() + + ctx, run := EnsureBuiltinRun(t.Context()) + require.NotNil(t, run) + nested, owner := EnsureBuiltinRun(ctx) + assert.Same(t, ctx, nested) + assert.Nil(t, owner) + require.NoError(t, run.Close(context.Background())) + require.NoError(t, run.Close(context.Background())) + + output, err := runBuiltinModule(ctx, builtinEchoModule(t), []string{"echo"}, strings.NewReader("")) + require.ErrorContains(t, err, "closed") + assert.Nil(t, output) +} + +func TestBuiltinRunCloseJoinsInvocations(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + started := make(chan struct{}, 1) + release := make(chan struct{}) + releaseAll := sync.OnceFunc(func() { close(release) }) + ctx := experimental.WithFunctionListenerFactory(t.Context(), experimental.FunctionListenerFactoryFunc(func(def api.FunctionDefinition) experimental.FunctionListener { + if !slices.Contains(def.ExportNames(), "_start") { + return nil + } + return experimental.FunctionListenerFunc(func(context.Context, api.Module, api.FunctionDefinition, []uint64, experimental.StackIterator) { + started <- struct{}{} + <-release + }) + })) + ctx, run := EnsureBuiltinRun(ctx) + defer func() { + releaseAll() + assert.NoError(t, run.Close(context.Background())) + }() + results := make(chan error, 1) + module := builtinEchoModule(t) + go func() { + _, err := runBuiltinModule(ctx, module, []string{"echo"}, strings.NewReader("first")) + results <- err + }() + synctest.Wait() + require.Len(t, started, 1) + + closed := make(chan error, 1) + go func() { closed <- run.Close(context.Background()) }() + synctest.Wait() + assert.Empty(t, closed, "Close must join the invocation") + _, err := runBuiltinModule(ctx, module, []string{"echo"}, strings.NewReader("late")) + require.ErrorContains(t, err, "closed") + + releaseAll() + synctest.Wait() + require.Len(t, closed, 1) + require.NoError(t, <-closed) + require.Len(t, results, 1) + require.ErrorIs(t, <-results, context.Canceled) + }) +} + +func TestBuiltinRunCanceledWaiterDoesNotCancelCompilation(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + var compilations atomic.Int64 + compiling := make(chan struct{}, 1) + release := make(chan struct{}) + releaseAll := sync.OnceFunc(func() { close(release) }) + ctx := experimental.WithFunctionListenerFactory(t.Context(), experimental.FunctionListenerFactoryFunc(func(def api.FunctionDefinition) experimental.FunctionListener { + if slices.Contains(def.ExportNames(), "_start") { + compilations.Add(1) + compiling <- struct{}{} + <-release + } + return nil + })) + ctx, run := EnsureBuiltinRun(ctx) + defer func() { + releaseAll() + assert.NoError(t, run.Close(context.Background())) + }() + module := builtinEchoModule(t) + caller, cancel := context.WithCancel(ctx) + defer cancel() + first := make(chan error, 1) + go func() { + _, err := runBuiltinModule(caller, module, []string{"echo"}, strings.NewReader("canceled")) + first <- err + }() + synctest.Wait() + require.Len(t, compiling, 1) + cancel() + synctest.Wait() + require.Len(t, first, 1, "the initiating caller must stop waiting promptly") + require.ErrorIs(t, <-first, context.Canceled) + + second := make(chan error, 1) + go func() { + output, err := runBuiltinModule(ctx, module, []string{"echo"}, strings.NewReader("survived")) + if err == nil && string(output) != "survived" { + err = errors.New("unexpected output: " + string(output)) + } + second <- err + }() + synctest.Wait() + assert.Empty(t, second) + releaseAll() + synctest.Wait() + require.Len(t, second, 1) + require.NoError(t, <-second) + assert.EqualValues(t, 1, compilations.Load()) + }) +} + +func TestBuiltinRunCloseJoinsCompilation(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + compiling := make(chan struct{}, 1) + release := make(chan struct{}) + releaseAll := sync.OnceFunc(func() { close(release) }) + ctx := experimental.WithFunctionListenerFactory(t.Context(), experimental.FunctionListenerFactoryFunc(func(api.FunctionDefinition) experimental.FunctionListener { + compiling <- struct{}{} + <-release + return nil + })) + ctx, run := EnsureBuiltinRun(ctx) + defer func() { + releaseAll() + assert.NoError(t, run.Close(context.Background())) + }() + module := builtinEchoModule(t) + result := make(chan error, 1) + go func() { + _, err := run.compiledModule(ctx, module) + result <- err + }() + synctest.Wait() + require.Len(t, compiling, 1) + + closed := make(chan error, 1) + go func() { closed <- run.Close(context.Background()) }() + synctest.Wait() + assert.Empty(t, closed, "Close must wait for compilation to finish") + require.Len(t, result, 1) + require.ErrorIs(t, <-result, context.Canceled) + + releaseAll() + synctest.Wait() + require.Len(t, closed, 1) + require.NoError(t, <-closed) + }) +} + +func TestBuiltinRunCompilesEachDistinctBinary(t *testing.T) { + t.Parallel() + + var compilations atomic.Int64 + ctx := experimental.WithFunctionListenerFactory(t.Context(), experimental.FunctionListenerFactoryFunc(func(api.FunctionDefinition) experimental.FunctionListener { + compilations.Add(1) + return nil + })) + ctx, run := EnsureBuiltinRun(ctx) + defer func() { assert.NoError(t, run.Close(context.Background())) }() + module := builtinEchoModule(t) + first, err := run.compiledModule(ctx, module) + require.NoError(t, err) + copy, err := run.compiledModule(ctx, bytes.Clone(module)) + require.NoError(t, err) + assert.Same(t, first, copy) + + // Append a custom section to create different valid binary contents. + different := append(bytes.Clone(module), 0, 3, 1, 'x', 0) + second, err := run.compiledModule(ctx, different) + require.NoError(t, err) + assert.NotSame(t, first, second) + assert.EqualValues(t, 2, compilations.Load()) +} + +func TestBuiltinRunFailedCompilationIsMemoizedAndCleanupErrorsSurvive(t *testing.T) { + t.Parallel() + + ctx, run := EnsureBuiltinRun(t.Context()) + runtimeCloseErr := errors.New("runtime close failed") + cacheCloseErr := errors.New("cache close failed") + var compilations, runtimeCloses, cacheCloses atomic.Int64 + closeContexts := make(chan error, 2) + run.runtimeFactory = func(ctx context.Context, config wazero.RuntimeConfig) wazero.Runtime { + return &builtinRuntimeProbe{ + Runtime: wazero.NewRuntimeWithConfig(ctx, config), compilations: &compilations, + closes: &runtimeCloses, closeContexts: closeContexts, closeErr: runtimeCloseErr, + } + } + run.cache = &builtinCacheProbe{CompilationCache: run.cache, closes: &cacheCloses, closeErr: cacheCloseErr} + + _, firstErr := run.compiledModule(ctx, []byte("invalid WASM")) + require.Error(t, firstErr) + _, secondErr := run.compiledModule(ctx, []byte("invalid WASM")) + require.EqualError(t, secondErr, firstErr.Error()) + assert.EqualValues(t, 1, compilations.Load()) + + canceled, cancel := context.WithCancel(ctx) + cancel() + err := run.Close(canceled) + require.ErrorIs(t, err, runtimeCloseErr) + require.ErrorIs(t, err, cacheCloseErr) + assert.NoError(t, <-closeContexts, "cleanup must receive an uncanceled context") + assert.EqualValues(t, 1, runtimeCloses.Load()) + assert.EqualValues(t, 1, cacheCloses.Load()) + assert.Equal(t, err, run.Close(canceled)) + assert.EqualValues(t, 1, runtimeCloses.Load()) + assert.EqualValues(t, 1, cacheCloses.Load()) +} + +func TestBuiltinRunInvocationRetainsCleanupErrors(t *testing.T) { + t.Parallel() + + readErr := errors.New("stdin failed") + tests := []struct { + name string + reader func() io.Reader + want error + }{ + {name: "successful_execution", reader: func() io.Reader { return strings.NewReader("hello") }}, + {name: "failed_execution", reader: func() io.Reader { return builtinErrorReader{err: readErr} }, want: readErr}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + ctx, run := EnsureBuiltinRun(t.Context()) + closeErr := errors.New("runtime close failed") + var compilations, closes atomic.Int64 + closeContexts := make(chan error, 2) + run.runtimeFactory = func(ctx context.Context, config wazero.RuntimeConfig) wazero.Runtime { + return &builtinRuntimeProbe{ + Runtime: wazero.NewRuntimeWithConfig(ctx, config), compilations: &compilations, + closes: &closes, closeContexts: closeContexts, closeErr: closeErr, + } + } + + output, err := runBuiltinModule(ctx, builtinEchoModule(t), []string{"echo"}, tt.reader()) + + require.ErrorIs(t, err, closeErr) + assert.Nil(t, output) + if tt.want != nil { + assert.ErrorIs(t, err, tt.want) + } + assert.NoError(t, <-closeContexts) + require.ErrorIs(t, run.Close(context.Background()), closeErr) + assert.NoError(t, <-closeContexts) + assert.EqualValues(t, 2, closes.Load()) + }) + } +} + +func TestBuiltinRunCanceledCommandDoesNotCompile(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithCancel(t.Context()) + ctx, run := EnsureBuiltinRun(ctx) + cancel() + var runtimes atomic.Int64 + run.runtimeFactory = func(ctx context.Context, config wazero.RuntimeConfig) wazero.Runtime { + runtimes.Add(1) + return wazero.NewRuntimeWithConfig(ctx, config) + } + + output, err := runBuiltinModule(ctx, builtinEchoModule(t), []string{"echo"}, strings.NewReader("")) + + require.ErrorIs(t, err, context.Canceled) + assert.Nil(t, output) + assert.Zero(t, runtimes.Load()) + require.NoError(t, run.Close(ctx)) +} + +func TestBuiltinRunCancellationTerminatesWasm(t *testing.T) { + t.Parallel() + + synctest.Test(t, func(t *testing.T) { + started := make(chan struct{}, 1) + ctx := experimental.WithFunctionListenerFactory(t.Context(), experimental.FunctionListenerFactoryFunc(func(def api.FunctionDefinition) experimental.FunctionListener { + if !slices.Contains(def.ExportNames(), "_start") { + return nil + } + return experimental.FunctionListenerFunc(func(context.Context, api.Module, api.FunctionDefinition, []uint64, experimental.StackIterator) { + started <- struct{}{} + }) + })) + ctx, run := EnsureBuiltinRun(ctx) + defer func() { assert.NoError(t, run.Close(context.Background())) }() + caller, cancel := context.WithCancel(ctx) + defer cancel() + module := builtinWasmHex(t, "0061736d0100000001040160000003020100070a01065f737461727400000a0901070003400c000b0b") + results := make(chan error, 1) + go func() { + _, err := runBuiltinModule(caller, module, []string{"loop"}, strings.NewReader("")) + results <- err + }() + <-started + cancel() + require.ErrorIs(t, <-results, context.Canceled) + }) +} + +func builtinEchoModule(t *testing.T) []byte { + t.Helper() + // This WASI fixture imports env.memory and echoes stdin. Both a mutable + // global and memory byte 100 trap when reused by a later invocation. + return builtinWasmHex(t, "0061736d01000000010c0260047f7f7f7f017f60000002550316776173695f736e617073686f745f70726576696577310766645f72656164000016776173695f736e617073686f745f70726576696577310866645f7772697465000003656e76066d656d6f727902010d808004030201010606017f0141000b071302066d656d6f72790200065f737461727400020a50014e0023000440000b4101240041e4002d00000440000b41e40041013a000041004180083602004104418002360200410041004101410810001a41044108280200360200410141004101410c10011a0b") +} + +func builtinWasmHex(t *testing.T, encoded string) []byte { + t.Helper() + binary, err := hex.DecodeString(encoded) + require.NoError(t, err) + return binary +} + +type builtinRuntimeProbe struct { + wazero.Runtime + compilations *atomic.Int64 + closes *atomic.Int64 + closeContexts chan<- error + closeErr error +} + +func (r *builtinRuntimeProbe) CompileModule(ctx context.Context, binary []byte) (wazero.CompiledModule, error) { + r.compilations.Add(1) + return r.Runtime.CompileModule(ctx, binary) +} + +func (r *builtinRuntimeProbe) Close(ctx context.Context) error { + r.closes.Add(1) + r.closeContexts <- ctx.Err() + return errors.Join(r.Runtime.Close(ctx), r.closeErr) +} + +type builtinCacheProbe struct { + wazero.CompilationCache + closes *atomic.Int64 + closeErr error +} + +func (c *builtinCacheProbe) Close(ctx context.Context) error { + c.closes.Add(1) + return errors.Join(c.CompilationCache.Close(ctx), c.closeErr) +} + +type builtinErrorReader struct{ err error } + +func (r builtinErrorReader) Read([]byte) (int, error) { return 0, r.err } diff --git a/internal/core/generate.go b/internal/core/generate.go index cda73678..a53b3148 100644 --- a/internal/core/generate.go +++ b/internal/core/generate.go @@ -25,8 +25,8 @@ import ( pluginexecutor "github.com/easyp-tech/easyp/internal/adapters/plugin" "github.com/easyp-tech/easyp/internal/core/path_helpers" + "github.com/easyp-tech/easyp/internal/parallel" "github.com/easyp-tech/easyp/internal/sourceview" - "github.com/easyp-tech/easyp/internal/version" ) // GenerationPlan holds a compiled graph with managed options already applied. @@ -204,104 +204,39 @@ func (p *GenerationPlan) Execute(ctx context.Context) error { return bucket.DumpToFs(ctx) } -// ExecuteInto stages plugin results in a shared bucket, rejecting conflicting outputs. -func (p *GenerationPlan) ExecuteInto(ctx context.Context, filesToWrite *GenerateBucket) error { - c, root := p.core, p.root - files, fileDescriptors, dependencyFiles := p.files, p.descriptors, p.dependencyFiles - - for _, plugin := range c.plugins { - filesToGenerate := slices.Clone(files) - - if plugin.WithImports { - for _, dependency := range dependencyFiles { - if !slices.Contains(filesToGenerate, dependency) { - filesToGenerate = append(filesToGenerate, dependency) - } +// ExecuteInto stages plugin results in configuration order after bounded parallel execution. +func (p *GenerationPlan) ExecuteInto(ctx context.Context, filesToWrite *GenerateBucket) (resultErr error) { + ctx, builtinRun := pluginexecutor.EnsureBuiltinRun(ctx) + if builtinRun != nil { + defer func() { + err := builtinRun.Close(context.WithoutCancel(ctx)) + if err != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("Close: %w", err)) } + }() + } + results := make([]pluginResult, len(p.core.plugins)) + err := parallel.Run(ctx, len(results), maxPluginWorkers, func(index int) error { + results[index] = p.invokePlugin(ctx, p.core.plugins[index]) + return nil + }) + if err != nil { + return fmt.Errorf("Run: %w", err) + } + for index, result := range results { + if err := ctx.Err(); err != nil { + return err } - - req := &pluginpb.CodeGeneratorRequest{ - FileToGenerate: filesToGenerate, - ProtoFile: fileDescriptors, - CompilerVersion: version.CompilerVersion(), - } - - // Executors receive independent requests; prepared descriptors stay immutable. - req = proto.Clone(req).(*pluginpb.CodeGeneratorRequest) - - executor := c.getExecutor(plugin) - - source := plugin.Source.Name - if plugin.Source.Remote != "" { - source = plugin.Source.Remote - } - - if plugin.Source.Path != "" { - source = plugin.Source.Path + if result.err != nil { + return result.err } - - resp, err := executor.Execute(ctx, pluginexecutor.Info{ - Source: source, - WorkDir: c.pluginWorkDir, - Command: plugin.Source.Command, - Options: plugin.Options, - }, req) + err := p.applyPluginResponse(ctx, p.core.plugins[index], result, filesToWrite) if err != nil { - return fmt.Errorf("execute plugin %s: %w, executor: %s", source, err, executor.GetName()) - } - - if resp.Error != nil { - return fmt.Errorf("plugin %s error: %s, executor: %s", plugin.Source, *resp.Error, executor.GetName()) - } - - outputDir := root - if plugin.Out != "" { - outputDir = filepath.Join(root, plugin.Out) - } - var goOutputDirs map[string]string - paths := plugin.Options["paths"] - if c.goPackageOutputPrefix != "" && len(paths) > 0 && paths[len(paths)-1] == "source_relative" { - goOutputDirs = goPackageOutputDirs(c.goPackageOutputPrefix, filesToGenerate, fileDescriptors, plugin.Options) - } - for _, file := range resp.File { - originalName := file.GetName() - if !filepath.IsLocal(filepath.FromSlash(originalName)) { - return fmt.Errorf("plugin returned invalid output path %q", originalName) - } - outputName := originalName - if file.GetInsertionPoint() == "" { - outputName = goPackageOutputPath(file, goOutputDirs) - } - if !filepath.IsLocal(filepath.FromSlash(outputName)) { - return fmt.Errorf("plugin returned invalid output path %q", outputName) - } - path := filepath.Join(outputDir, filepath.FromSlash(outputName)) - originalPath := filepath.Join(outputDir, filepath.FromSlash(originalName)) - if file.GetInsertionPoint() != "" { - path, err = filesToWrite.insertionOutputPath(originalPath) - if err != nil { - return fmt.Errorf("insertionOutputPath: %w", err) - } - } - - c.logger.Debug(ctx, "generated file", - slog.String("plugin", source), - slog.String("file", outputName), - slog.String("plugin_out", plugin.Out), - slog.String("full_path", path), - ) - - if err := addFileWithInsertionPoint(ctx, path, file, filesToWrite); err != nil { - return fmt.Errorf("addFileWithInsertionPoint: %w", err) - } - if file.GetInsertionPoint() == "" { - filesToWrite.recordOutputPath(originalPath, path) - } + return err } + results[index] = pluginResult{} } - - c.logger.Info(ctx, "code generation completed") - + p.core.logger.Info(ctx, "code generation completed") return nil } diff --git a/internal/core/generate_bucket.go b/internal/core/generate_bucket.go index 5383e918..e5007776 100644 --- a/internal/core/generate_bucket.go +++ b/internal/core/generate_bucket.go @@ -5,6 +5,7 @@ import ( "fmt" "go/parser" "go/token" + "maps" "os" "path/filepath" "slices" @@ -81,13 +82,21 @@ func (b *GenerateBucket) insertionOutputPath(originalPath string) (string, error } } -func (b *GenerateBucket) DumpToFs(_ context.Context) error { +func (b *GenerateBucket) DumpToFs(ctx context.Context) error { b.lock.Lock() defer b.lock.Unlock() + if err := ctx.Err(); err != nil { + return err + } + paths := slices.Sorted(maps.Keys(b.filesToWrite)) // Validate the complete generated Go layout before writing any file. packages := make(map[string]string) - for path, file := range b.filesToWrite { + for _, path := range paths { + if err := ctx.Err(); err != nil { + return err + } + file := b.filesToWrite[path] if !strings.HasSuffix(path, ".go") { continue } @@ -104,8 +113,12 @@ func (b *GenerateBucket) DumpToFs(_ context.Context) error { } packages[key] = parsed.Name.Name } - // TODO: Есть возможность писать файлы асинхронно (MkdirAll до) - for path, file := range b.filesToWrite { + // Apply the completed bucket in path order so write failures are deterministic too. + for _, path := range paths { + if err := ctx.Err(); err != nil { + return err + } + file := b.filesToWrite[path] dir := filepath.Dir(path) if err := os.MkdirAll(dir, 0755); err != nil { return fmt.Errorf("os.MkdirAll %s: %w", dir, err) diff --git a/internal/core/generate_bucket_test.go b/internal/core/generate_bucket_test.go new file mode 100644 index 00000000..8387c9f6 --- /dev/null +++ b/internal/core/generate_bucket_test.go @@ -0,0 +1,35 @@ +package core + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGenerateBucketCancellationPreventsWrites(t *testing.T) { + t.Parallel() + root := t.TempDir() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + bucket := NewGenerateBucket() + output := filepath.Join(root, "out.txt") + bucket.PutFile(ctx, output, []byte("data")) + require.ErrorIs(t, bucket.DumpToFs(ctx), context.Canceled) + _, err := os.Stat(output) + assert.ErrorIs(t, err, os.ErrNotExist) +} + +func TestGenerateBucketValidationUsesPathOrder(t *testing.T) { + t.Parallel() + root := t.TempDir() + for range 20 { + bucket := NewGenerateBucket() + bucket.PutFile(t.Context(), filepath.Join(root, "z.go"), []byte("package {")) + bucket.PutFile(t.Context(), filepath.Join(root, "a.go"), []byte("package {")) + require.ErrorContains(t, bucket.DumpToFs(t.Context()), filepath.Join(root, "a.go")) + } +} diff --git a/internal/core/generate_compiler_version_test.go b/internal/core/generate_compiler_version_test.go index cdd4f3e0..23b7d5d9 100644 --- a/internal/core/generate_compiler_version_test.go +++ b/internal/core/generate_compiler_version_test.go @@ -6,6 +6,7 @@ import ( "os/exec" "path/filepath" "strings" + "sync" "testing" "google.golang.org/protobuf/proto" @@ -16,11 +17,20 @@ import ( ) type captureExecutor struct { - requests []*pluginpb.CodeGeneratorRequest + mu sync.Mutex + requests []*pluginpb.CodeGeneratorRequest + requestsBySource map[string]*pluginpb.CodeGeneratorRequest } -func (c *captureExecutor) Execute(_ context.Context, _ pluginexecutor.Info, request *pluginpb.CodeGeneratorRequest) (*pluginpb.CodeGeneratorResponse, error) { - c.requests = append(c.requests, proto.Clone(request).(*pluginpb.CodeGeneratorRequest)) +func (c *captureExecutor) Execute(_ context.Context, info pluginexecutor.Info, request *pluginpb.CodeGeneratorRequest) (*pluginpb.CodeGeneratorResponse, error) { + c.mu.Lock() + defer c.mu.Unlock() + request = proto.Clone(request).(*pluginpb.CodeGeneratorRequest) + c.requests = append(c.requests, request) + if c.requestsBySource == nil { + c.requestsBySource = make(map[string]*pluginpb.CodeGeneratorRequest) + } + c.requestsBySource[info.Source] = request return &pluginpb.CodeGeneratorResponse{}, nil } diff --git a/internal/core/generate_descriptors_test.go b/internal/core/generate_descriptors_test.go index bfa27212..d5d9eef5 100644 --- a/internal/core/generate_descriptors_test.go +++ b/internal/core/generate_descriptors_test.go @@ -2,6 +2,7 @@ package core import ( "context" + "fmt" "os" "path/filepath" "testing" @@ -72,7 +73,7 @@ func TestGenerateDescriptors(t *testing.T) { writeGenerationSources(t, root, tt.files) plugins := make([]Plugin, len(tt.withImports)) for i, withImports := range tt.withImports { - plugins[i] = Plugin{Source: PluginSource{Name: "custom-plugin"}, WithImports: withImports} + plugins[i] = Plugin{Source: PluginSource{Name: fmt.Sprintf("custom-plugin-%d", i)}, WithImports: withImports} } executor := &captureExecutor{} app := testCoreWithPlugins(plugins, executor) @@ -82,7 +83,9 @@ func TestGenerateDescriptors(t *testing.T) { require.NoError(t, err) require.Len(t, executor.requests, len(tt.wantGenerated)) - for i, request := range executor.requests { + for i, plugin := range plugins { + request := executor.requestsBySource[plugin.Source.Name] + require.NotNil(t, request) assert.Equal(t, tt.wantOrder, descriptorNames(request.GetProtoFile())) assert.Equal(t, tt.wantGenerated[i], request.GetFileToGenerate()) } diff --git a/internal/core/generate_parallel_test.go b/internal/core/generate_parallel_test.go new file mode 100644 index 00000000..7e2de196 --- /dev/null +++ b/internal/core/generate_parallel_test.go @@ -0,0 +1,191 @@ +package core + +import ( + "context" + "errors" + "os" + "path/filepath" + "sync/atomic" + "testing" + "testing/synctest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/descriptorpb" + "google.golang.org/protobuf/types/pluginpb" + + pluginexecutor "github.com/easyp-tech/easyp/internal/adapters/plugin" +) + +type parallelPluginExecutor struct { + execute func(context.Context, pluginexecutor.Info, *pluginpb.CodeGeneratorRequest) (*pluginpb.CodeGeneratorResponse, error) +} + +func (e parallelPluginExecutor) Execute(ctx context.Context, info pluginexecutor.Info, request *pluginpb.CodeGeneratorRequest) (*pluginpb.CodeGeneratorResponse, error) { + return e.execute(ctx, info, request) +} + +func (parallelPluginExecutor) GetName() string { return "parallelPluginExecutor" } + +func TestPluginsOverlapAndApplyResponsesInConfigurationOrder(t *testing.T) { + t.Parallel() + root := t.TempDir() + synctest.Test(t, func(t *testing.T) { + started := make(chan string, 3) + release := map[string]chan struct{}{ + "producer": make(chan struct{}), + "first": make(chan struct{}), + "second": make(chan struct{}), + } + responses := map[string]*pluginpb.CodeGeneratorResponse{ + "producer": {File: []*pluginpb.CodeGeneratorResponse_File{{ + Name: proto.String("out.txt"), Content: proto.String("base\n// @@protoc_insertion_point(items)\n"), + }}}, + "first": {File: []*pluginpb.CodeGeneratorResponse_File{{ + Name: proto.String("out.txt"), InsertionPoint: proto.String("items"), Content: proto.String("first\n"), + }}}, + "second": {File: []*pluginpb.CodeGeneratorResponse_File{{ + Name: proto.String("out.txt"), InsertionPoint: proto.String("items"), Content: proto.String("second\n"), + }}}, + } + executor := parallelPluginExecutor{execute: func(_ context.Context, info pluginexecutor.Info, request *pluginpb.CodeGeneratorRequest) (*pluginpb.CodeGeneratorResponse, error) { + // Every executor may mutate its request, including nested descriptors. + request.FileToGenerate[0] = info.Source + request.ProtoFile[0].Name = proto.String(info.Source) + request.Parameter = proto.String(info.Source) + started <- info.Source + <-release[info.Source] + return responses[info.Source], nil + }} + app := testCoreWithPlugins([]Plugin{ + {Source: PluginSource{Name: "producer"}}, + {Source: PluginSource{Remote: "first"}}, + {Source: PluginSource{Name: "second", Command: []string{"custom-plugin"}}}, + }, executor) + plan := &GenerationPlan{core: app, root: root, files: []string{"input.proto"}, + descriptors: []*descriptorpb.FileDescriptorProto{{Name: proto.String("input.proto")}}, + } + bucket := NewGenerateBucket() + done := make(chan error, 1) + go func() { done <- plan.ExecuteInto(t.Context(), bucket) }() + synctest.Wait() + assert.Len(t, started, 3, "independent plugin calls must overlap") + // Complete responses in reverse order and let each worker finish. + close(release["second"]) + synctest.Wait() + close(release["first"]) + synctest.Wait() + close(release["producer"]) + require.NoError(t, <-done) + output, ok := bucket.GetFile(t.Context(), filepath.Join(root, "out.txt")) + require.True(t, ok) + assert.Equal(t, "base\nfirst\nsecond\n// @@protoc_insertion_point(items)", string(output.Data())) + assert.Equal(t, []string{"input.proto"}, plan.files) + assert.Equal(t, "input.proto", plan.descriptors[0].GetName()) + }) +} + +func TestParallelPluginErrorsFollowConfigurationOrder(t *testing.T) { + t.Parallel() + root := t.TempDir() + firstErr := errors.New("first plugin failed") + laterErr := errors.New("later plugin failed") + synctest.Test(t, func(t *testing.T) { + laterFinished := make(chan struct{}) + executor := parallelPluginExecutor{execute: func(_ context.Context, info pluginexecutor.Info, _ *pluginpb.CodeGeneratorRequest) (*pluginpb.CodeGeneratorResponse, error) { + if info.Source == "first" { + <-laterFinished + return nil, firstErr + } + close(laterFinished) + return nil, laterErr + }} + plan := &GenerationPlan{core: testCoreWithPlugins([]Plugin{ + {Source: PluginSource{Name: "first"}}, {Source: PluginSource{Name: "later"}}, + }, executor), root: root} + require.ErrorIs(t, plan.Execute(t.Context()), firstErr) + }) +} + +func TestEarlierInvalidOutputPrecedesLaterExecutionError(t *testing.T) { + t.Parallel() + executor := parallelPluginExecutor{execute: func(_ context.Context, info pluginexecutor.Info, _ *pluginpb.CodeGeneratorRequest) (*pluginpb.CodeGeneratorResponse, error) { + if info.Source == "invalid-output" { + return &pluginpb.CodeGeneratorResponse{File: []*pluginpb.CodeGeneratorResponse_File{{Name: proto.String("../escape")}}}, nil + } + return nil, errors.New("later execution error") + }} + plan := &GenerationPlan{core: testCoreWithPlugins([]Plugin{ + {Source: PluginSource{Name: "invalid-output"}}, {Source: PluginSource{Name: "later"}}, + }, executor), root: t.TempDir()} + require.ErrorContains(t, plan.Execute(t.Context()), "invalid output path") +} + +func TestParallelPluginFailureDoesNotWriteSuccessfulOutputs(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + name string + response *pluginpb.CodeGeneratorResponse + message string + }{ + {name: "protocol error", response: &pluginpb.CodeGeneratorResponse{Error: proto.String("generation failed")}, message: "generation failed"}, + {name: "missing response", message: "returned no response"}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + executor := parallelPluginExecutor{execute: func(_ context.Context, info pluginexecutor.Info, _ *pluginpb.CodeGeneratorRequest) (*pluginpb.CodeGeneratorResponse, error) { + if info.Source == "failed" { + return tt.response, nil + } + return &pluginpb.CodeGeneratorResponse{File: []*pluginpb.CodeGeneratorResponse_File{{ + Name: proto.String("out.txt"), Content: proto.String("successful output"), + }}}, nil + }} + root := t.TempDir() + plan := &GenerationPlan{core: testCoreWithPlugins([]Plugin{ + {Source: PluginSource{Name: "successful"}}, {Source: PluginSource{Name: "failed"}}, + }, executor), root: root} + require.ErrorContains(t, plan.Execute(t.Context()), tt.message) + _, err := os.Stat(filepath.Join(root, "out.txt")) + assert.ErrorIs(t, err, os.ErrNotExist) + }) + } +} + +func TestParallelPluginsAreBoundedAndCancellationJoins(t *testing.T) { + t.Parallel() + root := t.TempDir() + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + release := make(chan struct{}) + var started atomic.Int64 + executor := parallelPluginExecutor{execute: func(_ context.Context, _ pluginexecutor.Info, _ *pluginpb.CodeGeneratorRequest) (*pluginpb.CodeGeneratorResponse, error) { + started.Add(1) + <-release + return &pluginpb.CodeGeneratorResponse{File: []*pluginpb.CodeGeneratorResponse_File{{Name: proto.String("out.txt"), Content: proto.String("data")}}}, nil + }} + plugins := make([]Plugin, 12) + for index := range plugins { + plugins[index].Source.Name = "plugin" + } + plan := &GenerationPlan{core: testCoreWithPlugins(plugins, executor), root: root} + done := make(chan error, 1) + go func() { done <- plan.Execute(ctx) }() + synctest.Wait() + assert.EqualValues(t, 4, started.Load()) + cancel() + synctest.Wait() + select { + case <-done: + t.Error("returned before active plugin calls finished") + default: + } + close(release) + require.ErrorIs(t, <-done, context.Canceled) + assert.EqualValues(t, 4, started.Load()) + _, err := os.Stat(filepath.Join(root, "out.txt")) + assert.ErrorIs(t, err, os.ErrNotExist) + }) +} diff --git a/internal/core/generate_plugins.go b/internal/core/generate_plugins.go new file mode 100644 index 00000000..e3b72244 --- /dev/null +++ b/internal/core/generate_plugins.go @@ -0,0 +1,119 @@ +package core + +import ( + "context" + "fmt" + "log/slog" + "path/filepath" + "slices" + + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/pluginpb" + + pluginexecutor "github.com/easyp-tech/easyp/internal/adapters/plugin" + "github.com/easyp-tech/easyp/internal/version" +) + +// Bound concurrent plugin processes and independent WASM memories as well as CPU work. +const maxPluginWorkers = 4 + +type pluginResult struct { + source string + filesToGenerate []string + response *pluginpb.CodeGeneratorResponse + err error +} + +// invokePlugin owns its request and response; prepared descriptors remain immutable. +func (p *GenerationPlan) invokePlugin(ctx context.Context, plugin Plugin) pluginResult { + filesToGenerate := slices.Clone(p.files) + if plugin.WithImports { + for _, dependency := range p.dependencyFiles { + if !slices.Contains(filesToGenerate, dependency) { + filesToGenerate = append(filesToGenerate, dependency) + } + } + } + request := &pluginpb.CodeGeneratorRequest{ + FileToGenerate: filesToGenerate, + ProtoFile: p.descriptors, + CompilerVersion: version.CompilerVersion(), + } + request = proto.Clone(request).(*pluginpb.CodeGeneratorRequest) + source := plugin.Source.Name + if plugin.Source.Remote != "" { + source = plugin.Source.Remote + } + if plugin.Source.Path != "" { + source = plugin.Source.Path + } + result := pluginResult{source: source, filesToGenerate: filesToGenerate} + executor := p.core.getExecutor(plugin) + response, err := executor.Execute(ctx, pluginexecutor.Info{ + Source: source, WorkDir: p.core.pluginWorkDir, + Command: plugin.Source.Command, Options: plugin.Options, + }, request) + if err != nil { + result.err = fmt.Errorf("execute plugin %s: %w, executor: %s", source, err, executor.GetName()) + return result + } + if response == nil { + result.err = fmt.Errorf("plugin %s returned no response, executor: %s", source, executor.GetName()) + return result + } + if response.Error != nil { + result.err = fmt.Errorf("plugin %s error: %s, executor: %s", plugin.Source, *response.Error, executor.GetName()) + return result + } + result.response = response + return result +} + +// applyPluginResponse runs serially in configuration order, including insertion points. +func (p *GenerationPlan) applyPluginResponse(ctx context.Context, plugin Plugin, result pluginResult, bucket *GenerateBucket) error { + c := p.core + outputDir := p.root + if plugin.Out != "" { + outputDir = filepath.Join(p.root, plugin.Out) + } + var goOutputDirs map[string]string + paths := plugin.Options["paths"] + if c.goPackageOutputPrefix != "" && len(paths) > 0 && paths[len(paths)-1] == "source_relative" { + goOutputDirs = goPackageOutputDirs(c.goPackageOutputPrefix, result.filesToGenerate, p.descriptors, plugin.Options) + } + for _, file := range result.response.File { + originalName := file.GetName() + if !filepath.IsLocal(filepath.FromSlash(originalName)) { + return fmt.Errorf("plugin returned invalid output path %q", originalName) + } + outputName := originalName + if file.GetInsertionPoint() == "" { + outputName = goPackageOutputPath(file, goOutputDirs) + } + if !filepath.IsLocal(filepath.FromSlash(outputName)) { + return fmt.Errorf("plugin returned invalid output path %q", outputName) + } + path := filepath.Join(outputDir, filepath.FromSlash(outputName)) + originalPath := filepath.Join(outputDir, filepath.FromSlash(originalName)) + if file.GetInsertionPoint() != "" { + var err error + path, err = bucket.insertionOutputPath(originalPath) + if err != nil { + return fmt.Errorf("insertionOutputPath: %w", err) + } + } + c.logger.Debug(ctx, "generated file", + slog.String("plugin", result.source), + slog.String("file", outputName), + slog.String("plugin_out", plugin.Out), + slog.String("full_path", path), + ) + if err := addFileWithInsertionPoint(ctx, path, file, bucket); err != nil { + return fmt.Errorf("addFileWithInsertionPoint: %w", err) + } + if file.GetInsertionPoint() == "" { + bucket.recordOutputPath(originalPath, path) + } + } + return nil +} diff --git a/internal/core/generate_selection_test.go b/internal/core/generate_selection_test.go index 75e06815..628db021 100644 --- a/internal/core/generate_selection_test.go +++ b/internal/core/generate_selection_test.go @@ -20,8 +20,8 @@ func TestPrepareGenerationFilesKeepsClosureAndPluginSelections(t *testing.T) { }) executor := &captureExecutor{} app := New(Options{Logger: logger.NewNop(), Inputs: Inputs{InputFilesDir: []InputFilesDir{{Root: ".", Path: "."}}}, Plugins: []Plugin{ - {Source: PluginSource{Name: "custom-plugin"}, WithImports: true}, - {Source: PluginSource{Name: "custom-plugin"}}, + {Source: PluginSource{Name: "with-imports"}, WithImports: true}, + {Source: PluginSource{Name: "without-imports"}}, }}) app.localExecutor = executor plan, err := app.PrepareGenerationFiles(t.Context(), root, []string{"api/first.proto", "api/second.proto"}) @@ -30,8 +30,8 @@ func TestPrepareGenerationFilesKeepsClosureAndPluginSelections(t *testing.T) { assert.Equal(t, []string{"api/first.proto", "api/second.proto"}, descriptorNames(plan.DescriptorSet(false).File)) require.NoError(t, plan.Execute(t.Context())) require.Len(t, executor.requests, 2) - assert.ElementsMatch(t, []string{"api/first.proto", "api/second.proto", "shared.proto"}, executor.requests[0].FileToGenerate) - assert.Equal(t, []string{"api/first.proto", "api/second.proto"}, executor.requests[1].FileToGenerate) + assert.ElementsMatch(t, []string{"api/first.proto", "api/second.proto", "shared.proto"}, executor.requestsBySource["with-imports"].FileToGenerate) + assert.Equal(t, []string{"api/first.proto", "api/second.proto"}, executor.requestsBySource["without-imports"].FileToGenerate) _, err = app.PrepareGenerationFiles(t.Context(), root, []string{"absent.proto"}) require.ErrorContains(t, err, "outside the module inputs") _, err = app.PrepareGenerationFiles(t.Context(), root, nil) diff --git a/internal/generation/descriptor_set.go b/internal/generation/descriptor_set.go index 98215f03..3e2eb4c6 100644 --- a/internal/generation/descriptor_set.go +++ b/internal/generation/descriptor_set.go @@ -55,6 +55,7 @@ func generateV1DescriptorSet(ctx context.Context, log logger.Logger, cache modul } func prepareDescriptorTargets(ctx context.Context, log logger.Logger, cache modules.Cache, request Request, targets []generationTarget) ([]preparedDescriptorTarget, error) { + index := newV1WorkspaceModuleIndex(request.WorkspaceRoot) prepared := make([]preparedDescriptorTarget, 0, len(targets)) seen := make(map[string]v1ModuleSelection) requested := make(map[string]*sourceSelectorMatches) @@ -67,7 +68,7 @@ func prepareDescriptorTargets(ctx context.Context, log logger.Logger, cache modu if err := ctx.Err(); err != nil { return nil, err } - selected, err := resolveV1GenerationModule(ctx, cache, request.WorkspaceRoot, filepath.Dir(target.configPath), target.module, request.Frozen) + selected, err := resolveV1GenerationModuleWithIndex(ctx, cache, request.WorkspaceRoot, filepath.Dir(target.configPath), target.module, request.Frozen, index) if err != nil { return nil, fmt.Errorf("resolveV1GenerationModule for %s: %w", target.configPath, err) } diff --git a/internal/generation/frozen.go b/internal/generation/frozen.go index 3aeb9f88..b01bc695 100644 --- a/internal/generation/frozen.go +++ b/internal/generation/frozen.go @@ -22,6 +22,10 @@ func readFrozenGenerationModule(ctx context.Context, cache modules.Cache, direct } func resolveFrozenGenerationModule(ctx context.Context, cache modules.Cache, repoRoot, configDir string, selection v1ModuleSelection) (v1GenerationModule, error) { + return resolveFrozenGenerationModuleWithIndex(ctx, cache, repoRoot, configDir, selection, newV1WorkspaceModuleIndex(repoRoot)) +} + +func resolveFrozenGenerationModuleWithIndex(ctx context.Context, cache modules.Cache, repoRoot, configDir string, selection v1ModuleSelection, index *v1WorkspaceModuleIndex) (v1GenerationModule, error) { if selection.directory != "" { return readFrozenGenerationModule(ctx, cache, selection.directory) } @@ -60,7 +64,7 @@ func resolveFrozenGenerationModule(ctx context.Context, cache modules.Cache, rep } return v1GenerationModule{directory: directory, resolutionDir: consumerDir, module: module, sources: sources, dependencies: others}, nil } - localDir, err := findV1LocalModuleByName(repoRoot, selection.source) + localDir, err := index.find(selection.source) if err != nil { return v1GenerationModule{}, fmt.Errorf("findV1LocalModuleByName: %w", err) } diff --git a/internal/generation/generate.go b/internal/generation/generate.go index 48527414..efc8b09f 100644 --- a/internal/generation/generate.go +++ b/internal/generation/generate.go @@ -3,11 +3,13 @@ package generation import ( "bytes" "context" + "errors" "fmt" "io/fs" "os" "path/filepath" + "github.com/easyp-tech/easyp/internal/adapters/plugin" v1 "github.com/easyp-tech/easyp/internal/config/v1" "github.com/easyp-tech/easyp/internal/core/path_helpers" "github.com/easyp-tech/easyp/internal/logger" @@ -38,7 +40,16 @@ type generationTarget struct { // Run discovers generator files, prepares selected modules, and executes each generation. // Missing locked dependencies are installed in the supplied cache. -func Run(ctx context.Context, log logger.Logger, cache modules.Cache, request Request) error { +func Run(ctx context.Context, log logger.Logger, cache modules.Cache, request Request) (resultErr error) { + ctx, builtinRun := plugin.EnsureBuiltinRun(ctx) + if builtinRun != nil { + defer func() { + err := builtinRun.Close(context.WithoutCancel(ctx)) + if err != nil { + resultErr = errors.Join(resultErr, fmt.Errorf("Close: %w", err)) + } + }() + } if request.DescriptorSetOut != "" && request.DescriptorSetOutDir != "" { return fmt.Errorf("--descriptor_set_out and --descriptor_set_out_dir are mutually exclusive") } diff --git a/internal/generation/selection.go b/internal/generation/selection.go index a7e95e8a..387e7b98 100644 --- a/internal/generation/selection.go +++ b/internal/generation/selection.go @@ -1,11 +1,9 @@ package generation import ( - "bytes" "context" "errors" "fmt" - "io/fs" "os" "path/filepath" "strings" @@ -104,13 +102,17 @@ func generateSelectedV1Module(ctx context.Context, log logger.Logger, cache modu } func resolveV1GenerationModule(ctx context.Context, cache modules.Cache, repoRoot, configDir string, selection v1ModuleSelection, frozen bool) (v1GenerationModule, error) { + return resolveV1GenerationModuleWithIndex(ctx, cache, repoRoot, configDir, selection, frozen, newV1WorkspaceModuleIndex(repoRoot)) +} + +func resolveV1GenerationModuleWithIndex(ctx context.Context, cache modules.Cache, repoRoot, configDir string, selection v1ModuleSelection, frozen bool, index *v1WorkspaceModuleIndex) (v1GenerationModule, error) { if frozen { - return resolveFrozenGenerationModule(ctx, cache, repoRoot, configDir, selection) + return resolveFrozenGenerationModuleWithIndex(ctx, cache, repoRoot, configDir, selection, index) } if selection.directory != "" { return readV1GenerationModule(ctx, cache, selection.directory) } - localDir, err := findV1LocalModuleByName(repoRoot, selection.source) + localDir, err := index.find(selection.source) if err != nil { return v1GenerationModule{}, fmt.Errorf("findV1LocalModuleByName: %w", err) } @@ -207,42 +209,5 @@ func readV1GenerationModule(ctx context.Context, cache modules.Cache, directory } func findV1LocalModuleByName(repoRoot, name string) (string, error) { - var found string - err := workspace.Walk(repoRoot, func(path string, entry fs.DirEntry, walkErr error) error { - if walkErr != nil { - // Module lookup reads declarations; source aliases are validated after selection. - if filepath.Base(path) != v1.ModuleFile { - return nil - } - return walkErr - } - if entry.IsDir() { - if workspace.SkipDirectory(repoRoot, path) { - return filepath.SkipDir - } - return nil - } - if entry.Name() != v1.ModuleFile { - return nil - } - manifest, err := workspace.ReadFile(repoRoot, path) - if err != nil { - return fmt.Errorf("ReadFile: %w", err) - } - if !v1.IsModuleManifest(manifest) { - return nil - } - module, parseErr := v1.ParseModule(bytes.NewReader(manifest)) - if parseErr != nil { - return fmt.Errorf("%s: %w", path, parseErr) - } - if module.Name == name { - if found != "" { - return fmt.Errorf("module identity %q is declared in both %s and %s", name, found, filepath.Dir(path)) - } - found = filepath.Dir(path) - } - return nil - }) - return found, err + return newV1WorkspaceModuleIndex(repoRoot).find(name) } diff --git a/internal/generation/workspace_index.go b/internal/generation/workspace_index.go new file mode 100644 index 00000000..2731bfa5 --- /dev/null +++ b/internal/generation/workspace_index.go @@ -0,0 +1,84 @@ +package generation + +import ( + "bytes" + "fmt" + "io/fs" + "path/filepath" + "sync" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" + "github.com/easyp-tech/easyp/internal/workspace" +) + +// v1WorkspaceModuleIndex snapshots declarations lazily for one generation run. +// Dependency graphs and selected module metadata remain consumer-owned. +type v1WorkspaceModuleIndex struct { + root string + walk func(string, func(string, fs.DirEntry, error) error) error + readFile func(string, string) ([]byte, error) + once sync.Once + modules map[string]v1WorkspaceModuleDeclaration + err error +} + +type v1WorkspaceModuleDeclaration struct { + directory string + duplicate string +} + +func newV1WorkspaceModuleIndex(root string) *v1WorkspaceModuleIndex { + return &v1WorkspaceModuleIndex{root: root, walk: workspace.Walk, readFile: workspace.ReadFile} +} + +func (index *v1WorkspaceModuleIndex) find(name string) (string, error) { + index.once.Do(index.load) + declaration := index.modules[name] + if declaration.duplicate != "" { + return declaration.directory, fmt.Errorf("module identity %q is declared in both %s and %s", name, declaration.directory, declaration.duplicate) + } + return declaration.directory, index.err +} + +func (index *v1WorkspaceModuleIndex) load() { + index.modules = make(map[string]v1WorkspaceModuleDeclaration) + index.err = index.walk(index.root, func(path string, entry fs.DirEntry, walkErr error) error { + if walkErr != nil { + // Source aliases are validated after selection, independently of declarations. + if filepath.Base(path) != v1.ModuleFile { + return nil + } + return walkErr + } + if entry.IsDir() { + if workspace.SkipDirectory(index.root, path) { + return filepath.SkipDir + } + return nil + } + if entry.Name() != v1.ModuleFile { + return nil + } + manifest, err := index.readFile(index.root, path) + if err != nil { + return fmt.Errorf("ReadFile: %w", err) + } + if !v1.IsModuleManifest(manifest) { + return nil + } + module, err := v1.ParseModule(bytes.NewReader(manifest)) + if err != nil { + return fmt.Errorf("%s: %w", path, err) + } + declaration := index.modules[module.Name] + if declaration.directory == "" { + declaration.directory = filepath.Dir(path) + } else if declaration.duplicate == "" { + // Keep the first duplicate without failing unrelated identity lookups. + // It also precedes any later walk/read/parse error, just as in a direct walk. + declaration.duplicate = filepath.Dir(path) + } + index.modules[module.Name] = declaration + return nil + }) +} diff --git a/internal/generation/workspace_index_test.go b/internal/generation/workspace_index_test.go new file mode 100644 index 00000000..256b02da --- /dev/null +++ b/internal/generation/workspace_index_test.go @@ -0,0 +1,303 @@ +package generation + +import ( + "context" + "fmt" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" + "github.com/easyp-tech/easyp/internal/logger" + "github.com/easyp-tech/easyp/internal/modules" + "github.com/easyp-tech/easyp/internal/workspace" +) + +func TestWorkspaceModuleIndexReadsDeclarationsOnce(t *testing.T) { + t.Parallel() + root := t.TempDir() + for _, name := range []string{"a", "b"} { + writeV1GenerateFixture(t, root, name+"/"+v1.ModuleFile, "module example.test/"+name+"\n") + } + writeV1GenerateFixture(t, root, "legacy/"+v1.ModuleFile, "direct (\n)\n") + for _, directory := range []string{".hidden", "easyp_vendor", "node_modules", "nested"} { + writeV1GenerateFixture(t, root, directory+"/"+v1.ModuleFile, "module example.test/broken\nunknown directive\n") + } + writeV1GenerateFixture(t, root, "nested/.git", "gitdir: /unused\n") + require.NoError(t, os.Symlink("nested", filepath.Join(root, "nested-alias"))) + index := newV1WorkspaceModuleIndex(root) + walks := 0 + reads := make(map[string]int) + index.walk = func(boundary string, visit func(string, fs.DirEntry, error) error) error { + walks++ + return workspace.Walk(boundary, visit) + } + index.readFile = func(boundary, path string) ([]byte, error) { + reads[path]++ + return workspace.ReadFile(boundary, path) + } + assert.Zero(t, walks, "index creation must stay lazy") + for _, name := range []string{"a", "b", "a", "b"} { + directory, err := index.find("example.test/" + name) + require.NoError(t, err) + assert.Equal(t, filepath.Join(root, name), directory) + } + directory, err := index.find("example.test/missing") + require.NoError(t, err) + assert.Empty(t, directory) + assert.Equal(t, 1, walks) + assert.Equal(t, map[string]int{ + filepath.Join(root, "a", v1.ModuleFile): 1, + filepath.Join(root, "b", v1.ModuleFile): 1, + filepath.Join(root, "legacy", v1.ModuleFile): 1, + }, reads) +} + +func TestModuleSelectionKeepsUnusedIndexLazy(t *testing.T) { + t.Parallel() + root := t.TempDir() + writeV1GenerateFixture(t, root, v1.ModuleFile, "module example.test/consumer\n") + writeV1GenerateFixture(t, root, v1.LockFile, "version: 1\nmodules: []\n") + writeV1GenerateFixture(t, root, "unrelated/"+v1.ModuleFile, "module example.test/broken\nunknown directive\n") + + selected, err := resolveV1GenerationModule(t.Context(), nil, root, root, v1ModuleSelection{directory: root}, false) + require.NoError(t, err) + assert.Equal(t, root, selected.directory) + selected, err = resolveFrozenGenerationModule(t.Context(), nil, root, root, v1ModuleSelection{source: "example.test/consumer"}) + require.NoError(t, err) + assert.Equal(t, root, selected.directory) + selected, err = resolveFrozenGenerationModule(t.Context(), nil, root, root, v1ModuleSelection{directory: root}) + require.NoError(t, err) + assert.Equal(t, root, selected.directory) + + external := t.TempDir() + writeV1GenerateFixture(t, external, v1.ModuleFile, "module example.test/external\n") + writeV1GenerateFixture(t, external, v1.LockFile, "version: 1\nmodules: []\n") + for _, frozen := range []bool{false, true} { + selected, err = resolveV1GenerationModule(t.Context(), nil, root, external, v1ModuleSelection{directory: external}, frozen) + require.NoError(t, err) + assert.Equal(t, external, selected.directory) + assert.Equal(t, "example.test/external", selected.module.Name) + } +} + +func TestRunIndexesWorkspaceDeclarationsOnce(t *testing.T) { + t.Parallel() + tests := []struct { + name string + frozen bool + descriptors bool + }{ + {name: "normal generation"}, + {name: "frozen generation", frozen: true}, + {name: "normal descriptors", descriptors: true}, + {name: "frozen descriptors", frozen: true, descriptors: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + root, snapshot := t.TempDir(), t.TempDir() + const dependency = "example.test/shared" + writeV1GenerateFixture(t, root, v1.ModuleFile, "module example.test/consumer\n") + writeV1GenerateFixture(t, root, v1.LockFile, "version: 1\nmodules: []\n") + writeV1GenerateFixture(t, root, v1.GenerateFile, "version: v1\ngenerate:\n modules: [example.test/a, example.test/b, example.test/a]\n packages: [a.v1, b.v1]\n") + for _, name := range []string{"a", "b"} { + manifest := "module example.test/" + name + "\nroots proto\n" + if name == "a" { + manifest += "require " + dependency + " v1.0.0\n" + } + writeV1GenerateFixture(t, root, name+"/"+v1.ModuleFile, manifest) + writeV1GenerateFixture(t, root, name+"/"+v1.LockFile, "version: 1\nmodules: []\n") + writeV1GenerateFixture(t, root, name+"/proto/"+name+".proto", "syntax = \"proto3\"; package "+name+".v1; message Item {}\n") + } + entry := v1.LockedModule{Source: dependency, Version: "v1.0.0", Commit: strings.Repeat("a", 40), Hash: "h1:" + strings.Repeat("A", 43) + "="} + require.NoError(t, writeTestLock(filepath.Join(root, "a"), v1.Lock{Version: 1, Modules: []v1.LockedModule{entry}})) + writeV1GenerateFixture(t, snapshot, v1.ModuleFile, "module "+dependency+"\n") + writeV1GenerateFixture(t, snapshot, "shared.proto", "syntax = \"proto3\"; package shared; message Shared {}\n") + writeV1GenerateFixture(t, root, "unused/"+v1.ModuleFile, "module example.test/unused\n") + cache := &workspaceDeclarationMutationCache{ + indexedGenerationCache: &indexedGenerationCache{directory: snapshot, module: v1.Module{Name: dependency, Roots: []string{"."}}, files: []string{v1.ModuleFile, "shared.proto"}}, + manifest: filepath.Join(root, "unused", v1.ModuleFile), + } + request := Request{WorkDir: root, WorkspaceRoot: root, Frozen: tt.frozen} + if tt.descriptors { + request.DescriptorSetOut = filepath.Join(root, "all.pb") + } + + err := Run(t.Context(), logger.NewNop(), cache, request) + + require.NoError(t, err, "later selections must reuse declarations read before dependency preparation") + require.True(t, cache.changed, "the fixture must change a declaration between target lookups") + if tt.descriptors { + assert.ElementsMatch(t, []string{"a.proto", "b.proto"}, descriptorNames(readDescriptorSet(t, request.DescriptorSetOut))) + } + err = Run(t.Context(), logger.NewNop(), cache, request) + require.ErrorContains(t, err, cache.manifest, "a new command must read changed declarations") + }) + } +} + +func TestFindV1LocalModulePreservesDeclarationErrors(t *testing.T) { + t.Parallel() + tests := []struct { + name string + files map[string]string + selection string + wantDir string + wantError string + }{ + { + name: "unrelated duplicates are allowed", + files: map[string]string{ + "a/protobuf.mod": "module example.test/duplicate\n", + "b/protobuf.mod": "module example.test/duplicate\n", + "c/protobuf.mod": "module example.test/selected\n", + }, + selection: "example.test/selected", wantDir: "c", + }, + { + name: "requested duplicates precede later invalid declarations", + files: map[string]string{ + "a/protobuf.mod": "module example.test/selected\n", + "b/protobuf.mod": "module example.test/selected\n", + "z/protobuf.mod": "module example.test/broken\nunknown directive\n", + }, + selection: "example.test/selected", wantDir: "a", wantError: "declared in both", + }, + { + name: "invalid declarations precede later duplicates", + files: map[string]string{ + "a/protobuf.mod": "module example.test/selected\n", + "b/protobuf.mod": "module example.test/broken\nunknown directive\n", + "z/protobuf.mod": "module example.test/selected\n", + }, + selection: "example.test/selected", wantDir: "a", wantError: "unknown directive", + }, + { + name: "invalid unrelated native declaration fails lookup", + files: map[string]string{ + "a/protobuf.mod": "module example.test/broken\nunknown directive\n", + "b/protobuf.mod": "module example.test/selected\n", + }, + selection: "example.test/selected", wantError: "unknown directive", + }, + { + name: "legacy declarations are ignored", + files: map[string]string{ + "a/protobuf.mod": "direct (\n)\n", + "b/protobuf.mod": "module example.test/selected\n", + }, + selection: "example.test/selected", wantDir: "b", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + root := t.TempDir() + for path, contents := range tt.files { + writeV1GenerateFixture(t, root, path, contents) + } + + directory, err := findV1LocalModuleByName(root, tt.selection) + + if tt.wantError == "" { + require.NoError(t, err) + } else { + require.ErrorContains(t, err, tt.wantError) + } + wantDirectory := "" + if tt.wantDir != "" { + wantDirectory = filepath.Join(root, tt.wantDir) + } + assert.Equal(t, wantDirectory, directory) + }) + } +} + +func TestRunWorkspaceIndexKeepsConsumerLocksIndependent(t *testing.T) { + t.Parallel() + for _, frozen := range []bool{false, true} { + t.Run(fmt.Sprintf("frozen=%t", frozen), func(t *testing.T) { + t.Parallel() + root := t.TempDir() + const dependency = "example.test/shared" + cache := workspaceVersionCache{} + locks := make(map[string][]byte) + for i, project := range []string{"a", "b"} { + version := fmt.Sprintf("v1.%d.0", i) + snapshot := t.TempDir() + writeV1GenerateFixture(t, snapshot, v1.ModuleFile, "module "+dependency+"\n") + writeV1GenerateFixture(t, snapshot, "shared.proto", fmt.Sprintf("syntax = \"proto3\"; package shared; message Shared { string value%d = 1; }\n", i)) + cache[version] = snapshot + directory := filepath.Join(root, project) + writeV1GenerateFixture(t, directory, v1.ModuleFile, "module example.test/"+project+"\nrequire "+dependency+" "+version+"\n") + writeV1GenerateFixture(t, directory, v1.GenerateFile, "version: v1\ngenerate:\n modules: ["+dependency+"]\n") + entry := v1.LockedModule{Source: dependency, Version: version, Commit: strings.Repeat(fmt.Sprint(i+1), 40), Hash: "h1:" + strings.Repeat("A", 43) + "="} + require.NoError(t, writeTestLock(directory, v1.Lock{Version: 1, Modules: []v1.LockedModule{entry}})) + var err error + locks[project], err = os.ReadFile(filepath.Join(directory, v1.LockFile)) + require.NoError(t, err) + } + request := Request{WorkDir: root, WorkspaceRoot: root, AllProjects: true, Frozen: frozen, DescriptorSetOutDir: filepath.Join(root, "sets")} + + require.NoError(t, Run(t.Context(), logger.NewNop(), cache, request)) + + for i, project := range []string{"a", "b"} { + outputs, err := filepath.Glob(filepath.Join(request.DescriptorSetOutDir, project, "*.pb")) + require.NoError(t, err) + require.Len(t, outputs, 1) + descriptors := readDescriptorSet(t, outputs[0]) + require.Len(t, descriptors.File, 1) + assert.Equal(t, fmt.Sprintf("value%d", i), descriptors.File[0].MessageType[0].Field[0].GetName()) + lock, err := os.ReadFile(filepath.Join(root, project, v1.LockFile)) + require.NoError(t, err) + assert.Equal(t, locks[project], lock) + } + }) + } +} + +type workspaceVersionCache map[string]string + +func (workspaceVersionCache) Install(context.Context, v1.Lock) error { return nil } + +func (cache workspaceVersionCache) Cached(entry v1.LockedModule) (string, v1.Module, error) { + directory, exists := cache[entry.Version] + if !exists { + return "", v1.Module{}, fmt.Errorf("unknown cached version %s", entry.Version) + } + _, module, err := modules.ReadManifest(directory) + if err != nil { + return "", v1.Module{}, fmt.Errorf("ReadManifest: %w", err) + } + if module.Name != entry.Source { + return "", v1.Module{}, fmt.Errorf("unknown cached module %s", entry.Source) + } + return directory, module, nil +} + +type workspaceDeclarationMutationCache struct { + *indexedGenerationCache + manifest string + changed bool +} + +func (cache *workspaceDeclarationMutationCache) Cached(entry v1.LockedModule) (string, v1.Module, error) { + directory, module, err := cache.indexedGenerationCache.Cached(entry) + if err != nil { + return "", v1.Module{}, fmt.Errorf("Cached: %w", err) + } + if !cache.changed { + err = os.WriteFile(cache.manifest, []byte("module example.test/unused\nunknown directive\n"), 0o600) + if err != nil { + return "", v1.Module{}, fmt.Errorf("WriteFile: %w", err) + } + cache.changed = true + } + return directory, module, nil +} diff --git a/internal/modules/fetch_preparation.go b/internal/modules/fetch_preparation.go new file mode 100644 index 00000000..0f4d182b --- /dev/null +++ b/internal/modules/fetch_preparation.go @@ -0,0 +1,129 @@ +package modules + +import ( + "context" + "fmt" + "reflect" + "strings" + + "golang.org/x/mod/semver" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" + "github.com/easyp-tech/easyp/internal/parallel" +) + +// FetchRequest describes independent revision and source-scope preparation. +// Roots and Inspect preserve the root-selection source contracts when a wrapper +// prepares a request on behalf of the resolver. +type FetchRequest struct { + Source string + Version string + Roots []string + Inspect bool +} + +// PreparedFetch completes prepared work in the resolver's original request +// order. Preparation must release temporary resources before returning; a +// provisional graph may discard the callback without consuming it. +type PreparedFetch func(context.Context) (Fetched, error) + +// fetchPreparer is optional: ordinary Source implementations remain serial. +// PrepareFetch may run concurrently for independent modules. Its callback must +// defer mutable resolver, metadata-backend, and import-root state until consumed. +type fetchPreparer interface { + PreparationSource() Source + PrepareFetch(context.Context, FetchRequest) (PreparedFetch, error) +} + +const moduleFetchLimit = 8 + +type preparedRevision struct { + complete PreparedFetch + err error +} + +func sourceFetchPreparer(source Source) (fetchPreparer, bool) { + preparer, supported := source.(fetchPreparer) + if !supported || !reflect.ValueOf(source).Comparable() { + return nil, false + } + // Promoted methods on an embedding wrapper do not advertise safety for + // that wrapper's overridden Fetch/root methods. It must opt in itself. + if preparer.PreparationSource() != source { + return nil, false + } + return preparer, true +} + +func (loader *revisionLoader) prepareFetches(ctx context.Context, queue []v1.Requirement, requirements map[string]*versionRequirements, visited map[string]bool) error { + preparer, supported := sourceFetchPreparer(loader.source) + // Local replacement lookup can itself discover stateful graph edges. Keep + // its requests serial until that lookup has selected a remote revision. + if !supported || loader.local != nil { + return nil + } + var requests []FetchRequest + seen := make(map[string]bool) + for _, requirement := range queue { + version := requirement.Version + // Versionless nodes must wait until stronger requirements are known. + if version == "" || v1.ValidateModuleVersion(requirement.Module, version) != nil { + continue + } + if v1.IsCommitRef(version) { + version = strings.ToLower(version) + if previous := requirements[requirement.Module]; previous != nil && previous.commit != "" && previous.commit != version { + continue + } + } else if !semver.IsValid(version) { + continue + } + // One revision per module preserves tag-to-commit reuse and avoids + // fetching a later conflicting exact requirement speculatively. + if seen[requirement.Module] { + continue + } + seen[requirement.Module] = true + key := requirement.Module + "@" + version + if visited[key] || loader.failures[key] != nil { + continue + } + if _, exists := loader.prepared[key]; exists { + continue + } + if _, exists := loader.fetched[key]; exists { + continue + } + if v1.IsCommitRef(version) { + known := false + for _, fetched := range loader.fetched { + if fetched.Lock.Source == requirement.Module && strings.EqualFold(fetched.Lock.Commit, version) { + known = true + break + } + } + if known { + continue + } + } + requests = append(requests, FetchRequest{Source: requirement.Module, Version: version}) + } + results := make([]preparedRevision, len(requests)) + err := parallel.Run(ctx, len(requests), moduleFetchLimit, func(index int) error { + complete, err := preparer.PrepareFetch(ctx, requests[index]) + results[index] = preparedRevision{complete: complete, err: err} + // A provisional graph can discard this revision. Report its error only + // when the serial solver actually consumes the corresponding request. + return nil + }) + if err != nil { + return fmt.Errorf("Run: %w", err) + } + if len(requests) > 0 && loader.prepared == nil { + loader.prepared = make(map[string]preparedRevision) + } + for index, request := range requests { + loader.prepared[request.Source+"@"+request.Version] = results[index] + } + return nil +} diff --git a/internal/modules/fetch_preparation_test.go b/internal/modules/fetch_preparation_test.go new file mode 100644 index 00000000..10943d00 --- /dev/null +++ b/internal/modules/fetch_preparation_test.go @@ -0,0 +1,246 @@ +package modules + +import ( + "context" + "errors" + "strings" + "sync/atomic" + "testing" + "testing/synctest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" +) + +func TestResolvePreparesBoundedRevisionsAndConsumesInOrder(t *testing.T) { + t.Parallel() + synctest.Test(t, func(t *testing.T) { + const count = 12 + started := make(chan FetchRequest, count) + release := make(chan struct{}) + backend := &preparedTestSource{started: started, release: release} + root := v1.Module{} + var want []string + for index := range count { + name := "example.test/" + string(rune('a'+index)) + root.Requires = append(root.Requires, v1.Requirement{Module: name, Version: "v1.0.0"}) + want = append(want, name+"@v1.0.0") + } + root.Requires = append(root.Requires, root.Requires[0], v1.Requirement{Module: root.Requires[0].Module, Version: versionlessCommitA}) + done := make(chan error, 1) + var lock v1.Lock + go func() { + var err error + lock, err = Resolve(t.Context(), root, backend, nil) + done <- err + }() + synctest.Wait() + assert.Len(t, started, 8, "known independent revisions should fill the fetch worker pool") + close(release) + require.NoError(t, <-done) + assert.EqualValues(t, count, backend.preparations.Load()) + assert.Zero(t, backend.fetches.Load(), "prepared results must be consumed without refetching") + assert.Equal(t, want, backend.consumed) + require.Len(t, lock.Modules, count) + assert.Equal(t, "v1.0.0", lock.Modules[0].Version) + assert.Equal(t, versionlessCommitA, lock.Modules[0].Commit) + }) +} + +func TestResolvePreparationCancellationJoinsWorkers(t *testing.T) { + t.Parallel() + synctest.Test(t, func(t *testing.T) { + started := make(chan FetchRequest, 12) + release := make(chan struct{}) + backend := &preparedTestSource{started: started, release: release} + root := v1.Module{} + for index := range 12 { + root.Requires = append(root.Requires, v1.Requirement{Module: "example.test/" + string(rune('a'+index)), Version: "v1.0.0"}) + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + done := make(chan error, 1) + go func() { + _, err := Resolve(ctx, root, backend, nil) + done <- err + }() + synctest.Wait() + assert.Len(t, started, 8) + cancel() + synctest.Wait() + select { + case <-done: + t.Error("Resolve returned while preparation workers were active") + default: + } + close(release) + require.ErrorIs(t, <-done, context.Canceled) + assert.EqualValues(t, 8, backend.preparations.Load()) + assert.Empty(t, backend.consumed) + }) +} + +func TestResolvePreparedErrorsFollowRequestOrder(t *testing.T) { + t.Parallel() + first := errors.New("first module unavailable") + second := errors.New("second module unavailable") + secondPrepared := make(chan struct{}) + backend := &preparedTestSource{prepare: func(request FetchRequest) (PreparedFetch, error) { + if request.Source == "a" { + <-secondPrepared + return nil, first + } + close(secondPrepared) + return nil, second + }} + _, err := Resolve(t.Context(), v1.Module{Requires: []v1.Requirement{{Module: "a", Version: "v1.0.0"}, {Module: "b", Version: "v1.0.0"}}}, backend, nil) + require.ErrorIs(t, err, first) +} + +func TestResolveDoesNotPrepareThroughUnadvertisedSourceWrapper(t *testing.T) { + t.Parallel() + backend := &preparedTestSource{} + source := &overriddenPreparedSource{preparedTestSource: backend} + _, err := Resolve(t.Context(), v1.Module{Requires: []v1.Requirement{{Module: "a", Version: "v1.0.0"}}}, source, nil) + require.NoError(t, err) + assert.Equal(t, 1, source.calls) + assert.Zero(t, backend.preparations.Load(), "embedding a source must not bypass overridden Fetch methods") +} + +func TestResolveUncomparablePreparationSourceRemainsSerial(t *testing.T) { + t.Parallel() + source := uncomparablePreparedSource{values: []string{"a"}} + _, err := Resolve(t.Context(), v1.Module{Requires: []v1.Requirement{{Module: "a", Version: "v1.0.0"}}}, source, nil) + require.NoError(t, err) +} + +type uncomparablePreparedSource struct{ values []string } + +func (source uncomparablePreparedSource) PreparationSource() Source { return source } + +func (uncomparablePreparedSource) PrepareFetch(context.Context, FetchRequest) (PreparedFetch, error) { + return nil, errors.New("uncomparable source cannot advertise receiver identity") +} + +func (uncomparablePreparedSource) Fetch(ctx context.Context, name, version string) (Fetched, error) { + return (&preparedTestSource{}).Fetch(ctx, name, version) +} + +type overriddenPreparedSource struct { + *preparedTestSource + calls int +} + +func (source *overriddenPreparedSource) Fetch(ctx context.Context, name, version string) (Fetched, error) { + source.calls++ + return source.preparedTestSource.Fetch(ctx, name, version) +} + +func TestImportRootPreparationDefersRootBookkeeping(t *testing.T) { + t.Parallel() + synctest.Test(t, func(t *testing.T) { + started := make(chan FetchRequest, 2) + release := make(chan struct{}) + backend := &preparedRootTestSource{preparedTestSource: preparedTestSource{started: started, release: release}} + source := newImportRootSource(backend, v1.Lock{}, map[string][]string{"a": {"api"}}) + done := make(chan error, 1) + go func() { + _, err := Resolve(t.Context(), v1.Module{Requires: []v1.Requirement{{Module: "a", Version: "v1.0.0"}, {Module: "b", Version: "v1.0.0"}}}, source, nil) + done <- err + }() + synctest.Wait() + assert.Len(t, started, 2) + assert.Empty(t, source.fetched, "preparation must not mutate the root-resolution maps") + close(release) + require.NoError(t, <-done) + assert.Equal(t, []string{"a@v1.0.0", "b@v1.0.0"}, backend.consumed) + assert.Len(t, source.fetched, 2) + if len(started) == 2 { + requests := []FetchRequest{<-started, <-started} + for _, request := range requests { + assert.True(t, request.Inspect) + if request.Source == "a" { + assert.Equal(t, []string{"api"}, request.Roots) + } + } + } + }) +} + +func TestImportRootPreparationChecksTagBeforeRetainedScope(t *testing.T) { + t.Parallel() + old := v1.LockedModule{Source: "a", Version: "v1.0.0", Commit: strings.Repeat("b", 40), Hash: versionlessHash, Roots: []string{"api"}} + backend := &preparedRootTestSource{} + source := newImportRootSource(backend, v1.Lock{Modules: []v1.LockedModule{old}}, nil) + preparer, supported := any(source).(fetchPreparer) + require.True(t, supported, "root wrapper must advertise independent preparation") + complete, err := preparer.PrepareFetch(t.Context(), FetchRequest{Source: "a", Version: "v1.0.0"}) + require.NoError(t, err) + assert.Empty(t, source.fetched) + _, err = complete(t.Context()) + require.ErrorIs(t, err, ErrLockedVersionChanged) + assert.Zero(t, backend.selections) + assert.Empty(t, source.fetched) +} + +type preparedRootTestSource struct { + preparedTestSource + selections int +} + +func (source *preparedRootTestSource) PreparationSource() Source { return source } + +func (source *preparedRootTestSource) FetchForRootResolution(ctx context.Context, name, version string) (Fetched, error) { + return source.complete(ctx, FetchRequest{Source: name, Version: version, Inspect: true}) +} + +func (source *preparedRootTestSource) FetchWithRoots(ctx context.Context, name, version string, roots []string) (Fetched, error) { + source.selections++ + return source.complete(ctx, FetchRequest{Source: name, Version: version, Roots: roots, Inspect: true}) +} + +type preparedTestSource struct { + started chan<- FetchRequest + release <-chan struct{} + prepare func(FetchRequest) (PreparedFetch, error) + preparations atomic.Int64 + fetches atomic.Int64 + consumed []string +} + +func (source *preparedTestSource) PreparationSource() Source { return source } + +func (source *preparedTestSource) Fetch(ctx context.Context, name, version string) (Fetched, error) { + source.fetches.Add(1) + return source.complete(ctx, FetchRequest{Source: name, Version: version}) +} + +func (source *preparedTestSource) PrepareFetch(_ context.Context, request FetchRequest) (PreparedFetch, error) { + source.preparations.Add(1) + if source.prepare != nil { + return source.prepare(request) + } + if source.started != nil { + source.started <- request + <-source.release + } + return func(ctx context.Context) (Fetched, error) { return source.complete(ctx, request) }, nil +} + +func (source *preparedTestSource) complete(_ context.Context, request FetchRequest) (Fetched, error) { + source.consumed = append(source.consumed, request.Source+"@"+request.Version) + fetched := Fetched{ + Module: v1.Module{Name: request.Source, Roots: []string{"."}}, + Lock: v1.LockedModule{Source: request.Source, Version: request.Version, Commit: versionlessCommitA, Hash: "h1:" + strings.Repeat("A", 43) + "="}, + } + if len(request.Roots) > 0 { + fetched.Module.Roots = request.Roots + fetched.Lock.Roots = request.Roots + } + if request.Inspect { + fetched.Inspection = &RootInspection{} + } + return fetched, nil +} diff --git a/internal/modules/import_roots_preparation.go b/internal/modules/import_roots_preparation.go new file mode 100644 index 00000000..89916dcd --- /dev/null +++ b/internal/modules/import_roots_preparation.go @@ -0,0 +1,51 @@ +package modules + +import ( + "context" + "fmt" + "slices" +) + +// PreparationSource explicitly opts this wrapper into independent preparation. +func (source *importRootSource) PreparationSource() Source { return source } + +// PrepareFetch prepares only independent backend work. Root integrity checks, +// retained scope lookups, and namespace bookkeeping run when consumed serially. +func (source *importRootSource) PrepareFetch(ctx context.Context, request FetchRequest) (PreparedFetch, error) { + backend, supported := sourceFetchPreparer(source.Source) + if !supported { + return func(ctx context.Context) (Fetched, error) { + return source.Fetch(ctx, request.Source, request.Version) + }, nil + } + hint := source.hints[request.Source] + inspector, inspectionSupported := source.Source.(rootInspectionSource) + if len(hint) > 0 { + if _, supported := source.Source.(rootSelectionSource); !supported { + return nil, fmt.Errorf("module %s: repository cannot verify explicit import roots", request.Source) + } + } + request.Roots = slices.Clone(hint) + request.Inspect = inspectionSupported + prepared, err := backend.PrepareFetch(ctx, request) + if err != nil { + return nil, fmt.Errorf("Fetch: %w", err) + } + return func(ctx context.Context) (Fetched, error) { + fetched, err := prepared(ctx) + if err != nil { + return Fetched{}, fmt.Errorf("Fetch: %w", err) + } + if len(hint) > 0 && inspectionSupported && fetched.Inspection == nil { + inspected, err := inspector.FetchForRootResolution(ctx, request.Source, fetched.Lock.Commit) + if err != nil { + return Fetched{}, fmt.Errorf("Fetch: %w", err) + } + fetched.Inspection = cloneRootInspection(inspected.Inspection) + if fetched.Inspection != nil { + fetched.Inspection.Provisional = false + } + } + return source.acceptRootFetch(ctx, request.Source, request.Version, fetched) + }, nil +} diff --git a/internal/modules/import_roots_resolution.go b/internal/modules/import_roots_resolution.go index d3b83512..63ec7441 100644 --- a/internal/modules/import_roots_resolution.go +++ b/internal/modules/import_roots_resolution.go @@ -76,6 +76,11 @@ func (source *importRootSource) Fetch(ctx context.Context, name, version string) if err != nil { return Fetched{}, fmt.Errorf("Fetch: %w", err) } + return source.acceptRootFetch(ctx, name, version, fetched) +} + +func (source *importRootSource) acceptRootFetch(ctx context.Context, name, version string, fetched Fetched) (Fetched, error) { + hint := source.hints[name] fetched = cloneRootFetched(fetched) if v1.IsCommitRef(version) && !strings.EqualFold(version, fetched.Lock.Commit) { return Fetched{}, fmt.Errorf("module %s: fetched commit %s differs from requested commit %s", name, fetched.Lock.Commit, version) diff --git a/internal/modules/resolve.go b/internal/modules/resolve.go index 9a595973..f1cc7348 100644 --- a/internal/modules/resolve.go +++ b/internal/modules/resolve.go @@ -34,6 +34,7 @@ type revisionLoader struct { source Source fetched map[string]Fetched failures map[string]error + prepared map[string]preparedRevision pins map[string]v1.LockedModule local func(string, string) (v1.Module, bool, error) } @@ -59,7 +60,17 @@ func (l *revisionLoader) fetch(ctx context.Context, source, version string) (Fet } } } - result, err := l.source.Fetch(ctx, source, version) + var result Fetched + var err error + if prepared, exists := l.prepared[key]; exists { + delete(l.prepared, key) + err = prepared.err + if err == nil { + result, err = prepared.complete(ctx) + } + } else { + result, err = l.source.Fetch(ctx, source, version) + } if err != nil { if l.failures == nil { l.failures = make(map[string]error) @@ -158,73 +169,81 @@ func (l *revisionLoader) resolvePass(ctx context.Context, root v1.Module, hints queue := slices.Clone(root.Requires) for { for len(queue) > 0 { - if err := ctx.Err(); err != nil { + if err := l.prepareFetches(ctx, queue, requirements, visited); err != nil { pass.deferError(err) return pass } - requirement := queue[0] - queue = queue[1:] - if err := v1.ValidateModuleVersion(requirement.Module, requirement.Version); err != nil { - pass.deferError(fmt.Errorf("ValidateModuleVersion: %w", err)) - continue - } - if l.local != nil { - // The main module and replaced nodes have no remote revision. - if requirement.Module == root.Name { - continue - } - module, local, err := l.local(requirement.Module, requirement.Version) - if err != nil { + // Consume this frontier in its original FIFO order. New dependency + // edges form the next preparation frontier, without changing MVS. + for range len(queue) { + if err := ctx.Err(); err != nil { pass.deferError(err) + return pass + } + requirement := queue[0] + queue = queue[1:] + if err := v1.ValidateModuleVersion(requirement.Module, requirement.Version); err != nil { + pass.deferError(fmt.Errorf("ValidateModuleVersion: %w", err)) continue } - if local { - if localVisited[requirement.Module] { + if l.local != nil { + // The main module and replaced nodes have no remote revision. + if requirement.Module == root.Name { continue } - localVisited[requirement.Module] = true - pass.locals = append(pass.locals, requirement.Module) - queue = append(queue, module.Requires...) + module, local, err := l.local(requirement.Module, requirement.Version) + if err != nil { + pass.deferError(err) + continue + } + if local { + if localVisited[requirement.Module] { + continue + } + localVisited[requirement.Module] = true + pass.locals = append(pass.locals, requirement.Module) + queue = append(queue, module.Requires...) + continue + } + } + version := requirement.Version + if version != "" && !v1.IsCommitRef(version) && !semver.IsValid(version) { + pass.deferError(fmt.Errorf("require %s: expected semantic version or full Git commit, got %q", requirement.Module, version)) continue } - } - version := requirement.Version - if version != "" && !v1.IsCommitRef(version) && !semver.IsValid(version) { - pass.deferError(fmt.Errorf("require %s: expected semantic version or full Git commit, got %q", requirement.Module, version)) - continue - } - r := requirements[requirement.Module] - if r == nil { - r = &versionRequirements{} - requirements[requirement.Module] = r - } - switch { - case version == "": - r.weak = true - continue - case v1.IsCommitRef(version): - version = strings.ToLower(version) - if r.commit != "" && r.commit != version { - pass.deferError(fmt.Errorf("module %s has conflicting requirements %s and %s", requirement.Module, r.commit, version)) + r := requirements[requirement.Module] + if r == nil { + r = &versionRequirements{} + requirements[requirement.Module] = r + } + switch { + case version == "": + r.weak = true continue + case v1.IsCommitRef(version): + version = strings.ToLower(version) + if r.commit != "" && r.commit != version { + pass.deferError(fmt.Errorf("module %s has conflicting requirements %s and %s", requirement.Module, r.commit, version)) + continue + } + r.commit = version + default: + if r.minimum == "" || semver.Compare(version, r.minimum) > 0 || (semver.Compare(version, r.minimum) == 0 && version > r.minimum) { + r.minimum = version + } } - r.commit = version - default: - if r.minimum == "" || semver.Compare(version, r.minimum) > 0 || (semver.Compare(version, r.minimum) == 0 && version > r.minimum) { - r.minimum = version + key := requirement.Module + "@" + version + if visited[key] { + continue } + visited[key] = true + result, err := l.fetch(ctx, requirement.Module, version) + if err != nil { + pass.deferError(err) + continue + } + queue = append(queue, result.Module.Requires...) } - key := requirement.Module + "@" + version - if visited[key] { - continue - } - visited[key] = true - result, err := l.fetch(ctx, requirement.Module, version) - if err != nil { - pass.deferError(err) - continue - } - queue = append(queue, result.Module.Requires...) } pending := make([]string, 0) for name, r := range requirements { diff --git a/internal/parallel/parallel.go b/internal/parallel/parallel.go new file mode 100644 index 00000000..5a4408c4 --- /dev/null +++ b/internal/parallel/parallel.go @@ -0,0 +1,59 @@ +// Package parallel runs bounded independent tasks with deterministic error order. +package parallel + +import ( + "context" + "runtime" + "sync" + "sync/atomic" +) + +// Run runs count tasks with at most limit workers and waits for all of them. +// Each task owns its index, so callers can write independent indexed results +// without locking. Errors are returned in index order, not completion order. +// Cancellation stops new tasks and takes precedence after active tasks finish. +// A non-positive limit uses the number of available CPUs. +func Run(ctx context.Context, count, limit int, task func(index int) error) error { + if err := ctx.Err(); err != nil { + return err + } + if count <= 0 { + return nil + } + if limit <= 0 { + limit = runtime.NumCPU() + } + workers := min(count, limit) + errors := make([]error, count) + var next atomic.Int64 + var group sync.WaitGroup + group.Add(workers) + for range workers { + go func() { + defer group.Done() + for { + if ctx.Err() != nil { + return + } + index := int(next.Add(1) - 1) + if index >= count { + return + } + if ctx.Err() != nil { + return + } + errors[index] = task(index) + } + }() + } + group.Wait() + if err := ctx.Err(); err != nil { + return err + } + for _, err := range errors { + if err != nil { + return err + } + } + return nil +} diff --git a/internal/parallel/parallel_test.go b/internal/parallel/parallel_test.go new file mode 100644 index 00000000..e7a63af4 --- /dev/null +++ b/internal/parallel/parallel_test.go @@ -0,0 +1,112 @@ +package parallel + +import ( + "context" + "errors" + "sync/atomic" + "testing" + "testing/synctest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRunBoundsWorkersAndExecutesEachTaskOnce(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + name string + limit int + }{ + {name: "serial", limit: 1}, + {name: "two workers", limit: 2}, + {name: "more workers than tasks", limit: 20}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + synctest.Test(t, func(t *testing.T) { + const count = 6 + started := make(chan int, count) + release := make(chan struct{}) + done := make(chan error, 1) + var calls [count]atomic.Int64 + go func() { + done <- Run(t.Context(), count, tt.limit, func(index int) error { + calls[index].Add(1) + started <- index + <-release + return nil + }) + }() + synctest.Wait() + assert.Len(t, started, min(count, tt.limit)) + close(release) + require.NoError(t, <-done) + for index := range calls { + assert.EqualValues(t, 1, calls[index].Load(), "task %d", index) + } + }) + }) + } +} + +func TestRunReturnsErrorInTaskOrder(t *testing.T) { + t.Parallel() + first := errors.New("first task") + second := errors.New("second task") + completedSecond := make(chan struct{}) + err := Run(t.Context(), 3, 2, func(index int) error { + switch index { + case 0: + <-completedSecond + return first + case 1: + close(completedSecond) + return second + default: + return nil + } + }) + require.ErrorIs(t, err, first) +} + +func TestRunCancellationStopsSchedulingAndJoinsWorkers(t *testing.T) { + t.Parallel() + synctest.Test(t, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + release := make(chan struct{}) + done := make(chan error, 1) + var calls atomic.Int64 + go func() { + done <- Run(ctx, 10, 2, func(int) error { + calls.Add(1) + <-release + return nil + }) + }() + synctest.Wait() + require.EqualValues(t, 2, calls.Load()) + cancel() + synctest.Wait() + select { + case <-done: + t.Fatal("Run returned while its tasks were still active") + default: + } + close(release) + require.ErrorIs(t, <-done, context.Canceled) + assert.EqualValues(t, 2, calls.Load()) + }) +} + +func TestRunCanceledAndEmpty(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + err := Run(ctx, 3, 2, func(int) error { + t.Error("canceled run started a task") + return nil + }) + require.ErrorIs(t, err, context.Canceled) + require.NoError(t, Run(t.Context(), 0, 0, nil)) +}