diff --git a/.spec/ARCHITECTURE.md b/.spec/ARCHITECTURE.md index e23d5e39..afe87faf 100644 --- a/.spec/ARCHITECTURE.md +++ b/.spec/ARCHITECTURE.md @@ -87,11 +87,13 @@ mod vendor EnsureLockedSources may install dependencies. ReadManifest, ReadLock, LocalSources, and CachedSources do not download or rewrite project files. CachedSources assumes installation/hash verification has already occurred. +Verified installed modules can supply an immutable regular-file inventory through the optional SourceFiles cache capability. SourceRoots uses it for dependency discovery and nested boundaries without repeated filesystem traversal. Local roots and replacements continue to resolve current filesystem state. Generation computes collision checks and file ownership together and shares the result with managed options and descriptor output. + Generation, policy commands, and module operations share root/collision checks. WalkProtoFiles owns dependency-source traversal, including exclusion of hidden directories, vendored sources and nested module boundaries. Generator discovery and policy ancestor traversal remain separate: their inclusion and inheritance rules differ. Nested protobuf module roots are rebased by module_config from the nested manifest directory into the checkout directory. Import names remain relative to those roots; repository/module prefixes do not become part of an import. -The Git cache lives beneath the CLI-supplied EasyP directory in v1/git; internal/adapters/gitmodules/object_cache.go maintains reusable bare object repositories with OS locks. Cache.Cached reads installed metadata; Install supplies content verification before cached metadata is trusted. +The Git cache lives beneath the CLI-supplied EasyP directory in v1/git; internal/adapters/gitmodules/object_cache.go maintains reusable bare object repositories with OS locks. Cache.Cached reads installed metadata; Install supplies content verification before cached metadata is trusted. Successful verification and the inventory are reused within one cache handle, keyed by source, commit, content hash and selected roots. A new handle rechecks the snapshot; explicit VerifyCached bypasses the in-memory result. Revision resolution atomically retains completed snapshots so installation does not materialize them a second time. Local replacements use an ephemeral main-module overlay graph and never alter the shared lock. Tidy validates the overlay; get/update may edit explicit manifest requirements. Vendor copies the effective local graph without touching the lock. Explicit frozen operation verifies existing native manifests and locked graphs; automatic unknown-import-to-module discovery is intentionally not implemented. diff --git a/.spec/config/dependency.md b/.spec/config/dependency.md index 24205f26..977cfbca 100644 --- a/.spec/config/dependency.md +++ b/.spec/config/dependency.md @@ -60,7 +60,13 @@ See [Go major version suffixes](https://go.dev/ref/mod#major-version-suffixes) a The CLI resolves EASYPPATH once for a command that needs the cache (default $HOME/.easyp). gitmodules.Cache owns <EASYPPATH>/v1/git, source keys, temporary checkout names and installation paths. Installed snapshots live under its modules/<source-key>/<snapshot-key> layout, where the snapshot key hashes commit plus content hash; reusable bare object stores live under objects/<remote-key>, with OS locks for concurrent fetches. Source/commit bindings under sources/ identify these stores for read-only policy access. Application callers request cached module metadata and its physical directory; they do not assemble cache paths. -Cache.Install verifies locked contents, including cache hits. A newly installed snapshot is checked with the full h1: directory hash. On Darwin/Linux, later unchanged hits use a sidecar verification stamp keyed by the expected lock hash plus an inode/ctime metadata fingerprint; any metadata change invalidates the stamp and forces the full directory hash again. Content changes therefore remain detectable even when size and mtime are restored. Platforms without a strong change token keep the full-hash path. Stamp hits first check the current snapshot selection policy. Read-only validation never writes verification stamps. The stamp is an optimization of the per-user local cache, not an additional source of dependency identity. Cached objects support shallow pinned-commit fetches with an advertised-history fallback; neither path substitutes HEAD for an unavailable pin. Cache.Cached reads installed metadata without downloading or rewriting module contents and requires prior verification. Git credentials and transport configuration remain the responsibility of system Git. Ordinary native source use does not require objects for unselected policies. Explicit mod download repairs missing policy object sources, preserving every reached pin in a shared repository; validation reports missing sources without fetching or writing. +Cache.Install verifies a snapshot on its first use by a cache handle, then reuses that successful verification and its file inventory within the command. The key includes source, commit, content hash and root selection. Installed snapshots are immutable for that lifetime; local replacements are never admitted to this cache. A new command creates a new handle and checks installed contents again. Cache.VerifyCached explicitly rechecks contents even on the same handle, without downloading or writing verification stamps; a failed check invalidates the remembered result. + +A newly acquired snapshot is checked with the full h1: directory hash. On Darwin/Linux, an unchanged snapshot from a previous command can use a sidecar stamp keyed by the expected lock hash plus an inode/ctime metadata fingerprint; any metadata change forces the full directory hash again. Content changes remain detectable on the next command or explicit verification even when size and mtime are restored. Platforms without a strong change token keep the full-hash path on first use. Stamp hits first check the current snapshot selection policy. The stamp is an optimization of the per-user local cache, not an additional source of dependency identity. A completed, hashed snapshot produced during revision resolution is installed atomically and reused by the following install step; provisional root inspections cannot publish an incomplete snapshot. + +The optional Cache.SourceFiles capability exposes a copy of the verified snapshot's regular-file inventory, including configuration markers needed for nested module boundaries. Dependency source discovery, global import-collision checks and module ownership use this inventory instead of walking the installed filesystem for each target. Generation shares one collision/ownership index across managed options and descriptor attribution. This does not limit collision detection to files reached by compilation. + +Cached objects support shallow pinned-commit fetches with an advertised-history fallback; neither path substitutes HEAD for an unavailable pin. Cache.Cached reads installed metadata without downloading or rewriting module contents and requires prior verification. Git credentials and transport configuration remain the responsibility of system Git. Ordinary native source use does not require objects for unselected policies. Explicit mod download repairs missing policy object sources, preserving every reached pin in a shared repository; validation reports missing sources without fetching or writing. EASYP_PROXY, when set to a host or host:port, replaces the host of Git remotes used to fetch dependencies. A module declared as github.com/group/package is cloned from https://gitlab.com/group/package when the variable is gitlab.com. Explicit https and ssh URLs keep their path and user; only the host changes, including a port supplied by the variable. Absolute local paths are unchanged. The published module identity, lock source, and content hash stay the original module path. An empty value disables substitution. config.Config.LoadEnvironment reads the variable into Config.Proxy; it is not an easyp.yaml field. @@ -81,8 +87,10 @@ phases. Start/finish records include status and elapsed time; inventories includ file counts and selected roots/filters. Slow phases report elapsed progress every 15 seconds at info level, including without --debug. These logs do not change source selection or dependency identity. Indexed tree and blob -metadata are reused only within one immutable Git filesystem; mutable local -source resolution remains uncached. +metadata are reused within one immutable Git filesystem. Verified installed +snapshot inventories are reused within one command. Mutable local source +resolution remains uncached between calls. A walk resolves its local boundary +once; reusing descendant metadata is restricted to explicitly immutable views. Deleting the old easyp.lock is not required for migration. Its absence does not change an explicit git_repo.url: repo@<commit> pin. @@ -163,7 +171,7 @@ Explicit --frozen requires protobuf.mod and prot All root replacement directives are forbidden, even when unused. Validation rejects them before opening replacement directories or accessing dependency caches/remotes. Dependency manifests still supply transitive requirements; dependency-local replacements remain ignored. The locked closure must satisfy direct and transitive requirements with no missing or stale entries. Malformed locks, incompatible versions/commits and unexpected graph entries fail instead of triggering resolution. -The frozen path does not call the version resolver, query HEAD or tags, or create/rewrite manifests and locks. Exact locked commits may be downloaded when absent from cache; installed sources are lock-hash verified on cold paths, while unchanged Darwin/Linux warm hits reuse the verified metadata stamp and fall back to a full h1: check after any detected filesystem change. This permits reproducible dependency selection with a cold cache and does not imply offline execution. Existing Git credentials, transport configuration, and plugin network requirements still apply. +The frozen path does not call the version resolver, query HEAD or tags, or create/rewrite manifests and locks. Exact locked commits may be downloaded when absent from cache. First use by the command verifies the snapshot against the lock, using the Darwin/Linux metadata stamp where available and falling back to a full h1: check after a detected filesystem change. Later targets reuse that verified immutable snapshot within the same command. This permits reproducible dependency selection with a cold cache and does not imply offline execution. Existing Git credentials, transport configuration, and plugin network requirements still apply. - mod download validates and installs the locked graph; mod vendor validates it before replacing vendor output. - mod tidy, get, mod update, and init refuse frozen operation. migrate, including preview and interactive mode, is also rejected. diff --git a/internal/adapters/gitmodules/bsr_test.go b/internal/adapters/gitmodules/bsr_test.go index c2d66b0f..b043aef3 100644 --- a/internal/adapters/gitmodules/bsr_test.go +++ b/internal/adapters/gitmodules/bsr_test.go @@ -56,7 +56,11 @@ func TestFetchBSRDependencies(t *testing.T) { entries, readErr := os.ReadDir(cache.root) require.NoError(t, readErr) for _, entry := range entries { - assert.Equal(t, "objects", entry.Name(), "temporary checkout survived Fetch") + allowed := []string{"objects"} + if err == nil { + allowed = append(allowed, "modules", "sources") + } + assert.Contains(t, allowed, entry.Name(), "temporary checkout survived Fetch") } if tt.wantErrIs != nil { require.ErrorIs(t, err, tt.wantErrIs) diff --git a/internal/adapters/gitmodules/cache.go b/internal/adapters/gitmodules/cache.go index 9f1ffec7..41106356 100644 --- a/internal/adapters/gitmodules/cache.go +++ b/internal/adapters/gitmodules/cache.go @@ -6,6 +6,7 @@ import ( "fmt" "log/slog" "path/filepath" + "sync" "github.com/easyp-tech/easyp/internal/adapters/bsr" "github.com/easyp-tech/easyp/internal/adapters/gitcommand" @@ -21,14 +22,16 @@ type Cache struct { bsrResolver modules.BSRResolver logger logger.Logger proxy string + stateMu sync.Mutex + state *cacheRunState } // WithLogger returns an isolated cache handle with dependency progress logging. -// Historical-pin scopes preserve this logger through their existing shallow copy. +// Logger and historical-pin scopes share verified immutable snapshot state. func (c *Cache) WithLogger(log logger.Logger) *Cache { - scoped := *c + scoped := c.scoped() scoped.logger = log - return &scoped + return scoped } func (c *Cache) operationContext(ctx context.Context, source, version string) context.Context { diff --git a/internal/adapters/gitmodules/cache_run.go b/internal/adapters/gitmodules/cache_run.go new file mode 100644 index 00000000..1536d301 --- /dev/null +++ b/internal/adapters/gitmodules/cache_run.go @@ -0,0 +1,99 @@ +package gitmodules + +import ( + "context" + "fmt" + "os" + "sync" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" +) + +// A cache handle belongs to one command. Only its successfully installed, +// immutable snapshots share verification and file inventories within that run. +type cacheRunState struct { + mu sync.Mutex + snapshots map[string]*cachedSnapshot +} + +type cachedSnapshot struct { + gate chan struct{} + verified bool + files []string + identity os.FileInfo +} + +func (snapshot *cachedSnapshot) lock(ctx context.Context) error { + select { + case snapshot.gate <- struct{}{}: + if err := ctx.Err(); err != nil { + snapshot.unlock() + return err + } + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +func (snapshot *cachedSnapshot) unlock() { <-snapshot.gate } + +// accept records a snapshot hashed before its atomic publication. The caller +// holds the per-snapshot gate; no second file-content pass is needed. +func (snapshot *cachedSnapshot) accept(installed string, files []string) error { + info, err := os.Lstat(installed) + if err != nil { + return fmt.Errorf("Lstat: %w", err) + } + // Persistent verification stamps require matching fingerprints before and + // after the content hash. Publication has only the already computed hash; + // a later command verifies the installed tree before writing a stamp. + snapshot.files, snapshot.identity, snapshot.verified = files, info, true + return nil +} + +func (c *Cache) runState() *cacheRunState { + c.stateMu.Lock() + defer c.stateMu.Unlock() + if c.state == nil { + c.state = &cacheRunState{snapshots: make(map[string]*cachedSnapshot)} + } + return c.state +} + +func (c *Cache) scoped() *Cache { + return &Cache{root: c.root, bsrResolver: c.bsrResolver, logger: c.logger, proxy: c.proxy, state: c.runState()} +} + +func (c *Cache) snapshotState(installed string) *cachedSnapshot { + state := c.runState() + state.mu.Lock() + defer state.mu.Unlock() + if snapshot := state.snapshots[installed]; snapshot != nil { + return snapshot + } + snapshot := &cachedSnapshot{gate: make(chan struct{}, 1)} + state.snapshots[installed] = snapshot + return snapshot +} + +// SourceFiles returns the sorted logical file inventory of a snapshot verified +// by this cache handle. Paths are relative to its installation directory and +// include metadata so consumers can preserve nested module ownership. +// The returned slice is independent of the command's immutable cached inventory. +// A nil slice means this handle has no verified inventory and callers must use +// their ordinary source traversal. A verified empty snapshot returns an empty, +// non-nil slice. This method never verifies or trusts a directory's existence. +func (c *Cache) SourceFiles(entry v1.LockedModule) ([]string, error) { + snapshot := c.snapshotState(v1ModuleCachePath(c.root, entry)) + if err := snapshot.lock(context.Background()); err != nil { + return nil, fmt.Errorf("lock: %w", err) + } + defer snapshot.unlock() + if !snapshot.verified { + return nil, nil + } + files := make([]string, len(snapshot.files)) + copy(files, snapshot.files) + return files, nil +} diff --git a/internal/adapters/gitmodules/cache_run_test.go b/internal/adapters/gitmodules/cache_run_test.go new file mode 100644 index 00000000..5826cd1f --- /dev/null +++ b/internal/adapters/gitmodules/cache_run_test.go @@ -0,0 +1,128 @@ +package gitmodules + +import ( + "context" + "fmt" + "os" + "path/filepath" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" +) + +func TestSnapshotVerificationWaitHonorsCancellation(t *testing.T) { + t.Parallel() + cache := New(t.TempDir()) + entry := v1.LockedModule{ + Source: "example.test/module", Version: "v1.0.0", + Commit: "1111111111111111111111111111111111111111", + Hash: "h1:AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", + } + snapshot := cache.snapshotState(v1ModuleCachePath(cache.root, entry)) + require.NoError(t, snapshot.lock(t.Context())) + defer snapshot.unlock() + ctx, cancel := context.WithTimeout(t.Context(), 20*time.Millisecond) + defer cancel() + err := cache.Install(ctx, v1.Lock{Version: 1, Modules: []v1.LockedModule{entry}}) + require.ErrorIs(t, err, context.DeadlineExceeded) +} + +func TestSourceFilesDistinguishesMissingAndEmptyInventory(t *testing.T) { + t.Parallel() + cache := New(t.TempDir()) + entry := v1.LockedModule{ + Source: "example.test/module", Version: "v1.0.0", + Commit: "1111111111111111111111111111111111111111", + } + files, err := cache.SourceFiles(entry) + require.NoError(t, err) + require.Nil(t, files) + // Compute the actual empty-directory hash before selecting the cache path. + entry.Hash, err = hashSnapshotV1Files(t.TempDir()) + require.NoError(t, err) + directory := v1ModuleCachePath(cache.root, entry) + require.NoError(t, os.MkdirAll(directory, 0o755)) + require.NoError(t, cache.VerifyCached(t.Context(), v1.Lock{Version: 1, Modules: []v1.LockedModule{entry}})) + files, err = cache.SourceFiles(entry) + require.NoError(t, err) + require.NotNil(t, files) + require.Empty(t, files) +} + +func TestInstalledSnapshotInventoryIsReusedWithinRun(t *testing.T) { + t.Parallel() + remote, commit := diagnosticRepository(t) + cache := New(t.TempDir()) + fetched, err := cache.Fetch(t.Context(), remote, commit) + require.NoError(t, err) + lock := v1.Lock{Version: 1, Modules: []v1.LockedModule{fetched.Lock}} + require.NoError(t, cache.Install(t.Context(), lock)) + files, err := cache.SourceFiles(fetched.Lock) + require.NoError(t, err) + assert.Equal(t, []string{"api.proto", "easyp.yaml"}, files) + files[0] = "caller mutation" + files, err = cache.WithLogger(nil).SourceFiles(fetched.Lock) + require.NoError(t, err) + assert.Equal(t, []string{"api.proto", "easyp.yaml"}, files) + + other := fetched.Lock + other.Roots = []string{"elsewhere"} + files, err = cache.SourceFiles(other) + require.NoError(t, err) + assert.Nil(t, files) + + directory, _, err := cache.Cached(fetched.Lock) + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(directory, "api.proto"), []byte("tampered"), 0o644)) + // Ordinary use trusts an installed snapshot for this command. A fresh handle + // and explicit verification still detect corruption, without changing files. + require.NoError(t, cache.Install(t.Context(), lock)) + fresh := &Cache{root: cache.root} + files, err = fresh.SourceFiles(fetched.Lock) + require.NoError(t, err) + assert.Nil(t, files) + require.ErrorContains(t, fresh.Install(t.Context(), lock), "hash mismatch") + require.ErrorContains(t, cache.VerifyCached(t.Context(), lock), "hash mismatch") + files, err = cache.SourceFiles(fetched.Lock) + require.NoError(t, err) + assert.Nil(t, files) + require.ErrorContains(t, cache.Install(t.Context(), lock), "hash mismatch") +} + +func TestInstalledSnapshotConcurrentReaders(t *testing.T) { + t.Parallel() + remote, commit := diagnosticRepository(t) + producer := New(t.TempDir()) + fetched, err := producer.Fetch(t.Context(), remote, commit) + require.NoError(t, err) + lock := v1.Lock{Version: 1, Modules: []v1.LockedModule{fetched.Lock}} + cache := &Cache{root: producer.root} + var group sync.WaitGroup + failures := make(chan error, 16) + for range 16 { + group.Go(func() { + if err := cache.Install(t.Context(), lock); err != nil { + failures <- err + return + } + files, err := cache.SourceFiles(fetched.Lock) + if err != nil { + failures <- err + return + } + if len(files) != 2 || files[0] != "api.proto" || files[1] != "easyp.yaml" { + failures <- fmt.Errorf("unexpected inventory %v", files) + } + }) + } + group.Wait() + close(failures) + for err := range failures { + require.NoError(t, err) + } +} diff --git a/internal/adapters/gitmodules/cache_verification.go b/internal/adapters/gitmodules/cache_verification.go index ae3496a9..44f96c55 100644 --- a/internal/adapters/gitmodules/cache_verification.go +++ b/internal/adapters/gitmodules/cache_verification.go @@ -11,8 +11,6 @@ import ( "path" "path/filepath" - "golang.org/x/mod/sumdb/dirhash" - v1 "github.com/easyp-tech/easyp/internal/config/v1" ) @@ -26,41 +24,42 @@ type cacheVerificationStamp struct { } func verifyInstalledV1Module(installed string, entry v1.LockedModule, writeStamp bool) error { - files, err := snapshotV1Files(installed) + _, err := verifiedSnapshotFiles(installed, entry, writeStamp) + return err +} + +func verifiedSnapshotFiles(installed string, entry v1.LockedModule, writeStamp bool) ([]string, error) { + files, fingerprint, fast, err := cacheTreeInventory(installed) if err != nil { - return fmt.Errorf("snapshotV1Files: %w", err) + return nil, fmt.Errorf("cacheTreeInventory: %w", err) } for _, name := range files { if path.Ext(name) != ".proto" && !snapshotConfigFile(name) { - return fmt.Errorf("unrelated snapshot file %q; regenerate the unreleased v1 lock with easyp mod tidy", name) + return nil, fmt.Errorf("unrelated snapshot file %q; regenerate the unreleased v1 lock with easyp mod tidy", name) } } - fingerprint, fast, err := cacheTreeFingerprint(installed) - if err != nil { - return fmt.Errorf("cacheTreeFingerprint: %w", err) - } if fast { stamp, readErr := readCacheVerificationStamp(installed) if readErr == nil && stamp.Version == cacheVerificationVersion && stamp.Hash == entry.Hash && stamp.Roots == v1RootSelectionKey(entry.Roots) && stamp.Fingerprint == fingerprint { - return nil + return files, nil } } - actual, err := dirhash.HashDir(installed, "", dirhash.Hash1) + actual, err := hashV1Files(installed, files) if err != nil { - return fmt.Errorf("HashDir: %w", err) + return nil, fmt.Errorf("hashV1Files: %w", err) } if actual != entry.Hash { - return fmt.Errorf("cached %s@%s hash mismatch: got %s, want %s; inspect or quarantine only cache directory %q, then rerun easyp mod download; keep protobuf.lock unchanged", entry.Source, entry.Commit, actual, entry.Hash, installed) + return nil, fmt.Errorf("cached %s@%s hash mismatch: got %s, want %s; inspect or quarantine only cache directory %q, then rerun easyp mod download; keep protobuf.lock unchanged", entry.Source, entry.Commit, actual, entry.Hash, installed) } if fast { after, afterFast, fingerprintErr := cacheTreeFingerprint(installed) if fingerprintErr != nil { - return fmt.Errorf("cacheTreeFingerprint after hash: %w", fingerprintErr) + return nil, fmt.Errorf("cacheTreeFingerprint after hash: %w", fingerprintErr) } if !afterFast || after != fingerprint { - return fmt.Errorf("cached %s@%s changed while its lock hash was being verified; retry after concurrent cache writes stop", entry.Source, entry.Commit) + return nil, fmt.Errorf("cached %s@%s changed while its lock hash was being verified; retry after concurrent cache writes stop", entry.Source, entry.Commit) } if writeStamp { _ = writeCacheVerificationStamp(installed, cacheVerificationStamp{ @@ -71,7 +70,7 @@ func verifyInstalledV1Module(installed string, entry v1.LockedModule, writeStamp }) } } - return nil + return files, nil } func cacheVerificationStampPath(installed string) string { @@ -120,6 +119,12 @@ func writeCacheVerificationStamp(installed string, stamp cacheVerificationStamp) } func cacheTreeFingerprint(root string) (string, bool, error) { + _, fingerprint, supported, err := cacheTreeInventory(root) + return fingerprint, supported, err +} + +func cacheTreeInventory(root string) ([]string, string, bool, error) { + var files []string hash := sha256.New() supported := true err := filepath.WalkDir(root, func(path string, entry fs.DirEntry, walkErr error) error { @@ -145,6 +150,9 @@ func cacheTreeFingerprint(root string) (string, bool, error) { relative = "" } name := filepath.ToSlash(relative) + if info.Mode().IsRegular() { + files = append(files, name) + } if err := binary.Write(hash, binary.LittleEndian, uint32(len(name))); err != nil { return err } @@ -167,7 +175,7 @@ func cacheTreeFingerprint(root string) (string, bool, error) { return nil }) if err != nil { - return "", false, err + return nil, "", false, err } - return "m1:" + base64.StdEncoding.EncodeToString(hash.Sum(nil)), supported, nil + return files, "m1:" + base64.StdEncoding.EncodeToString(hash.Sum(nil)), supported, nil } diff --git a/internal/adapters/gitmodules/checkout_test.go b/internal/adapters/gitmodules/checkout_test.go index e8933930..e8ef5d8f 100644 --- a/internal/adapters/gitmodules/checkout_test.go +++ b/internal/adapters/gitmodules/checkout_test.go @@ -62,7 +62,7 @@ func TestFetchV1ModuleRemovesTemporaryCheckouts(t *testing.T) { entries, err := os.ReadDir(cache) require.NoError(t, err) for _, entry := range entries { - assert.Equal(t, "objects", entry.Name(), "temporary checkout was not removed") + assert.Contains(t, []string{"objects", "modules", "sources"}, entry.Name(), "temporary checkout was not removed") } }) } diff --git a/internal/adapters/gitmodules/diagnostics_test.go b/internal/adapters/gitmodules/diagnostics_test.go index b05c575f..38f60f9b 100644 --- a/internal/adapters/gitmodules/diagnostics_test.go +++ b/internal/adapters/gitmodules/diagnostics_test.go @@ -39,7 +39,7 @@ func TestCacheIntegrityDiagnosticIdentifiesRecoveryPath(t *testing.T) { require.NoError(t, err) name := filepath.Join(installed, "api.proto") require.NoError(t, os.WriteFile(name, []byte("tampered"), 0o644)) - err = cache.Install(t.Context(), lock) + err = (&Cache{root: cache.root}).Install(t.Context(), lock) require.ErrorContains(t, err, "hash mismatch") assert.Contains(t, err.Error(), installed) assert.Contains(t, err.Error(), "protobuf.lock unchanged") diff --git a/internal/adapters/gitmodules/download.go b/internal/adapters/gitmodules/download.go index 5667598f..88ca3dbe 100644 --- a/internal/adapters/gitmodules/download.go +++ b/internal/adapters/gitmodules/download.go @@ -20,8 +20,8 @@ func v1ModuleCachePath(cacheRoot string, entry v1.LockedModule) string { return filepath.Join(cacheRoot, "modules", v1CacheSourceKey(entry.Source), v1CacheSourceKey(identity)) } -// Install installs each locked Git commit, verifying its content hash -// before accepting a downloaded or already cached module. +// Install installs each locked Git commit and verifies its contents once per +// cache handle. Installed snapshots are immutable for the command's lifetime. func (c *Cache) Install(ctx context.Context, lock v1.Lock) error { return c.install(ctx, lock, true) } @@ -57,7 +57,20 @@ func (c *Cache) installEntry(ctx context.Context, entry v1.LockedModule, acquire finish := gitcommand.Start(ctx, "install module") defer func() { finish(err) }() installed := v1ModuleCachePath(c.root, entry) + snapshot := c.snapshotState(installed) + if err := snapshot.lock(ctx); err != nil { + return fmt.Errorf("lock: %w", err) + } + defer snapshot.unlock() + if !acquire { + // Explicit read-only verification must not reuse an earlier success. + // A failed check also invalidates later Install and SourceFiles calls. + snapshot.verified = false + } info, err := os.Lstat(installed) + if err != nil || !os.SameFile(snapshot.identity, info) { + snapshot.verified = false + } if errors.Is(err, os.ErrNotExist) { if !acquire { return fmt.Errorf("policy module %s@%s is not cached; run easyp mod download before validate-config: %w", entry.Source, entry.Commit, err) @@ -76,8 +89,12 @@ func (c *Cache) installEntry(ctx context.Context, entry v1.LockedModule, acquire // rather than racing a second rename into the same cache directory. info, err = os.Lstat(installed) if errors.Is(err, os.ErrNotExist) { - if err := fetchPinnedV1Module(ctx, entry, c.root, installed, c.proxy); err != nil { - return err + files, fetchErr := fetchPinnedV1Module(ctx, entry, c.root, installed, c.proxy) + if fetchErr != nil { + return fetchErr + } + if err := snapshot.accept(installed, files); err != nil { + return fmt.Errorf("accept: %w", err) } info, err = os.Lstat(installed) } @@ -88,10 +105,16 @@ func (c *Cache) installEntry(ctx context.Context, entry v1.LockedModule, acquire if !info.IsDir() { return fmt.Errorf("%s: cached path %q is not a directory; inspect this path and keep protobuf.lock unchanged", entry.Source, installed) } - if err := verifyInstalledV1Module(installed, entry, acquire); err != nil { - return fmt.Errorf("verify cached %s at %q: %w", entry.Source, installed, err) + if !snapshot.verified { + files, err := verifiedSnapshotFiles(installed, entry, acquire) + if err != nil { + return fmt.Errorf("verify cached %s at %q: %w", entry.Source, installed, err) + } + snapshot.files, snapshot.identity = files, info + snapshot.verified = true } if err := moduleconfig.ValidateLegacyMajor(installed, entry.Source, entry.Version); err != nil { + snapshot.verified = false return fmt.Errorf("ValidateLegacyMajor: %w", err) } return nil diff --git a/internal/adapters/gitmodules/git_object_sizes.go b/internal/adapters/gitmodules/git_object_sizes.go index 85516f47..dad7b347 100644 --- a/internal/adapters/gitmodules/git_object_sizes.go +++ b/internal/adapters/gitmodules/git_object_sizes.go @@ -14,13 +14,28 @@ import ( "github.com/go-git/go-git/v5/storage/filesystem" ) -// Go-git's blob reads follow alternates, but EncodedObjectSize does not. Keep -// header-only size queries working for our shared pinned-commit checkouts too. +// Shared checkouts resolve both objects and their sizes through explicitly +// rooted stores. Go-git's alternate filesystem can lose Windows drive names or +// reinterpret relative paths, and EncodedObjectSize never follows alternates. type snapshotObjectStorage struct { storage.Storer alternates []storage.Storer } +func (s snapshotObjectStorage) EncodedObject(kind plumbing.ObjectType, hash plumbing.Hash) (plumbing.EncodedObject, error) { + object, err := s.Storer.EncodedObject(kind, hash) + if !errors.Is(err, plumbing.ErrObjectNotFound) { + return object, err + } + for _, alternate := range s.alternates { + object, err = alternate.EncodedObject(kind, hash) + if !errors.Is(err, plumbing.ErrObjectNotFound) { + return object, err + } + } + return nil, err +} + func (s snapshotObjectStorage) EncodedObjectSize(hash plumbing.Hash) (int64, error) { size, err := s.Storer.EncodedObjectSize(hash) if !errors.Is(err, plumbing.ErrObjectNotFound) { diff --git a/internal/adapters/gitmodules/git_object_sizes_test.go b/internal/adapters/gitmodules/git_object_sizes_test.go new file mode 100644 index 00000000..15660c8c --- /dev/null +++ b/internal/adapters/gitmodules/git_object_sizes_test.go @@ -0,0 +1,66 @@ +package gitmodules + +import ( + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestSnapshotStorageReadsSharedObjects(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + name string + relative bool + }{ + {name: "absolute alternate"}, + {name: "relative alternate", relative: true}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + remote, commit := diagnosticRepository(t) + checkout := filepath.Join(t.TempDir(), "shared checkout") + runTestGit(t, remote, "clone", "--quiet", "--shared", "--no-checkout", "--", remote, checkout) + objects := filepath.Join(checkout, ".git", "objects") + require.NoFileExists(t, filepath.Join(objects, commit[:2], commit[2:])) + packs, err := os.ReadDir(filepath.Join(objects, "pack")) + require.NoError(t, err) + require.Empty(t, packs) + alternate := filepath.Join(remote, ".git", "objects") + if tt.relative { + var err error + alternate, err = filepath.Rel(objects, alternate) + require.NoError(t, err) + } + require.NoError(t, os.WriteFile(filepath.Join(objects, "info", "alternates"), []byte(filepath.ToSlash(alternate)+"\n"), 0o644)) + // The primary checkout has no object bodies. Native Git and EasyP + // must both read the same shared object database on every platform. + runTestGit(t, checkout, "cat-file", "-e", commit+"^{commit}") + store, err := snapshotStorage(checkout) + require.NoError(t, err) + repository, err := git.Open(store, nil) + require.NoError(t, err) + revision, err := repository.CommitObject(plumbing.NewHash(commit)) + require.NoError(t, err) + tree, err := revision.Tree() + require.NoError(t, err) + file, err := tree.File("api.proto") + require.NoError(t, err) + size, err := store.EncodedObjectSize(file.Hash) + require.NoError(t, err) + reader, err := file.Reader() + require.NoError(t, err) + contents, err := io.ReadAll(reader) + require.NoError(t, err) + require.NoError(t, reader.Close()) + assert.Equal(t, int64(len(contents)), size) + assert.True(t, strings.HasPrefix(string(contents), "syntax = \"proto3\";")) + }) + } +} diff --git a/internal/adapters/gitmodules/migration.go b/internal/adapters/gitmodules/migration.go index 05b4bc10..74c536ec 100644 --- a/internal/adapters/gitmodules/migration.go +++ b/internal/adapters/gitmodules/migration.go @@ -156,7 +156,7 @@ func (c *Cache) fetchMigration(ctx context.Context, source, version, legacyHash if err != nil { return modules.Fetched{}, fmt.Errorf("sourceV1SnapshotView: %w", err) } - if err := materializeSelectedSnapshotAliases(ctx, view, checkout.snapshot, checkout.module, nil); err != nil { + if err := materializeSelectedSnapshotAliases(ctx, newSnapshotWriter(view, checkout.snapshot), checkout.module, nil, true); err != nil { return modules.Fetched{}, fmt.Errorf("materializeSelectedSnapshotAliases: %w", err) } checkout.inspection, err = inspectV1Snapshot(ctx, view, checkout.snapshot, checkout.module, checkout.commit) diff --git a/internal/adapters/gitmodules/migration_bsr.go b/internal/adapters/gitmodules/migration_bsr.go index 088cc865..2140ded2 100644 --- a/internal/adapters/gitmodules/migration_bsr.go +++ b/internal/adapters/gitmodules/migration_bsr.go @@ -16,9 +16,9 @@ func (c *Cache) WithMigrationPins(pins []v1.Requirement) modules.MigrationReposi for _, pin := range pins { historical[pin.Module] = pin.Version } - scoped := *c + scoped := c.scoped() scoped.bsrResolver = migrationBSRResolver{base: c.bsrResolver, pins: historical} - return &scoped + return scoped } type migrationBSRResolver struct { diff --git a/internal/adapters/gitmodules/object_cache_test.go b/internal/adapters/gitmodules/object_cache_test.go index a7bd9640..a8b43263 100644 --- a/internal/adapters/gitmodules/object_cache_test.go +++ b/internal/adapters/gitmodules/object_cache_test.go @@ -35,7 +35,7 @@ func TestPinnedObjectCacheWorksWithoutRemote(t *testing.T) { dir, _, err := cache.Cached(first.Lock) require.NoError(t, err) require.NoError(t, os.WriteFile(filepath.Join(dir, "api.proto"), []byte("tampered"), 0o644)) - require.ErrorContains(t, cache.Install(t.Context(), v1.Lock{Version: 1, Modules: []v1.LockedModule{first.Lock}}), "hash mismatch") + require.ErrorContains(t, cache.VerifyCached(t.Context(), v1.Lock{Version: 1, Modules: []v1.LockedModule{first.Lock}}), "hash mismatch") } func TestObjectRepositoryLockHonorsCancellation(t *testing.T) { diff --git a/internal/adapters/gitmodules/root_selection_test.go b/internal/adapters/gitmodules/root_selection_test.go index 0c977dea..03bb4a51 100644 --- a/internal/adapters/gitmodules/root_selection_test.go +++ b/internal/adapters/gitmodules/root_selection_test.go @@ -56,6 +56,8 @@ func TestFetchWithRootsReplaysColdAndWarmSelections(t *testing.T) { lock := v1.Lock{Version: 1, Modules: []v1.LockedModule{entry}} require.NoError(t, cache.Install(t.Context(), lock)) require.NoError(t, cache.Install(t.Context(), lock)) + // A new command independently verifies the published tree and may stamp it. + require.NoError(t, (&Cache{root: cache.root}).Install(t.Context(), lock)) require.NoError(t, cache.VerifyCached(t.Context(), lock)) directory, module, err := cache.Cached(entry) require.NoError(t, err) diff --git a/internal/adapters/gitmodules/snapshot.go b/internal/adapters/gitmodules/snapshot.go index 47c598a0..6f0fc34d 100644 --- a/internal/adapters/gitmodules/snapshot.go +++ b/internal/adapters/gitmodules/snapshot.go @@ -34,7 +34,7 @@ func prepareV1Snapshot(ctx context.Context, checkout, commit, source, subdir str if err != nil { return "", v1.Module{}, nil, fmt.Errorf("sourceV1SnapshotTree: %w", err) } - view := sourceview.New(tree) + view := sourceview.NewImmutable(tree) directory, err = os.MkdirTemp(filepath.Dir(checkout), "snapshot-*") if err != nil { return "", v1.Module{}, nil, fmt.Errorf("MkdirTemp: %w", err) @@ -48,7 +48,8 @@ func prepareV1Snapshot(ctx context.Context, checkout, commit, source, subdir str } } }() - inventory, err := stageSnapshotMetadata(ctx, tree, view, directory, subdir) + writer := newSnapshotWriter(view, directory) + inventory, err := stageSnapshotMetadata(ctx, tree, writer, subdir) if err != nil { return "", v1.Module{}, nil, fmt.Errorf("stageSnapshotMetadata: %w", err) } @@ -85,21 +86,23 @@ func prepareV1Snapshot(ctx context.Context, checkout, commit, source, subdir str return "", v1.Module{}, nil, fmt.Errorf("pruneEmptySnapshotParents: %w", err) } } + // Ignored metadata may have removed directories with a different host spelling. + writer.destination.Reset() // Stage ordinary files only after metadata has selected Buf source paths. // Discarded proto names must not collide with retained names on the host. for _, name := range regular { if path.Ext(name) == ".proto" && len(selectV1ProtoFiles([]string{name}, module.ProtoFilters)) == 0 { continue } - if err := materializeSnapshotFile(ctx, view, directory, name); err != nil { - return "", v1.Module{}, nil, fmt.Errorf("materializeSnapshotFile: %w", err) + if err := writer.file(ctx, name); err != nil { + return "", v1.Module{}, nil, fmt.Errorf("file: %w", err) } } - if err := materializeSnapshotAuxiliaryAliases(ctx, view, directory, aliases); err != nil { + if err := materializeSnapshotAuxiliaryAliases(ctx, writer, aliases); err != nil { return "", v1.Module{}, nil, fmt.Errorf("materializeSnapshotAuxiliaryAliases: %w", err) } if !provisional { - if err := materializeSelectedSnapshotAliases(ctx, view, directory, module, ignoredMetadata); err != nil { + if err := materializeSelectedSnapshotAliases(ctx, writer, module, ignoredMetadata, len(aliases) > 0); err != nil { return "", v1.Module{}, nil, fmt.Errorf("materializeSelectedSnapshotAliases: %w", err) } } @@ -110,26 +113,6 @@ func prepareV1Snapshot(ctx context.Context, checkout, commit, source, subdir str } inspection.Provisional = provisional } - // Buf filters select the same logical paths for hashing and installation. - err = filepath.WalkDir(directory, func(file string, entry fs.DirEntry, walkErr error) error { - if walkErr != nil { - return walkErr - } - if entry.IsDir() || filepath.Ext(file) != ".proto" { - return nil - } - name, err := filepath.Rel(directory, file) - if err != nil { - return fmt.Errorf("Rel: %w", err) - } - if len(selectV1ProtoFiles([]string{filepath.ToSlash(name)}, module.ProtoFilters)) == 0 { - return os.Remove(file) - } - return nil - }) - if err != nil { - return "", v1.Module{}, nil, fmt.Errorf("WalkDir: %w", err) - } return directory, module, inspection, nil } @@ -160,18 +143,19 @@ func pruneEmptySnapshotParents(directory, name string) error { // stageSnapshotMetadata discovers immutable metadata before source filtering. // Failed markers preserve ownership and parser precedence without pointer reads. -func stageSnapshotMetadata(ctx context.Context, tree *gitsnapshot.FS, view *sourceview.View, directory, subdir string) (snapshotInventory, error) { +func stageSnapshotMetadata(ctx context.Context, tree *gitsnapshot.FS, writer *snapshotWriter, subdir string) (snapshotInventory, error) { + view := writer.view var aliases, regular []string var metadataFailures []snapshotMetadataFailure failedMarker := func(name string, cause error) error { failure := snapshotMetadataFailure{name: name, err: cause} - err := makeSnapshotDirectory(directory, name) + err := writer.mkdir(name) if errors.Is(err, gitsnapshot.ErrPathCollision) { // A failed alias has no retained bytes. Keep its witness separate // if the host cannot represent it beside staged metadata. failure.unstaged = true } else if err != nil { - return fmt.Errorf("makeSnapshotDirectory: %w", err) + return fmt.Errorf("mkdir: %w", err) } metadataFailures = append(metadataFailures, failure) return nil @@ -203,7 +187,7 @@ func stageSnapshotMetadata(ctx context.Context, tree *gitsnapshot.FS, view *sour } return nil } - return materializeSnapshotFile(ctx, view, directory, name) + return writer.file(ctx, name) }) if err != nil { return snapshotInventory{}, fmt.Errorf("WalkDir: %w", err) @@ -248,7 +232,7 @@ func stageSnapshotMetadata(ctx context.Context, tree *gitsnapshot.FS, view *sour return nil } if moduleconfig.IsGitDependencyConfigFile(path.Base(logical)) && !snapshotMetadataExcluded(logical, subdir) { - return materializeSnapshotFile(ctx, view, directory, logical) + return writer.file(ctx, logical) } return nil }) @@ -258,8 +242,8 @@ func stageSnapshotMetadata(ctx context.Context, tree *gitsnapshot.FS, view *sour continue } if moduleconfig.IsGitDependencyConfigFile(path.Base(name)) { - if err := materializeSnapshotFile(ctx, view, directory, name); err != nil { - return snapshotInventory{}, fmt.Errorf("materializeSnapshotFile: %w", err) + if err := writer.file(ctx, name); err != nil { + return snapshotInventory{}, fmt.Errorf("file: %w", err) } } } @@ -268,13 +252,14 @@ func stageSnapshotMetadata(ctx context.Context, tree *gitsnapshot.FS, view *sour // materializeSelectedSnapshotAliases validates declared roots and copies aliases // selected by source ownership and Buf filters. Auxiliary links are handled separately. -func materializeSelectedSnapshotAliases(ctx context.Context, view *sourceview.View, directory string, module v1.Module, ignoredMetadata map[string]bool) error { +func materializeSelectedSnapshotAliases(ctx context.Context, writer *snapshotWriter, module v1.Module, ignoredMetadata map[string]bool, hasAliases bool) error { + view, directory := writer.view, writer.directory for _, root := range module.Roots { logicalRoot := filepath.ToSlash(root) resolution, resolveErr := view.Resolve(ctx, logicalRoot) if resolveErr != nil { if errors.Is(resolveErr, fs.ErrNotExist) && len(resolution.Links) == 0 { - if err := makeSnapshotDirectory(directory, root); err != nil { + if err := writer.mkdir(root); err != nil { return fmt.Errorf("MkdirAll: %w", err) } continue @@ -284,9 +269,12 @@ func materializeSelectedSnapshotAliases(ctx context.Context, view *sourceview.Vi if !resolution.Info.IsDir() { return fmt.Errorf("module %s has invalid root %q: not a directory", module.Name, root) } - if err := makeSnapshotDirectory(directory, root); err != nil { + if err := writer.mkdir(root); err != nil { return fmt.Errorf("MkdirAll: %w", err) } + if !hasAliases { + continue + } err := view.WalkSelected(ctx, logicalRoot, snapshotSourceCandidate, func(logical string, resolved sourceview.Resolution, walkErr error) error { if ignoredMetadata[logical] { return nil @@ -317,7 +305,7 @@ func materializeSelectedSnapshotAliases(ctx context.Context, view *sourceview.Vi if !snapshotSelectedAlias(logical, module) { return nil } - return materializeSnapshotFile(ctx, view, directory, logical) + return writer.file(ctx, logical) }) if err != nil { return fmt.Errorf("Walk: root %q: %w", root, err) @@ -331,7 +319,7 @@ func sourceV1SnapshotView(ctx context.Context, checkout, commit string) (*source if err != nil { return nil, fmt.Errorf("sourceV1SnapshotTree: %w", err) } - return sourceview.New(tree), nil + return sourceview.NewImmutable(tree), nil } func sourceV1SnapshotTree(ctx context.Context, checkout, commit string) (*gitsnapshot.FS, error) { @@ -420,7 +408,8 @@ func snapshotFailedMetadataOwnsSources(name, directory string, module v1.Module) // materializeSnapshotAuxiliaryAliases retains bounded regular non-proto aliases. // Consumers can select arbitrary policy fragments; Fetch has no consumer context // with which to narrow those paths. Failed auxiliary aliases remain omitted. -func materializeSnapshotAuxiliaryAliases(ctx context.Context, view *sourceview.View, directory string, aliases []string) error { +func materializeSnapshotAuxiliaryAliases(ctx context.Context, writer *snapshotWriter, aliases []string) error { + view := writer.view for _, name := range aliases { resolved, err := view.Resolve(ctx, name) if err != nil { @@ -437,7 +426,7 @@ func materializeSnapshotAuxiliaryAliases(ctx context.Context, view *sourceview.V if resolved.Info.IsDir() || !snapshotConfigFile(logical) { return nil } - return materializeSnapshotFile(ctx, view, directory, logical) + return writer.file(ctx, logical) }) if err != nil { return fmt.Errorf("Walk: %w", err) @@ -447,8 +436,8 @@ func materializeSnapshotAuxiliaryAliases(ctx context.Context, view *sourceview.V if !snapshotConfigFile(name) { continue } - if err := materializeSnapshotFile(ctx, view, directory, name); err != nil { - return fmt.Errorf("materializeSnapshotFile: %w", err) + if err := writer.file(ctx, name); err != nil { + return fmt.Errorf("file: %w", err) } } return nil @@ -528,8 +517,18 @@ func snapshotExcludedDirectory(name, root, directory string, module v1.Module) b return true } -func materializeSnapshotFile(ctx context.Context, view *sourceview.View, directory, name string) error { - file, err := view.Open(ctx, name) +type snapshotWriter struct { + view *sourceview.View + directory string + destination *gitsnapshot.DestinationValidator +} + +func newSnapshotWriter(view *sourceview.View, directory string) *snapshotWriter { + return &snapshotWriter{view: view, directory: directory, destination: gitsnapshot.NewDestinationValidator(directory)} +} + +func (writer *snapshotWriter) file(ctx context.Context, name string) error { + file, err := writer.view.Open(ctx, name) if err != nil { return fmt.Errorf("Open: %w", err) } @@ -542,8 +541,8 @@ func materializeSnapshotFile(ctx context.Context, view *sourceview.View, directo _ = file.Close() return fmt.Errorf("non-regular snapshot file %q", name) } - destination := filepath.Join(directory, filepath.FromSlash(name)) - if err := gitsnapshot.ValidateDestination(directory, name); err != nil { + destination := filepath.Join(writer.directory, filepath.FromSlash(name)) + if err := writer.destination.Validate(name); err != nil { closeErr := file.Close() return errors.Join(fmt.Errorf("ValidateDestination: %w", err), closeErr) } @@ -561,16 +560,18 @@ func materializeSnapshotFile(ctx context.Context, view *sourceview.View, directo if err := errors.Join(copyErr, closeErr, ctx.Err()); err != nil { return fmt.Errorf("Copy: %w", err) } + writer.destination.Record(name) return nil } -func makeSnapshotDirectory(directory, name string) error { - if err := gitsnapshot.ValidateDestination(directory, name); err != nil { +func (writer *snapshotWriter) mkdir(name string) error { + if err := writer.destination.Validate(name); err != nil { return fmt.Errorf("ValidateDestination: %w", err) } - if err := os.MkdirAll(filepath.Join(directory, filepath.FromSlash(name)), 0o755); err != nil { + if err := os.MkdirAll(filepath.Join(writer.directory, filepath.FromSlash(name)), 0o755); err != nil { return fmt.Errorf("MkdirAll: %w", err) } + writer.destination.Record(name) return nil } diff --git a/internal/adapters/gitmodules/snapshot_checkout.go b/internal/adapters/gitmodules/snapshot_checkout.go index 21de7bb2..6daf5173 100644 --- a/internal/adapters/gitmodules/snapshot_checkout.go +++ b/internal/adapters/gitmodules/snapshot_checkout.go @@ -64,10 +64,15 @@ func (c *Cache) fetchWithRootSelection(ctx context.Context, source, version stri return modules.Fetched{}, fmt.Errorf("ValidateLegacyMajor: %w", err) } var hash string + var files []string if checkout.inspection == nil || !checkout.inspection.Provisional { - hash, err = hashSnapshotV1Files(checkout.snapshot) + files, err = snapshotV1Files(checkout.snapshot) if err != nil { - return modules.Fetched{}, fmt.Errorf("hashSnapshotV1Files: %w", err) + return modules.Fetched{}, fmt.Errorf("snapshotV1Files: %w", err) + } + hash, err = hashV1Files(checkout.snapshot, files) + if err != nil { + return modules.Fetched{}, fmt.Errorf("hashV1Files: %w", err) } } if version == "" { @@ -77,7 +82,13 @@ func (c *Cache) fetchWithRootSelection(ctx context.Context, source, version stri if err != nil { return modules.Fetched{}, fmt.Errorf("resolveBSRDependencies: %w", err) } - return modules.Fetched{Module: module, Lock: v1.LockedModule{Source: source, Version: version, Commit: checkout.commit, Hash: hash, Roots: lockedV1ModuleRoots(module, roots), BSR: bindings}, Inspection: checkout.inspection}, nil + entry := v1.LockedModule{Source: source, Version: version, Commit: checkout.commit, Hash: hash, Roots: lockedV1ModuleRoots(module, roots), BSR: bindings} + if hash != "" { + if err := c.retainFetchedSnapshot(ctx, checkout, entry, files); err != nil { + return modules.Fetched{}, fmt.Errorf("retainFetchedSnapshot: %w", err) + } + } + return modules.Fetched{Module: module, Lock: entry, Inspection: checkout.inspection}, nil } // checkoutV1ModuleWithRoots retains Git objects and an index for historical @@ -180,10 +191,10 @@ func checkoutSnapshotCandidate(ctx context.Context, dir, source, version string, return v1ModuleCheckout{dir: dir, module: module, commit: commit, snapshot: stage, inspection: inspection}, true, nil } -func fetchPinnedV1Module(ctx context.Context, entry v1.LockedModule, cacheRoot, installed, proxy string) (err error) { +func fetchPinnedV1Module(ctx context.Context, entry v1.LockedModule, cacheRoot, installed, proxy string) (files []string, err error) { checkout, err := checkoutV1ModuleWithRoots(ctx, entry.Source, entry.Commit, cacheRoot, entry.Roots, false, proxy) if err != nil { - return fmt.Errorf("checkoutV1ModuleWithRoots: %w", err) + return nil, fmt.Errorf("checkoutV1ModuleWithRoots: %w", err) } defer func() { for _, directory := range []string{checkout.snapshot, checkout.dir} { @@ -193,24 +204,28 @@ func fetchPinnedV1Module(ctx context.Context, entry v1.LockedModule, cacheRoot, } } }() - actual, err := hashSnapshotV1Files(checkout.snapshot) + files, err = snapshotV1Files(checkout.snapshot) + if err != nil { + return nil, fmt.Errorf("snapshotV1Files: %w", err) + } + actual, err := hashV1Files(checkout.snapshot, files) if err != nil { - return fmt.Errorf("hashSnapshotV1Files: %w", err) + return nil, fmt.Errorf("hashV1Files: %w", err) } if !strings.EqualFold(checkout.commit, entry.Commit) { - return fmt.Errorf("%s: snapshot commit does not match lock", entry.Source) + return nil, fmt.Errorf("%s: snapshot commit does not match lock", entry.Source) } if actual != entry.Hash { - return fmt.Errorf("%s@%s hash mismatch: got %s, want %s; downloaded contents do not match the pinned hash; investigate repository integrity and keep protobuf.lock unchanged", entry.Source, entry.Commit, actual, entry.Hash) + return nil, fmt.Errorf("%s@%s hash mismatch: got %s, want %s; downloaded contents do not match the pinned hash; investigate repository integrity and keep protobuf.lock unchanged", entry.Source, entry.Commit, actual, entry.Hash) } if err := moduleconfig.ValidateLegacyMajor(checkout.snapshot, entry.Source, entry.Version); err != nil { - return fmt.Errorf("ValidateLegacyMajor: %w", err) + return nil, fmt.Errorf("ValidateLegacyMajor: %w", err) } if err := writeSourceBinding(ctx, cacheRoot, checkout.dir, entry); err != nil { - return fmt.Errorf("writeSourceBinding: %w", err) + return nil, fmt.Errorf("writeSourceBinding: %w", err) } if err := os.Rename(checkout.snapshot, installed); err != nil { - return fmt.Errorf("Rename: %w", err) + return nil, fmt.Errorf("Rename: %w", err) } - return nil + return files, nil } diff --git a/internal/adapters/gitmodules/snapshot_reuse.go b/internal/adapters/gitmodules/snapshot_reuse.go new file mode 100644 index 00000000..7930aa9b --- /dev/null +++ b/internal/adapters/gitmodules/snapshot_reuse.go @@ -0,0 +1,107 @@ +package gitmodules + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" +) + +// retainFetchedSnapshot publishes the bytes just hashed by Fetch. Install can +// reuse this complete snapshot instead of materializing the same commit again. +// Provisional root inspections never reach this path. +func (c *Cache) retainFetchedSnapshot(ctx context.Context, checkout v1ModuleCheckout, entry v1.LockedModule, files []string) error { + installed := v1ModuleCachePath(c.root, entry) + snapshot := c.snapshotState(installed) + if err := snapshot.lock(ctx); err != nil { + return fmt.Errorf("lock: %w", err) + } + defer snapshot.unlock() + if err := os.MkdirAll(filepath.Dir(installed), 0o755); err != nil { + return fmt.Errorf("MkdirAll: %w", err) + } + unlock, err := lockObjectRepository(ctx, installed+".install.lock") + if err != nil { + return fmt.Errorf("lockObjectRepository: %w", err) + } + defer unlock() + info, err := os.Lstat(installed) + if err == nil { + if !info.IsDir() { + return fmt.Errorf("cached path %q is not a directory", installed) + } + if !snapshot.verified || !os.SameFile(snapshot.identity, info) { + snapshot.verified = false + files, err = verifiedSnapshotFiles(installed, entry, true) + if err != nil { + return fmt.Errorf("verifiedSnapshotFiles: %w", err) + } + } + } else if errors.Is(err, os.ErrNotExist) { + if err := retainFetchedSource(ctx, c.root, checkout.dir, entry); err != nil { + return fmt.Errorf("retainFetchedSource: %w", err) + } + if err := os.Rename(checkout.snapshot, installed); err != nil { + return fmt.Errorf("Rename: %w", err) + } + return snapshot.accept(installed, files) + } else { + return fmt.Errorf("Lstat: %w", err) + } + snapshot.files, snapshot.identity, snapshot.verified = files, info, true + return nil +} + +// retainFetchedSource copies already fetched objects locally. HEAD/tag Fetch +// must not download the revision a second time just to preserve policy objects. +func retainFetchedSource(ctx context.Context, root, checkout string, entry v1.LockedModule) error { + absoluteCheckout, err := filepath.Abs(checkout) + if err != nil { + return fmt.Errorf("Abs: %w", err) + } + checkout = absoluteCheckout + raw, err := gitV1(ctx, checkout, "config", "--get", "remote.origin.url") + if err != nil { + return fmt.Errorf("gitV1: %w", err) + } + origin := strings.TrimSpace(raw) + objectsRoot, err := filepath.Abs(filepath.Join(root, "objects")) + if err != nil { + return fmt.Errorf("Abs: %w", err) + } + relative, relErr := filepath.Rel(objectsRoot, origin) + if relErr == nil && filepath.IsLocal(relative) && !strings.ContainsAny(relative, "/\\") { + return writeSourceBindingRepository(root, origin, entry) + } + if err := os.MkdirAll(objectsRoot, 0o700); err != nil { + return fmt.Errorf("MkdirAll: %w", err) + } + repository := filepath.Join(objectsRoot, v1CacheSourceKey(origin)) + unlock, err := lockObjectRepository(ctx, repository+".lock") + if err != nil { + return fmt.Errorf("lockObjectRepository: %w", err) + } + defer unlock() + if _, err := os.Stat(filepath.Join(repository, "HEAD")); errors.Is(err, os.ErrNotExist) { + if _, err := gitV1(ctx, "", "clone", "--quiet", "--bare", "--no-hardlinks", "--", checkout, repository); err != nil { + return fmt.Errorf("gitV1: %w", err) + } + if _, err := gitV1(ctx, repository, "config", "gc.auto", "0"); err != nil { + return fmt.Errorf("gitV1: %w", err) + } + } else if err != nil { + return fmt.Errorf("Stat: %w", err) + } else { + if _, err := gitV1(ctx, repository, "fetch", "--quiet", "--no-tags", "--no-write-fetch-head", "--depth=1", "--", checkout, entry.Commit); err != nil { + return fmt.Errorf("gitV1: %w", err) + } + } + if _, err := gitV1(ctx, repository, "update-ref", "refs/easyp/"+strings.ToLower(entry.Commit), entry.Commit); err != nil { + return fmt.Errorf("gitV1: %w", err) + } + return writeSourceBindingRepository(root, repository, entry) +} diff --git a/internal/adapters/gitmodules/snapshot_reuse_test.go b/internal/adapters/gitmodules/snapshot_reuse_test.go new file mode 100644 index 00000000..bff70187 --- /dev/null +++ b/internal/adapters/gitmodules/snapshot_reuse_test.go @@ -0,0 +1,55 @@ +package gitmodules + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" +) + +func TestFetchRetainsSnapshotForOfflineInstall(t *testing.T) { + t.Parallel() + for _, tt := range []struct { + name, version string + commit bool + }{ + {name: "head"}, + {name: "tag", version: "v1.0.0"}, + {name: "commit", commit: true}, + } { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + remote, commit := diagnosticRepository(t) + version := tt.version + if tt.commit { + version = commit + } + cache := New(t.TempDir()) + fetched, err := cache.Fetch(t.Context(), remote, version) + require.NoError(t, err) + directory, _, err := cache.Cached(fetched.Lock) + require.NoError(t, err) + before, err := os.Stat(filepath.Join(directory, "api.proto")) + require.NoError(t, err) + lock := v1.Lock{Version: 1, Modules: []v1.LockedModule{fetched.Lock}} + require.NoError(t, os.Rename(remote, remote+"-offline")) + require.NoError(t, cache.Install(t.Context(), lock)) + assert.NoFileExists(t, cacheVerificationStampPath(directory), "publication must not attest a fingerprint collected after hashing") + after, err := os.Stat(filepath.Join(directory, "api.proto")) + require.NoError(t, err) + assert.True(t, os.SameFile(before, after), "Install must retain the already materialized file") + require.NoError(t, (&Cache{root: cache.root}).VerifyCached(t.Context(), lock)) + + // Retained local objects also reconstruct the same pinned snapshot + // after removing the installation, without the original repository. + require.NoError(t, os.RemoveAll(directory)) + fresh := &Cache{root: cache.root} + require.NoError(t, fresh.Install(t.Context(), lock)) + require.NoError(t, fresh.VerifyCached(t.Context(), lock)) + }) + } +} diff --git a/internal/adapters/gitmodules/source_binding.go b/internal/adapters/gitmodules/source_binding.go index b9f03be7..3e494d4d 100644 --- a/internal/adapters/gitmodules/source_binding.go +++ b/internal/adapters/gitmodules/source_binding.go @@ -48,6 +48,10 @@ func writeSourceBinding(ctx context.Context, root, checkout string, entry v1.Loc return fmt.Errorf("gitV1: %w", err) } repository := strings.TrimSpace(data) + return writeSourceBindingRepository(root, repository, entry) +} + +func writeSourceBindingRepository(root, repository string, entry v1.LockedModule) (err error) { objectsRoot, err := filepath.Abs(filepath.Join(root, "objects")) if err != nil { return fmt.Errorf("Abs: %w", err) diff --git a/internal/adapters/gitsnapshot/destination.go b/internal/adapters/gitsnapshot/destination.go index 3f576a26..94655613 100644 --- a/internal/adapters/gitsnapshot/destination.go +++ b/internal/adapters/gitsnapshot/destination.go @@ -14,12 +14,60 @@ var ErrPathCollision = errors.New("snapshot path collision") // ValidateDestination rejects distinct Git paths that the host filesystem aliases. // Git paths stay case-sensitive even on a case-insensitive staging filesystem. func ValidateDestination(directory, name string) error { + return validateDestination(directory, name, nil, nil) +} + +// DestinationValidator reuses validated path prefixes while one writer builds a +// private snapshot. It must be reset after removing or renaming staged paths. +type DestinationValidator struct { + directory string + checked map[string]bool + names map[string]map[string]bool +} + +// NewDestinationValidator creates a validator for an exclusively owned stage. +func NewDestinationValidator(directory string) *DestinationValidator { + return &DestinationValidator{directory: directory, checked: make(map[string]bool), names: make(map[string]map[string]bool)} +} + +// Validate rejects host aliases without rereading unchanged parent directories. +func (v *DestinationValidator) Validate(name string) error { + return validateDestination(v.directory, name, v.checked, v.names) +} + +// Record remembers a successfully created path after Validate has accepted it. +// The single snapshot writer must record every creation before the next check. +func (v *DestinationValidator) Record(name string) { + if name == "." || name == "" { + return + } + current := v.directory + for _, component := range strings.Split(filepath.FromSlash(name), string(filepath.Separator)) { + if names := v.names[current]; names != nil { + names[component] = true + } + current = filepath.Join(current, component) + v.checked[current] = true + } +} + +// Reset discards prefixes whose directories may have been removed or renamed. +func (v *DestinationValidator) Reset() { + clear(v.checked) + clear(v.names) +} + +func validateDestination(directory, name string, checked map[string]bool, directories map[string]map[string]bool) error { if name == "." || name == "" { return nil } current := directory for _, component := range strings.Split(filepath.FromSlash(name), string(filepath.Separator)) { requested := filepath.Join(current, component) + if checked[requested] { + current = requested + continue + } _, err := os.Lstat(requested) if errors.Is(err, os.ErrNotExist) { return nil @@ -27,17 +75,34 @@ func ValidateDestination(directory, name string) error { if err != nil { return fmt.Errorf("Lstat: %w", err) } - entries, err := os.ReadDir(current) - if err != nil { - return fmt.Errorf("ReadDir: %w", err) - } + names := directories[current] exact := false - for _, entry := range entries { - exact = exact || entry.Name() == component + if names == nil { + entries, err := os.ReadDir(current) + if err != nil { + return fmt.Errorf("ReadDir: %w", err) + } + if directories != nil { + names = make(map[string]bool, len(entries)) + } + for _, entry := range entries { + exact = exact || entry.Name() == component + if names != nil { + names[entry.Name()] = true + } + } + if directories != nil { + directories[current] = names + } + } else { + exact = names[component] } if !exact { return fmt.Errorf("%w: %q resolves to a different spelling below %q", ErrPathCollision, name, current) } + if checked != nil { + checked[requested] = true + } current = requested } return nil diff --git a/internal/adapters/gitsnapshot/destination_test.go b/internal/adapters/gitsnapshot/destination_test.go new file mode 100644 index 00000000..3de674e8 --- /dev/null +++ b/internal/adapters/gitsnapshot/destination_test.go @@ -0,0 +1,60 @@ +package gitsnapshot + +import ( + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDestinationValidatorRetainsHostCollisionChecks(t *testing.T) { + t.Parallel() + directory := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(directory, "Case", "api"), 0o755)) + validator := NewDestinationValidator(directory) + require.NoError(t, validator.Validate("Case/api/first.proto")) + require.NoError(t, validator.Validate("Case/api/second.proto")) + _, caseErr := os.Stat(filepath.Join(directory, "case")) + err := validator.Validate("case/api/other.proto") + if caseErr == nil { + require.ErrorIs(t, err, ErrPathCollision) + } else { + require.NoError(t, err) + } + require.NoError(t, os.RemoveAll(filepath.Join(directory, "Case"))) + require.NoError(t, os.MkdirAll(filepath.Join(directory, "case", "api"), 0o755)) + validator.Reset() + err = validator.Validate("Case/api/other.proto") + if caseErr == nil { + require.ErrorIs(t, err, ErrPathCollision) + } else { + require.NoError(t, err) + } +} + +func BenchmarkDestinationValidation(b *testing.B) { + directory := b.TempDir() + for i := range 1000 { + require.NoError(b, os.MkdirAll(filepath.Join(directory, "google", fmt.Sprintf("api%04d", i), "v1"), 0o755)) + } + for _, name := range []string{"stateless", "snapshot"} { + b.Run(name, func(b *testing.B) { + validator := NewDestinationValidator(directory) + b.ReportAllocs() + for i := range b.N { + path := fmt.Sprintf("google/api%04d/v1/file.proto", i%1000) + var err error + if name == "stateless" { + err = ValidateDestination(directory, path) + } else { + err = validator.Validate(path) + } + if err != nil { + b.Fatal(err) + } + } + }) + } +} diff --git a/internal/adapters/go_git/snapshot.go b/internal/adapters/go_git/snapshot.go index 61140eda..772c1974 100644 --- a/internal/adapters/go_git/snapshot.go +++ b/internal/adapters/go_git/snapshot.go @@ -79,7 +79,7 @@ func SnapshotRevision(ctx context.Context, directory, ref string) (_ *Snapshot, _ = snapshot.Close() } }() - view := sourceview.New(tree) + view := sourceview.NewImmutable(tree) err = view.Walk(ctx, ".", func(name string, resolved sourceview.Resolution, walkErr error) error { if walkErr != nil { if !snapshotInput(name) || snapshotAdditionalPolicy(name) { diff --git a/internal/core/core.go b/internal/core/core.go index 316b3830..531d18f6 100644 --- a/internal/core/core.go +++ b/internal/core/core.go @@ -26,6 +26,7 @@ type Core struct { importRoots []string importFileAllowed func(string) bool sourceFileOpen func(string) (io.ReadCloser, error) + sourceFilesWalk func(string, func(string) error) (bool, error) sourceBoundary string fileModules map[string]string managedMode ManagedModeConfig @@ -62,7 +63,11 @@ type Options struct { // files selected by dependency metadata. Nil permits every file. ImportFileAllowed func(string) bool // OpenSourceFile opens logical sources within their owning bounded roots. - OpenSourceFile func(string) (io.ReadCloser, error) + OpenSourceFile func(string) (io.ReadCloser, error) + // WalkSourceFiles optionally visits verified immutable proto inputs under an + // absolute import root. Paths passed to the visitor are absolute. Returning + // false selects the normal local filesystem walk instead. + WalkSourceFiles func(string, func(string) error) (bool, error) FileModules map[string]string CurrentProjectGitWalker CurrentProjectGitWalker BreakingCheckConfig BreakingCheckConfig @@ -88,6 +93,7 @@ func New(options Options) *Core { importRoots: slices.Clone(options.ImportRoots), importFileAllowed: options.ImportFileAllowed, sourceFileOpen: options.OpenSourceFile, + sourceFilesWalk: options.WalkSourceFiles, fileModules: maps.Clone(options.FileModules), currentProjectGitWalker: options.CurrentProjectGitWalker, breakingCheckConfig: options.BreakingCheckConfig, diff --git a/internal/core/generate.go b/internal/core/generate.go index 5969d249..cda73678 100644 --- a/internal/core/generate.go +++ b/internal/core/generate.go @@ -25,7 +25,6 @@ import ( pluginexecutor "github.com/easyp-tech/easyp/internal/adapters/plugin" "github.com/easyp-tech/easyp/internal/core/path_helpers" - "github.com/easyp-tech/easyp/internal/fs/fs" "github.com/easyp-tech/easyp/internal/sourceview" "github.com/easyp-tech/easyp/internal/version" ) @@ -85,14 +84,15 @@ func (c *Core) prepareGeneration(ctx context.Context, root string, selected []st found := make(map[string]bool) for _, inputFilesDir := range c.inputs.InputFilesDir { - searchPath := filepath.Join(inputFilesDir.Root, inputFilesDir.Path) - fsWalker := fs.NewFSWalker(root, searchPath) importRoot := filepath.Join(root, inputFilesDir.Root) if !slices.Contains(imports, importRoot) { imports = append(imports, importRoot) } - err := fsWalker.WalkDir(func(walkPath string, err error) error { + err := c.walkGenerationInputs(root, inputFilesDir, func(walkPath string, err error, indexed bool) error { + if err := ctx.Err(); err != nil { + return err + } addedFile := stripPrefix(walkPath, inputFilesDir.Root) if selected != nil && filepath.Ext(walkPath) == ".proto" && !selection[addedFile] { return nil @@ -108,7 +108,7 @@ func (c *Core) prepareGeneration(ctx context.Context, root string, selected []st return err case ctx.Err() != nil: return ctx.Err() - case path_helpers.ShouldSkipV1SourceDir(importRoot, filepath.Join(root, walkPath)): + case !indexed && path_helpers.ShouldSkipV1SourceDir(importRoot, filepath.Join(root, walkPath)): return stdfs.SkipDir case filepath.Ext(walkPath) != ".proto": return nil @@ -130,7 +130,7 @@ func (c *Core) prepareGeneration(ctx context.Context, root string, selected []st return nil }) if err != nil { - return nil, fmt.Errorf("WalkDir: %w", err) + return nil, fmt.Errorf("walkGenerationInputs: %w", err) } } diff --git a/internal/core/generate_inputs.go b/internal/core/generate_inputs.go new file mode 100644 index 00000000..33701353 --- /dev/null +++ b/internal/core/generate_inputs.go @@ -0,0 +1,40 @@ +package core + +import ( + "fmt" + "path/filepath" + + "github.com/easyp-tech/easyp/internal/fs/fs" +) + +func (c *Core) walkGenerationInputs(root string, input InputFilesDir, visit func(string, error, bool) error) error { + searchPath := filepath.Join(input.Root, input.Path) + if c.sourceFilesWalk != nil { + indexed, err := c.sourceFilesWalk(filepath.Join(root, input.Root), func(path string) error { + within, err := filepath.Rel(filepath.Join(root, searchPath), path) + if err != nil { + return fmt.Errorf("Rel: %w", err) + } + if !filepath.IsLocal(within) { + return nil + } + relative, err := filepath.Rel(root, path) + if err != nil { + return fmt.Errorf("Rel: %w", err) + } + if !filepath.IsLocal(relative) { + return fmt.Errorf("indexed source %q is outside the module", path) + } + return visit(filepath.ToSlash(relative), nil, true) + }) + if err != nil { + return fmt.Errorf("WalkSourceFiles: %w", err) + } + if indexed { + return nil + } + } + return fs.NewFSWalker(root, searchPath).WalkDir(func(path string, err error) error { + return visit(path, err, false) + }) +} diff --git a/internal/core/generate_inputs_test.go b/internal/core/generate_inputs_test.go new file mode 100644 index 00000000..e09e6a71 --- /dev/null +++ b/internal/core/generate_inputs_test.go @@ -0,0 +1,86 @@ +package core + +import ( + "io" + "os" + "path/filepath" + "strings" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/easyp-tech/easyp/internal/logger" +) + +func TestPrepareGenerationUsesVerifiedInventory(t *testing.T) { + t.Parallel() + root := t.TempDir() + contents := map[string]string{ + filepath.Join(root, "proto/api/item.proto"): "syntax = \"proto3\"; package api; import \"shared.proto\"; message Item { shared.Shared value = 1; }", + filepath.Join(root, "proto/shared.proto"): "syntax = \"proto3\"; package shared; message Shared {}", + filepath.Join(root, "proto/api/unused.proto"): "invalid unselected protobuf", + } + var unusedChecks atomic.Int64 + walks := 0 + app := New(Options{ + Logger: logger.NewNop(), + Inputs: Inputs{InputFilesDir: []InputFilesDir{{Root: "proto", Path: "api"}}}, + WalkSourceFiles: func(importRoot string, visit func(string) error) (bool, error) { + assert.Equal(t, filepath.Join(root, "proto"), importRoot) + walks++ + for _, name := range []string{"api/item.proto", "api/unused.proto", "shared.proto"} { + if err := visit(filepath.Join(importRoot, name)); err != nil { + return true, err + } + } + return true, nil + }, + ImportFileAllowed: func(path string) bool { + if filepath.Base(path) == "unused.proto" { + unusedChecks.Add(1) + } + _, ok := contents[filepath.Clean(path)] + return ok + }, + OpenSourceFile: func(path string) (io.ReadCloser, error) { + body, ok := contents[filepath.Clean(path)] + if !ok { + return nil, os.ErrNotExist + } + return io.NopCloser(strings.NewReader(body)), nil + }, + }) + // No source directories exist on disk. Both input discovery and imports + // must use the configured snapshot inventory and source accessor. + plan, err := app.PrepareGenerationFiles(t.Context(), root, []string{"api/item.proto"}) + require.NoError(t, err) + assert.Equal(t, 1, walks) + assert.Zero(t, unusedChecks.Load(), "selection runs before file metadata or membership checks") + assert.ElementsMatch(t, []string{"api/item.proto", "shared.proto"}, descriptorNames(plan.DescriptorSet(true).File)) + assert.Equal(t, []string{"api/item.proto"}, descriptorNames(plan.DescriptorSet(false).File)) + _, err = app.PrepareGenerationFiles(t.Context(), root, []string{"shared.proto"}) + require.ErrorContains(t, err, "outside the module inputs") +} + +func TestPrepareGenerationWithoutInventoryKeepsLocalSourcesFresh(t *testing.T) { + t.Parallel() + root := t.TempDir() + writeGenerationSources(t, root, map[string]string{"first.proto": "syntax = \"proto3\"; message First {}"}) + app := New(Options{ + Logger: logger.NewNop(), + Inputs: Inputs{InputFilesDir: []InputFilesDir{{Root: ".", Path: "."}}}, + WalkSourceFiles: func(string, func(string) error) (bool, error) { + return false, nil + }, + }) + first, err := app.PrepareGeneration(t.Context(), root) + require.NoError(t, err) + assert.Equal(t, []string{"first.proto"}, descriptorNames(first.DescriptorSet(false).File)) + require.NoError(t, os.Remove(filepath.Join(root, "first.proto"))) + writeGenerationSources(t, root, map[string]string{"second.proto": "syntax = \"proto3\"; message Second {}"}) + second, err := app.PrepareGeneration(t.Context(), root) + require.NoError(t, err) + assert.Equal(t, []string{"second.proto"}, descriptorNames(second.DescriptorSet(false).File)) +} diff --git a/internal/generation/descriptor_set.go b/internal/generation/descriptor_set.go index 9893414c..98215f03 100644 --- a/internal/generation/descriptor_set.go +++ b/internal/generation/descriptor_set.go @@ -99,7 +99,7 @@ func prepareDescriptorTargets(ctx context.Context, log logger.Logger, cache modu } } label := fmt.Sprintf("project %q, module %q (%s)", relativeDescriptorPath(request.WorkDir, filepath.Dir(target.configPath)), selected.module.Name, relativeDescriptorPath(request.WorkDir, selected.directory)) - app, err := prepareV1ModuleCore(log, request, target.configPath, selected.directory, target.config, selected.module, selected.dependencies) + app, owners, err := prepareV1ModuleCore(log, request, target.configPath, selected.directory, target.config, selected.module, selected.dependencies, selected.sources) if err != nil { return nil, fmt.Errorf("prepareV1ModuleCore for %s: %w", label, err) } @@ -116,15 +116,6 @@ func prepareDescriptorTargets(ctx context.Context, log logger.Logger, cache modu if _, err := protodesc.NewFiles(full); err != nil { return nil, fmt.Errorf("validate descriptor set for %s: %w", label, err) } - roots, err := modules.ModuleSources(selected.directory, selected.module) - if err != nil { - return nil, fmt.Errorf("ModuleSources: %w", err) - } - roots = append(roots, selected.dependencies...) - owners, err := roots.FileModules() - if err != nil { - return nil, fmt.Errorf("FileModules: %w", err) - } versions := descriptorVersions(request, selected) prepared = append(prepared, preparedDescriptorTarget{target: target, selected: selected, plan: plan, full: full, owners: owners, versions: versions, label: label}) } diff --git a/internal/generation/frozen.go b/internal/generation/frozen.go index a5882bc6..3aeb9f88 100644 --- a/internal/generation/frozen.go +++ b/internal/generation/frozen.go @@ -50,13 +50,15 @@ func resolveFrozenGenerationModule(ctx context.Context, cache modules.Cache, rep if err != nil { return v1GenerationModule{}, fmt.Errorf("Cached: %w", err) } - var others modules.SourceRoots + var sources, others modules.SourceRoots for _, root := range consumer.dependencies { if root.Module != selection.source { others = append(others, root) + } else { + sources = append(sources, root) } } - return v1GenerationModule{directory: directory, resolutionDir: consumerDir, module: module, dependencies: others}, nil + return v1GenerationModule{directory: directory, resolutionDir: consumerDir, module: module, sources: sources, dependencies: others}, nil } localDir, err := findV1LocalModuleByName(repoRoot, selection.source) if err != nil { diff --git a/internal/generation/module.go b/internal/generation/module.go index 2c554562..794b1d86 100644 --- a/internal/generation/module.go +++ b/internal/generation/module.go @@ -11,7 +11,7 @@ import ( ) func generateV1ModuleWithRoots(ctx context.Context, log logger.Logger, request Request, configPath, moduleDir string, gen v1.Generate, module v1.Module, importRoots modules.SourceRoots) error { - app, err := prepareV1ModuleCore(log, request, configPath, moduleDir, gen, module, importRoots) + app, _, err := prepareV1ModuleCore(log, request, configPath, moduleDir, gen, module, importRoots, nil) if err != nil { return fmt.Errorf("prepareV1ModuleCore: %w", err) } @@ -21,31 +21,34 @@ func generateV1ModuleWithRoots(ctx context.Context, log logger.Logger, request R return nil } -func prepareV1ModuleCore(log logger.Logger, request Request, configPath, moduleDir string, gen v1.Generate, module v1.Module, importRoots modules.SourceRoots) (*core.Core, error) { - sources, err := modules.ModuleSources(moduleDir, module) - if err != nil { - return nil, fmt.Errorf("ModuleSources: %w", err) +func prepareV1ModuleCore(log logger.Logger, request Request, configPath, moduleDir string, gen v1.Generate, module v1.Module, importRoots, sources modules.SourceRoots) (*core.Core, map[string]string, error) { + if sources == nil { + var err error + sources, err = modules.ModuleSources(moduleDir, module) + if err != nil { + return nil, nil, fmt.Errorf("ModuleSources: %w", err) + } } allSources := append(append(modules.SourceRoots(nil), sources...), importRoots...) - if err := modules.CheckSourceCollisions(allSources); err != nil { - return nil, fmt.Errorf("CheckSourceCollisions: %w", err) + // The consumer retains ownership when the same physical root is also + // present in its dependency graph. Indexing also validates every import. + indexRoots := append(append(modules.SourceRoots(nil), importRoots...), sources...) + owners, err := indexRoots.IndexFileModules() + if err != nil { + return nil, nil, fmt.Errorf("IndexFileModules: %w", err) } options, err := prepareV1GeneratorConfig(configPath, moduleDir, gen, module) if err != nil { - return nil, fmt.Errorf("prepareV1GeneratorConfig: %w", err) + return nil, nil, fmt.Errorf("prepareV1GeneratorConfig: %w", err) } options.Logger = log options.PluginWorkDir = request.WorkDir options.ImportRoots = importRoots.Paths() options.ImportFileAllowed = allSources.FileAllowed() options.OpenSourceFile = allSources.OpenSourceFile + options.WalkSourceFiles = sources.WalkSnapshotFiles if options.ManagedModeConfig.Enabled || options.ManagedModeConfig.GoPackageOnly { - roots := append(modules.SourceRoots(nil), importRoots...) - roots = append(roots, sources...) - options.FileModules, err = roots.FileModules() - if err != nil { - return nil, fmt.Errorf("FileModules: %w", err) - } + options.FileModules = owners } - return core.New(options), nil + return core.New(options), owners, nil } diff --git a/internal/generation/packages.go b/internal/generation/packages.go index a7150071..0d6fcaae 100644 --- a/internal/generation/packages.go +++ b/internal/generation/packages.go @@ -56,9 +56,13 @@ func (s *sourceSelectorMatches) validateMatches(section string) error { // names without compiling unrelated files. Package declarations are read only // when needed and after the path filter; required imports are compiled later. func selectedSourceFiles(ctx context.Context, selected v1GenerationModule, project, module *sourceSelectorMatches) ([]string, error) { - roots, err := modules.ModuleSources(selected.directory, selected.module) - if err != nil { - return nil, fmt.Errorf("ModuleSources: %w", err) + roots := selected.sources + if roots == nil { + var err error + roots, err = modules.ModuleSources(selected.directory, selected.module) + if err != nil { + return nil, fmt.Errorf("ModuleSources: %w", err) + } } allRoots := append(append(modules.SourceRoots(nil), roots...), selected.dependencies...) seen := make(map[string]bool) diff --git a/internal/generation/selection.go b/internal/generation/selection.go index 95fdda75..a7e95e8a 100644 --- a/internal/generation/selection.go +++ b/internal/generation/selection.go @@ -91,6 +91,7 @@ type v1GenerationModule struct { resolutionDir string directory string module v1.Module + sources modules.SourceRoots dependencies modules.SourceRoots } @@ -134,13 +135,15 @@ func resolveV1GenerationModule(ctx context.Context, cache modules.Cache, repoRoo return v1GenerationModule{}, fmt.Errorf("EnsureEffectiveGraph: %w", err) } if selected, ok := graph.Modules[selection.source]; ok { - var others modules.SourceRoots + var sources, others modules.SourceRoots for _, root := range graph.Sources { if root.Module != selection.source { others = append(others, root) + } else { + sources = append(sources, root) } } - return v1GenerationModule{directory: selected.Directory, resolutionDir: consumerDir, module: selected.Module, dependencies: others}, nil + return v1GenerationModule{directory: selected.Directory, resolutionDir: consumerDir, module: selected.Module, sources: sources, dependencies: others}, nil } for _, replacement := range consumer.Replaces { if replacement.Module == selection.source { @@ -180,12 +183,15 @@ func resolveV1GenerationModule(ctx context.Context, cache modules.Cache, repoRoo } otherRoots := make(modules.SourceRoots, 0, len(dependencyRoots)) + var sources modules.SourceRoots for _, root := range dependencyRoots { if root.Module != selection.source { otherRoots = append(otherRoots, root) + } else { + sources = append(sources, root) } } - return v1GenerationModule{directory: moduleDir, resolutionDir: consumerDir, module: module, dependencies: otherRoots}, nil + return v1GenerationModule{directory: moduleDir, resolutionDir: consumerDir, module: module, sources: sources, dependencies: otherRoots}, nil } func readV1GenerationModule(ctx context.Context, cache modules.Cache, directory string) (v1GenerationModule, error) { diff --git a/internal/generation/source_index_test.go b/internal/generation/source_index_test.go new file mode 100644 index 00000000..0313f2c0 --- /dev/null +++ b/internal/generation/source_index_test.go @@ -0,0 +1,82 @@ +package generation + +import ( + "context" + "fmt" + "path/filepath" + "slices" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" + "github.com/easyp-tech/easyp/internal/logger" +) + +func TestGenerationSelectsIndexedDependencyInputs(t *testing.T) { + t.Parallel() + root, snapshot := t.TempDir(), t.TempDir() + const source = "example.test/dependency" + files := map[string]string{ + v1.ModuleFile: "module " + source + "\nroots proto\n", + "proto/api/item.proto": "syntax = \"proto3\"; package api; import \"shared.proto\"; message Item { shared.Shared value = 1; }", + "proto/api/unused.proto": "invalid unselected protobuf", + "proto/shared.proto": "syntax = \"proto3\"; package shared; message Shared {}", + "proto/nested/protobuf.mod": "module example.test/nested\n", + "proto/nested/ignored.proto": "invalid nested module protobuf", + } + var names []string + for name, body := range files { + writeV1GenerateFixture(t, snapshot, name, body) + names = append(names, name) + } + slices.Sort(names) + writeV1GenerateFixture(t, root, v1.ModuleFile, "module example.test/consumer\nrequire "+source+" v1.0.0\n") + writeV1GenerateFixture(t, root, v1.GenerateFile, `version: v1 +generate: + modules: + - module: example.test/dependency + paths: [proto/api/item.proto] + managed: + enabled: true + override: + - module: example.test/dependency + file_option: go_package_prefix + value: example.test/generated +`) + entry := v1.LockedModule{Source: source, Version: "v1.0.0", Commit: strings.Repeat("a", 40), Hash: "h1:" + strings.Repeat("A", 43) + "="} + require.NoError(t, writeTestLock(root, v1.Lock{Version: 1, Modules: []v1.LockedModule{entry}})) + cache := &indexedGenerationCache{directory: snapshot, module: v1.Module{Name: source, Roots: []string{"proto"}}, files: names} + output := filepath.Join(root, "selected.pb") + err := Run(t.Context(), logger.NewNop(), cache, Request{WorkDir: root, WorkspaceRoot: root, DescriptorSetOut: output, IncludeImports: true}) + require.NoError(t, err) + assert.Equal(t, 1, cache.inventoryReads) + descriptors := readDescriptorSet(t, output) + assert.ElementsMatch(t, []string{"api/item.proto", "shared.proto"}, descriptorNames(descriptors)) + for _, file := range descriptors.File { + assert.Contains(t, file.GetOptions().GetGoPackage(), "example.test/generated") + } +} + +type indexedGenerationCache struct { + directory string + module v1.Module + files []string + inventoryReads int +} + +func (c *indexedGenerationCache) Install(context.Context, v1.Lock) error { return nil } + +func (c *indexedGenerationCache) Cached(entry v1.LockedModule) (string, v1.Module, error) { + if entry.Source != c.module.Name { + return "", v1.Module{}, fmt.Errorf("unknown cached module %s", entry.Source) + } + return c.directory, c.module, nil +} + +func (c *indexedGenerationCache) SourceFiles(v1.LockedModule) ([]string, error) { + c.inventoryReads++ + return slices.Clone(c.files), nil +} diff --git a/internal/modules/collisions.go b/internal/modules/collisions.go index 2d389081..ccbfce11 100644 --- a/internal/modules/collisions.go +++ b/internal/modules/collisions.go @@ -7,27 +7,37 @@ import ( // CheckSourceCollisions respects each module's source selection when comparing import names. func CheckSourceCollisions(roots SourceRoots) error { + _, err := roots.indexFiles(true) + return err +} + +func (roots SourceRoots) indexFiles(checkCollisions bool) (map[string]string, error) { seen := make(map[string]string) + owners := make(map[string]string) allowed := roots.FileAllowed() for _, root := range roots { err := root.walk(allowed, true, roots.ownsSelectedPhysical, func(path string) error { - if allowed != nil && !allowed(path) { - return nil - } importPath, err := filepath.Rel(root.Path, path) if err != nil { return err } importPath = filepath.ToSlash(importPath) - if previous, ok := seen[importPath]; ok && previous != path { + if previous, ok := seen[importPath]; checkCollisions && ok && previous != path { return fmt.Errorf("duplicate import path %q: %s and %s", importPath, previous, path) } seen[importPath] = path + owners[importPath] = root.Module return nil }) if err != nil { - return err + return nil, err } } - return nil + return owners, nil +} + +// IndexFileModules checks import collisions and records module ownership in a +// single source traversal. The returned map can be shared by generation stages. +func (roots SourceRoots) IndexFileModules() (map[string]string, error) { + return roots.indexFiles(true) } diff --git a/internal/modules/effective_graph.go b/internal/modules/effective_graph.go index 79635335..78c5a14e 100644 --- a/internal/modules/effective_graph.go +++ b/internal/modules/effective_graph.go @@ -63,8 +63,14 @@ func ensureEffectiveGraphWithRoots(ctx context.Context, root string, module v1.M if err != nil { return graph, fmt.Errorf("finalizeEffectiveImportRoots: %w", err) } - appendModule := func(name string, entry EffectiveModule) error { - roots, err := ModuleSources(entry.Directory, entry.Module) + appendModule := func(name string, entry EffectiveModule, locked *v1.LockedModule) error { + var roots SourceRoots + var err error + if locked == nil { + roots, err = ModuleSources(entry.Directory, entry.Module) + } else { + roots, err = cachedModuleSources(entry.Directory, entry.Module, *locked, cache) + } if err != nil { return fmt.Errorf("ModuleSources: %s: %w", name, err) } @@ -73,7 +79,7 @@ func ensureEffectiveGraphWithRoots(ctx context.Context, root string, module v1.M return nil } for _, name := range pass.locals { - if err := appendModule(name, locals.modules[name]); err != nil { + if err := appendModule(name, locals.modules[name], nil); err != nil { return graph, err } } @@ -82,7 +88,7 @@ func ensureEffectiveGraphWithRoots(ctx context.Context, root string, module v1.M if err != nil { return graph, fmt.Errorf("installed: %w", err) } - if err := appendModule(entry.Source, selected); err != nil { + if err := appendModule(entry.Source, selected, &entry); err != nil { return graph, err } } diff --git a/internal/modules/frozen.go b/internal/modules/frozen.go index d1fcf12b..d765f80d 100644 --- a/internal/modules/frozen.go +++ b/internal/modules/frozen.go @@ -89,9 +89,9 @@ func frozenCachedSources(ctx context.Context, module v1.Module, lock v1.Lock, re if err != nil { return nil, fmt.Errorf("dependency %s: validateFrozenRequirements: %w", dependency.Name, err) } - dependencyRoots, err := ModuleSources(directory, dependency) + dependencyRoots, err := cachedModuleSources(directory, dependency, entry, repository) if err != nil { - return nil, fmt.Errorf("ModuleSources: %w", err) + return nil, fmt.Errorf("cachedModuleSources: %w", err) } sources[requirement.Module] = dependencyRoots queue = append(queue, dependency.Requires...) diff --git a/internal/modules/locked_sources.go b/internal/modules/locked_sources.go index 1115cb6f..2182a39a 100644 --- a/internal/modules/locked_sources.go +++ b/internal/modules/locked_sources.go @@ -102,9 +102,9 @@ func cachedSources(lock v1.Lock, replaced map[string]bool, repository Cache) (So if err := ValidateRequirements(unreplacedRequirements(dependency.Requires, replaced), lock); err != nil { return nil, fmt.Errorf("dependency %s: %w", dependency.Name, err) } - dependencyRoots, err := ModuleSources(installDir, dependency) + dependencyRoots, err := cachedModuleSources(installDir, dependency, entry, repository) if err != nil { - return nil, fmt.Errorf("ModuleSources: %w", err) + return nil, fmt.Errorf("cachedModuleSources: %w", err) } roots = append(roots, dependencyRoots...) } diff --git a/internal/modules/operations_test.go b/internal/modules/operations_test.go index 941e9d5e..1f8338ef 100644 --- a/internal/modules/operations_test.go +++ b/internal/modules/operations_test.go @@ -241,6 +241,9 @@ func (r *fakeRepository) Install(context.Context, v1.Lock) error { r.installed = true return r.installErr } +func (r *fakeRepository) VerifyCached(context.Context, v1.Lock) error { + return r.installErr +} func (r *fakeRepository) Cached(v1.LockedModule) (string, v1.Module, error) { return r.directory, r.module, r.cachedErr } diff --git a/internal/modules/source_index.go b/internal/modules/source_index.go new file mode 100644 index 00000000..6e0170c1 --- /dev/null +++ b/internal/modules/source_index.go @@ -0,0 +1,125 @@ +package modules + +import ( + "fmt" + "path/filepath" + "slices" + "strings" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" + "github.com/easyp-tech/easyp/internal/core/path_helpers" +) + +// snapshotFileSource exposes names captured while verifying installed contents. +// Names are relative to the installed module directory and include metadata. +// Local sources never implement or use this immutable snapshot capability. +type snapshotFileSource interface { + SourceFiles(v1.LockedModule) ([]string, error) +} + +type sourceFileIndex struct { + paths []string + names map[string]bool + physicalRoot string +} + +// WalkSnapshotFiles visits the verified regular proto inventory for importRoot. +// It returns false without visiting anything when that root is a local source +// or its cache did not supply an immutable inventory. +func (roots SourceRoots) WalkSnapshotFiles(importRoot string, visit func(string) error) (bool, error) { + for _, root := range roots { + if root.index == nil || filepath.Clean(root.Path) != filepath.Clean(importRoot) { + continue + } + for _, path := range root.index.paths { + if err := visit(path); err != nil { + return true, err + } + } + return true, nil + } + return false, nil +} + +func cachedModuleSources(directory string, module v1.Module, entry v1.LockedModule, repository Cache) (SourceRoots, error) { + roots, err := ModuleSources(directory, module) + if err != nil { + return nil, fmt.Errorf("ModuleSources: %w", err) + } + indexed, ok := repository.(snapshotFileSource) + if !ok { + return roots, nil + } + files, err := indexed.SourceFiles(entry) + if err != nil { + return nil, fmt.Errorf("SourceFiles: %w", err) + } + if files == nil { + return roots, nil + } + return indexSnapshotSources(directory, roots, files) +} + +func indexSnapshotSources(directory string, roots SourceRoots, files []string) (SourceRoots, error) { + boundaries := make(map[string]bool) + var protos []string + for _, name := range files { + name = filepath.FromSlash(name) + if !filepath.IsLocal(name) || filepath.Clean(name) != name || name == "." { + return nil, fmt.Errorf("invalid snapshot file %q", name) + } + path := filepath.Join(directory, name) + switch filepath.Base(name) { + case ".git", v1.ModuleFile, "buf.work.yaml", "buf.yaml": + boundaries[filepath.Dir(path)] = true + } + if filepath.Ext(name) == ".proto" { + protos = append(protos, path) + } + } + slices.SortFunc(protos, compareSourcePaths) + for index := range roots { + root := &roots[index] + catalog := &sourceFileIndex{names: make(map[string]bool), physicalRoot: physicalSourcePath(root.Path)} + for _, path := range protos { + if !sourcePathWithin(path, root.Path) || path_helpers.HiddenOrVendorSourcePath(root.Path, filepath.Dir(path)) || + (root.fileAllowed != nil && !root.fileAllowed(path)) { + continue + } + nested := false + for parent := filepath.Dir(path); parent != filepath.Clean(root.Path); parent = filepath.Dir(parent) { + if boundaries[parent] { + nested = true + break + } + } + if nested { + continue + } + name, err := filepath.Rel(root.Path, path) + if err != nil { + return nil, fmt.Errorf("Rel: %w", err) + } + catalog.paths = append(catalog.paths, path) + catalog.names[name] = true + } + root.index = catalog + } + return roots, nil +} + +// Directory walks sort each component, so a/child.proto precedes a.proto. +// Preserve that order when an immutable inventory replaces filesystem walks. +func compareSourcePaths(first, second string) int { + for { + a, remainingA, nestedA := strings.Cut(first, string(filepath.Separator)) + b, remainingB, nestedB := strings.Cut(second, string(filepath.Separator)) + if compared := strings.Compare(a, b); compared != 0 { + return compared + } + if !nestedA || !nestedB { + return strings.Compare(remainingA, remainingB) + } + first, second = remainingA, remainingB + } +} diff --git a/internal/modules/source_index_test.go b/internal/modules/source_index_test.go new file mode 100644 index 00000000..fe634bdb --- /dev/null +++ b/internal/modules/source_index_test.go @@ -0,0 +1,137 @@ +package modules + +import ( + "os" + "path/filepath" + "slices" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + v1 "github.com/easyp-tech/easyp/internal/config/v1" +) + +func TestSnapshotSourceIndexPreservesSelection(t *testing.T) { + t.Parallel() + files := map[string]string{ + v1.ModuleFile: "module example.test/dependency\n", + "proto/api/model.proto": "syntax = \"proto3\";\n", + "proto/api.proto": "syntax = \"proto3\";\n", + "proto/excluded/model.proto": "syntax = \"proto3\";\n", + "proto/.hidden.proto": "syntax = \"proto3\";\n", + "proto/.hidden/ignored.proto": "syntax = \"proto3\";\n", + "proto/easyp_vendor/ignored.proto": "syntax = \"proto3\";\n", + "proto/native/protobuf.mod": "module example.test/nested\n", + "proto/native/ignored.proto": "syntax = \"proto3\";\n", + "proto/buf/buf.yaml": "version: v1\n", + "proto/buf/ignored.proto": "syntax = \"proto3\";\n", + "proto/work/buf.work.yaml": "version: v1\ndirectories: [nested]\n", + "proto/work/nested/ignored.proto": "syntax = \"proto3\";\n", + "proto/repository/.git": "gitdir: /unused\n", + "proto/repository/ignored.proto": "syntax = \"proto3\";\n", + "other/shared.proto": "syntax = \"proto3\";\n", + } + tests := []struct { + name string + roots []string + filters []v1.ProtoFileFilter + }{ + {name: "default root", roots: []string{"."}}, + {name: "multiple roots", roots: []string{"proto", "other"}}, + {name: "explicit hidden root", roots: []string{"proto/.hidden"}}, + {name: "explicit nested module root", roots: []string{"proto/buf"}}, + {name: "filtered root", roots: []string{"proto"}, filters: []v1.ProtoFileFilter{{Root: "proto", Includes: []string{"proto/api", "proto/.hidden.proto", "proto/excluded"}, Excludes: []string{"proto/excluded"}}}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + directory := t.TempDir() + var names []string + for name, body := range files { + writeV1GenerateFixture(t, directory, name, body) + names = append(names, name) + } + slices.Sort(names) + module := v1.Module{Name: "example.test/dependency", Roots: tt.roots, ProtoFilters: tt.filters} + local, err := ModuleSources(directory, module) + require.NoError(t, err) + want, err := local.IndexFileModules() + require.NoError(t, err) + indexed, err := ModuleSources(directory, module) + require.NoError(t, err) + indexed, err = indexSnapshotSources(directory, indexed, names) + require.NoError(t, err) + got, err := indexed.IndexFileModules() + require.NoError(t, err) + assert.Equal(t, want, got) + for index, root := range local { + var localOrder, indexedOrder []string + require.NoError(t, root.Walk(func(path string) error { localOrder = append(localOrder, path); return nil })) + require.NoError(t, indexed[index].Walk(func(path string) error { indexedOrder = append(indexedOrder, path); return nil })) + assert.Equal(t, localOrder, indexedOrder, "snapshot traversal keeps directory-walk order") + } + localAllowed, indexedAllowed := local.FileAllowed(), indexed.FileAllowed() + for _, root := range local { + for _, name := range names { + path := filepath.Join(directory, filepath.FromSlash(name)) + if sourcePathWithin(path, root.Path) && filepath.Ext(path) == ".proto" { + assert.Equal(t, localAllowed(path), indexedAllowed(path), name) + } + } + } + }) + } +} + +func TestSnapshotSourceIndexReusesVerifiedInventory(t *testing.T) { + t.Parallel() + directory := t.TempDir() + writeV1GenerateFixture(t, directory, "api/one.proto", "syntax = \"proto3\";\n") + module := v1.Module{Name: "example.test/dependency", Roots: []string{"."}} + roots, err := ModuleSources(directory, module) + require.NoError(t, err) + roots, err = indexSnapshotSources(directory, roots, []string{"api/one.proto"}) + require.NoError(t, err) + // Make traversal impossible after indexing. Looking up ownership must use + // the inventory already supplied by the verified snapshot boundary. + require.NoError(t, os.Rename(filepath.Join(directory, "api"), filepath.Join(directory, "relocated"))) + for range 3 { + owners, err := roots.IndexFileModules() + require.NoError(t, err) + assert.Equal(t, map[string]string{"api/one.proto": module.Name}, owners) + } +} + +func TestLocalSourceIndexObservesChanges(t *testing.T) { + t.Parallel() + directory := t.TempDir() + writeV1GenerateFixture(t, directory, "one.proto", "syntax = \"proto3\";\n") + module := v1.Module{Name: "example.test/local", Roots: []string{"."}} + roots, err := ModuleSources(directory, module) + require.NoError(t, err) + before, err := roots.IndexFileModules() + require.NoError(t, err) + assert.Equal(t, map[string]string{"one.proto": module.Name}, before) + require.NoError(t, os.Remove(filepath.Join(directory, "one.proto"))) + writeV1GenerateFixture(t, directory, "two.proto", "syntax = \"proto3\";\n") + after, err := roots.IndexFileModules() + require.NoError(t, err) + assert.Equal(t, map[string]string{"two.proto": module.Name}, after) +} + +func TestSnapshotSourceIndexChecksUnusedCollisions(t *testing.T) { + t.Parallel() + var all SourceRoots + for _, module := range []string{"example.test/a", "example.test/b"} { + directory := t.TempDir() + writeV1GenerateFixture(t, directory, "unused.proto", "syntax = \"proto3\";\n") + roots, err := ModuleSources(directory, v1.Module{Name: module, Roots: []string{"."}}) + require.NoError(t, err) + roots, err = indexSnapshotSources(directory, roots, []string{"unused.proto"}) + require.NoError(t, err) + all = append(all, roots...) + } + _, err := all.IndexFileModules() + require.ErrorContains(t, err, "duplicate import path \"unused.proto\"") +} diff --git a/internal/modules/sources.go b/internal/modules/sources.go index 5e1f2c6c..70c8fb67 100644 --- a/internal/modules/sources.go +++ b/internal/modules/sources.go @@ -23,6 +23,7 @@ type SourceRoot struct { fileAllowed func(string) bool directoryAllowed func(string) bool directory string + index *sourceFileIndex } // SourceRoots preserves import precedence and module ownership. @@ -51,7 +52,19 @@ func (roots SourceRoots) WalkSelected(root SourceRoot, selected func(string) boo } func (root SourceRoot) walk(selected func(string) bool, namesOnly bool, targetAllowed func(string) bool, visit func(string) error) error { + if root.index != nil { + for _, path := range root.index.paths { + if selected != nil && !selected(path) { + continue + } + if err := visit(path); err != nil { + return err + } + } + return nil + } boundary := root.boundary() + physicalBoundary := physicalSourcePath(boundary) relative, err := filepath.Rel(boundary, root.Path) if err != nil { return fmt.Errorf("Rel: %w", err) @@ -74,13 +87,13 @@ func (root SourceRoot) walk(selected func(string) bool, namesOnly bool, targetAl if root.directoryAllowed != nil && resolved.Info != nil && resolved.Info.IsDir() && !root.directoryAllowed(path) { return fs.SkipDir } - if path_helpers.ShouldSkipV1SourceDir(root.Path, path) { + if resolved.Info != nil && resolved.Info.IsDir() && path_helpers.ShouldSkipV1SourceDir(root.Path, path) { return fs.SkipDir } if filepath.Ext(path) == ".proto" && selected != nil && !selected(path) { return nil } - if errors.Is(walkErr, sourceview.ErrNestedRepository) && targetAllowed != nil && targetAllowed(filepath.Join(physicalSourcePath(boundary), filepath.FromSlash(resolved.Path))) { + if errors.Is(walkErr, sourceview.ErrNestedRepository) && targetAllowed != nil && targetAllowed(filepath.Join(physicalBoundary, filepath.FromSlash(resolved.Path))) { walkErr = nil } if walkErr != nil && namesOnly && filepath.Ext(path) == ".proto" { @@ -92,12 +105,12 @@ func (root SourceRoot) walk(selected func(string) bool, namesOnly bool, targetAl } return walkErr } - physical := filepath.Join(physicalSourcePath(boundary), filepath.FromSlash(resolved.Path)) + physical := filepath.Join(physicalBoundary, filepath.FromSlash(resolved.Path)) if !resolved.Info.IsDir() && root.fileAllowed != nil && !root.fileAllowed(physical) { return nil } if resolved.Info.IsDir() { - if path_helpers.ShouldSkipV1SourceDir(filepath.Join(physicalSourcePath(boundary), filepath.FromSlash(physicalRoot.Path)), physical) { + if path_helpers.ShouldSkipV1SourceDir(filepath.Join(physicalBoundary, filepath.FromSlash(physicalRoot.Path)), physical) { return fs.SkipDir } return nil @@ -162,6 +175,10 @@ func (root SourceRoot) allows(path, importRoot string) bool { } func (root SourceRoot) allowsPath(path, importRoot string) bool { + if root.index != nil { + relative, err := filepath.Rel(importRoot, path) + return err == nil && root.index.names[relative] + } if root.fileAllowed != nil && !root.fileAllowed(path) { return false } @@ -188,8 +205,13 @@ func (roots SourceRoots) FileAllowed() func(string) bool { selections := make([]selection, 0, len(roots)) for _, root := range roots { selections = append(selections, selection{path: filepath.Clean(root.Path), root: root}) - canonical, err := filepath.EvalSymlinks(root.Path) - if err == nil && canonical != root.Path { + canonical := "" + if root.index != nil { + canonical = root.index.physicalRoot + } else if resolved, err := filepath.EvalSymlinks(root.Path); err == nil { + canonical = resolved + } + if canonical != "" && canonical != root.Path { selections = append(selections, selection{path: canonical, root: root}) } } @@ -207,7 +229,18 @@ func (roots SourceRoots) FileAllowed() func(string) bool { return !inside } return func(path string) bool { - return allowedPath(filepath.Clean(path)) && allowedPath(physicalSourcePath(path)) + path = filepath.Clean(path) + if !allowedPath(path) { + return false + } + for _, selection := range selections { + if selection.root.index != nil && sourcePathWithin(path, selection.path) { + // Installed snapshots materialize aliases as regular files. Their + // verified inventory cannot acquire new links during this run. + return true + } + } + return allowedPath(physicalSourcePath(path)) } } @@ -294,25 +327,7 @@ func moduleFileSelection(directory string, filters []v1.ProtoFileFilter, include // FileModules maps import paths to their owning modules for managed selectors. // Callers check import collisions before using this mapping. func (roots SourceRoots) FileModules() (map[string]string, error) { - modules := make(map[string]string) - allowed := roots.FileAllowed() - for _, root := range roots { - err := root.walk(allowed, true, roots.ownsSelectedPhysical, func(path string) error { - if allowed != nil && !allowed(path) { - return nil - } - relative, err := filepath.Rel(root.Path, path) - if err != nil { - return err - } - modules[filepath.ToSlash(relative)] = root.Module - return nil - }) - if err != nil { - return nil, err - } - } - return modules, nil + return roots.indexFiles(false) } // ModuleSources validates module roots and resolves them relative to its directory. @@ -344,7 +359,12 @@ func ModuleSources(directory string, module v1.Module) (SourceRoots, error) { func (roots SourceRoots) ownsSelectedPhysical(path string) bool { for _, root := range roots { - physicalRoot := physicalSourcePath(root.Path) + physicalRoot := "" + if root.index != nil { + physicalRoot = root.index.physicalRoot + } else { + physicalRoot = physicalSourcePath(root.Path) + } if sourcePathWithin(path, physicalRoot) && root.allowsPath(path, physicalRoot) { return true } diff --git a/internal/modules/tidy_input_verification.go b/internal/modules/tidy_input_verification.go index eb523f0a..86ec6bbf 100644 --- a/internal/modules/tidy_input_verification.go +++ b/internal/modules/tidy_input_verification.go @@ -3,8 +3,13 @@ package modules import ( "context" "fmt" + "io" + "io/fs" + "os" "path/filepath" + "golang.org/x/mod/sumdb/dirhash" + v1 "github.com/easyp-tech/easyp/internal/config/v1" ) @@ -111,12 +116,66 @@ func verifyTidyCache(ctx context.Context, lock v1.Lock, repository Repository) e return fmt.Errorf("VerifyCached: %w", err) } } else { - // The base Cache contract verifies installed bytes against the lock. - // Callers recheck snapshots after this potentially repairing operation. + // Install may reuse an immutable snapshot verified earlier in the + // command. Tidy requires a fresh content check before rewriting inputs, + // including when a wrapper hides the optional explicit verifier. err = repository.Install(ctx, lock) if err != nil { return fmt.Errorf("Install: %w", err) } + for _, entry := range lock.Modules { + directory, _, err := repository.Cached(entry) + if err != nil { + return fmt.Errorf("Cached: %w", err) + } + if err := verifyTidySnapshot(ctx, directory, entry); err != nil { + return fmt.Errorf("verifyTidySnapshot: %w", err) + } + } + } + return nil +} + +func verifyTidySnapshot(ctx context.Context, directory string, entry v1.LockedModule) error { + var files []string + err := filepath.WalkDir(directory, func(path string, item fs.DirEntry, walkErr error) error { + if walkErr != nil { + return walkErr + } + if err := ctx.Err(); err != nil { + return fmt.Errorf("Err: %w", err) + } + if item.IsDir() { + return nil + } + if !item.Type().IsRegular() { + return fmt.Errorf("non-regular cached path %q", path) + } + name, err := filepath.Rel(directory, path) + if err != nil { + return fmt.Errorf("Rel: %w", err) + } + files = append(files, filepath.ToSlash(name)) + return nil + }) + if err != nil { + return fmt.Errorf("WalkDir: %w", err) + } + actual, err := dirhash.Hash1(files, func(name string) (io.ReadCloser, error) { + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("Err: %w", err) + } + file, err := os.Open(filepath.Join(directory, filepath.FromSlash(name))) + if err != nil { + return nil, fmt.Errorf("Open: %w", err) + } + return file, nil + }) + if err != nil { + return fmt.Errorf("Hash1: %w", err) + } + if actual != entry.Hash { + return fmt.Errorf("cached %s@%s hash mismatch: got %s, want %s; keep protobuf.lock unchanged", entry.Source, entry.Commit, actual, entry.Hash) } return nil } diff --git a/internal/modules/versionless_update_test.go b/internal/modules/versionless_update_test.go index ca25fc09..6a616c47 100644 --- a/internal/modules/versionless_update_test.go +++ b/internal/modules/versionless_update_test.go @@ -203,6 +203,8 @@ func (r *movingHeadRepository) Fetch(_ context.Context, source, version string) func (r *movingHeadRepository) Install(context.Context, v1.Lock) error { return nil } +func (r *movingHeadRepository) VerifyCached(context.Context, v1.Lock) error { return nil } + func (r *movingHeadRepository) Cached(entry v1.LockedModule) (string, v1.Module, error) { switch entry.Source { case "example.com/A": diff --git a/internal/sourceview/immutable_walk_test.go b/internal/sourceview/immutable_walk_test.go new file mode 100644 index 00000000..730bfd02 --- /dev/null +++ b/internal/sourceview/immutable_walk_test.go @@ -0,0 +1,134 @@ +package sourceview + +import ( + "fmt" + "io/fs" + "strings" + "testing" + "testing/fstest" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestImmutableWalkReusesParentResolution(t *testing.T) { + t.Parallel() + for _, depth := range []int{1, 8, 24} { + t.Run(fmt.Sprintf("depth_%d", depth), func(t *testing.T) { + t.Parallel() + backend := &countedWalkFS{MapFS: make(fstest.MapFS)} + prefix := strings.Repeat("nested/", depth) + const count = 100 + for index := range count { + backend.MapFS[fmt.Sprintf("%sfile-%d.proto", prefix, index)] = &fstest.MapFile{Data: []byte("contract")} + } + visited := 0 + err := NewImmutable(backend).Walk(t.Context(), ".", func(_ string, _ Resolution, err error) error { + visited++ + return err + }) + require.NoError(t, err) + assert.Equal(t, count+depth+1, visited) + assert.Equal(t, 1, backend.lstats, "only the walk root requires path resolution") + assert.Equal(t, count+depth, backend.infos, "each entry supplies its own metadata once") + }) + } +} + +func TestImmutableWalkPreservesInheritedAliases(t *testing.T) { + t.Parallel() + view := NewImmutable(fstest.MapFS{ + "real/nested/a.proto": {Data: []byte("a")}, + "real/nested/b.proto": {Data: []byte("b")}, + "alias": {Mode: fs.ModeSymlink, Data: []byte("real")}, + "chain": {Mode: fs.ModeSymlink, Data: []byte("alias")}, + }) + var files []string + err := view.Walk(t.Context(), "chain", func(logical string, resolved Resolution, err error) error { + require.NoError(t, err) + require.Len(t, resolved.Links, 2) + assert.Equal(t, "chain", resolved.Links[0].Path) + assert.Equal(t, "alias", resolved.Links[1].Path) + if !resolved.Info.IsDir() { + files = append(files, logical+"="+resolved.Path) + } + return nil + }) + require.NoError(t, err) + assert.Equal(t, []string{"chain/nested/a.proto=real/nested/a.proto", "chain/nested/b.proto=real/nested/b.proto"}, files) +} + +func TestMutableWalkObservesSourceChanges(t *testing.T) { + t.Parallel() + backend := fstest.MapFS{"directory/first.proto": {Data: []byte("first")}} + view := New(backend) + var first []string + err := view.Walk(t.Context(), ".", func(logical string, _ Resolution, err error) error { + first = append(first, logical) + return err + }) + require.NoError(t, err) + delete(backend, "directory/first.proto") + backend["directory/second.proto"] = &fstest.MapFile{Data: []byte("second")} + var second []string + err = view.Walk(t.Context(), ".", func(logical string, _ Resolution, err error) error { + second = append(second, logical) + return err + }) + require.NoError(t, err) + assert.Equal(t, []string{".", "directory", "directory/first.proto"}, first) + assert.Equal(t, []string{".", "directory", "directory/second.proto"}, second) +} + +func TestImmutableWalkRejectsUnsupportedEntries(t *testing.T) { + t.Parallel() + tests := []struct { + name string + file string + mode fs.FileMode + }{ + {name: "drive path", file: "C:outside.proto"}, + {name: "backslash", file: "unsafe\\outside.proto"}, + {name: "named pipe", file: "fifo.proto", mode: fs.ModeNamedPipe}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + view := NewImmutable(fstest.MapFS{tt.file: {Mode: tt.mode}}) + err := view.Walk(t.Context(), ".", func(_ string, _ Resolution, err error) error { return err }) + require.ErrorIs(t, err, ErrUnsupported) + }) + } +} + +type countedWalkFS struct { + fstest.MapFS + lstats int + infos int +} + +func (f *countedWalkFS) Lstat(name string) (fs.FileInfo, error) { + f.lstats++ + return f.MapFS.Lstat(name) +} + +func (f *countedWalkFS) ReadDir(name string) ([]fs.DirEntry, error) { + entries, err := f.MapFS.ReadDir(name) + if err != nil { + return nil, err + } + for index, entry := range entries { + entries[index] = countedWalkEntry{DirEntry: entry, owner: f} + } + return entries, nil +} + +type countedWalkEntry struct { + fs.DirEntry + owner *countedWalkFS +} + +func (entry countedWalkEntry) Info() (fs.FileInfo, error) { + entry.owner.infos++ + return entry.DirEntry.Info() +} diff --git a/internal/sourceview/view.go b/internal/sourceview/view.go index 116f1b73..c9721187 100644 --- a/internal/sourceview/view.go +++ b/internal/sourceview/view.go @@ -31,6 +31,7 @@ var ( type View struct { fsys fs.FS absoluteRoots []string + immutable bool } // Resolution describes a target and the physical links followed to reach it. @@ -59,6 +60,13 @@ func New(fsys fs.FS) *View { return &View{fsys: fsys} } +// NewImmutable borrows an immutable filesystem, such as a pinned Git tree. +// Walk may reuse resolved parent paths and directory-entry metadata. The caller +// must keep the filesystem unchanged for the lifetime of the view. +func NewImmutable(fsys fs.FS) *View { + return &View{fsys: fsys, immutable: true} +} + // NewLocal borrows fsys and accepts absolute links beneath canonicalRoot. // canonicalRoot must be the absolute, canonical OS path of fsys's boundary. // The caller must canonicalize it before opening the bounded filesystem. @@ -282,12 +290,12 @@ func (v *View) WalkSelected(ctx context.Context, logicalRoot string, selectEntry } func (v *View) walk(ctx context.Context, logical string, selectEntry func(string, fs.DirEntry) bool, fn WalkFunc, active map[string]bool) error { - err := ctx.Err() - if err != nil { - return &fs.PathError{Op: "Walk", Path: logical, Err: err} - } resolved, resolveErr := v.Resolve(ctx, logical) - err = ctx.Err() + return v.walkResolved(ctx, logical, resolved, resolveErr, selectEntry, fn, active) +} + +func (v *View) walkResolved(ctx context.Context, logical string, resolved Resolution, resolveErr error, selectEntry func(string, fs.DirEntry) bool, fn WalkFunc, active map[string]bool) error { + err := ctx.Err() if err != nil { return &fs.PathError{Op: "Walk", Path: logical, Err: err} } @@ -335,7 +343,28 @@ func (v *View) walk(ctx context.Context, logical string, selectEntry func(string if selectEntry != nil && !selectEntry(child, entry) { continue } - err = v.walk(ctx, child, selectEntry, fn, active) + if !v.immutable || entry.Type()&fs.ModeSymlink != 0 { + err = v.walk(ctx, child, selectEntry, fn, active) + } else { + info, infoErr := entry.Info() + if infoErr == nil && info.Mode()&fs.ModeSymlink != 0 { + // The entry became a link after ReadDir. Resolve it through + // the full bounded path instead of trusting stale metadata. + err = v.walk(ctx, child, selectEntry, fn, active) + } else { + childResolution := Resolution{Path: joinName(resolved.Path, name), Info: info, Links: slices.Clone(resolved.Links)} + if infoErr == nil { + infoErr = validatePointer(child) + if infoErr == nil && !info.IsDir() && !info.Mode().IsRegular() { + infoErr = ErrUnsupported + } + } + if infoErr != nil { + infoErr = &fs.PathError{Op: "Walk", Path: child, Err: infoErr} + } + err = v.walkResolved(ctx, child, childResolution, infoErr, selectEntry, fn, active) + } + } } if errors.Is(err, fs.SkipDir) { return nil