diff --git a/tooling/extension-bindings/IMPLEMENTATION.md b/tooling/extension-bindings/IMPLEMENTATION.md index 4fed09e..14424de 100644 --- a/tooling/extension-bindings/IMPLEMENTATION.md +++ b/tooling/extension-bindings/IMPLEMENTATION.md @@ -19,8 +19,8 @@ publication are outside the enabled Go workflows. The binding implementation remains in `extensions`. The downstream common `rc` executable belongs to `rc-cli` and will expose this implementation through a -thin adapter. Language-specific profilers remain independently implemented and -released programs. +thin library adapter. Language-specific profilers remain independently +implemented and released; the Go profiler also exposes a library API. The following requirements apply throughout the pipeline: @@ -716,9 +716,22 @@ rc bindings check rc bindings update ``` -The executable is built from `orchestrator/cmd/rc`. The reusable command -boundary is `orchestrator.Run(context.Context, []string, io.Writer, io.Writer)`; -it accepts the `bindings` command group and returns errors to its caller. +The standalone executable is built from `orchestrator/cmd/rc` and remains +supported independently of the common CLI. Its argument parser is +`orchestrator.Run(context.Context, []string, io.Writer, io.Writer)`. +It accepts the `bindings` command group and delegates to the library boundary: + +```go +orchestrator.Execute(ctx, cwd, operation, options, stdout, stderr) +``` + +`Execute` accepts a context, an absolute working directory, an operation name, +typed `Options`, and non-nil output writers. It parses no command arguments and +does not change the process working directory. Initialize options using +`orchestrator.DefaultOptions()` to obtain the standalone native-tool defaults +from the current environment, then apply explicit overrides. Explicit empty +values are preserved. Relative input, output, override, and executable paths +resolve against the supplied directory; bare tool names retain PATH lookup. The executable supplies process arguments and streams and chooses the exit code. | Command | Behavior and output | @@ -792,21 +805,48 @@ NOT maintain a second binding generator or duplicate compatibility rules. ### 13.2 Binding adapter and distribution -The reusable Go `orchestrator.Run` boundary and standalone command interface -provide integration boundaries. An adapter may invoke a versioned Go API or an -explicitly selected standalone tooling artifact, while preserving the command -contract defined here. - -For an in-process adapter, the orchestrator, normalizer, and Go emitter modules -must be available as a resolvable, versioned dependency set. Relative `replace` -directives used for development inside `extensions` do not supply dependencies -to a downstream module. The schema bundle and required emitter resources must -also be located through an explicit installed tooling path. - -For a subprocess adapter, invoke the selected binding tool with an argument -vector, preserve the caller's directory and streams, and propagate cancellation -and failure. Select its executable by an explicit managed path; dispatch MUST -NOT recursively resolve the common `rc` command as its own backend. +The common CLI MUST import the Go orchestrator and call `Execute` with typed +options from native Cobra commands. It MUST NOT rebuild an argument vector for +`Run` or dispatch to a separate `rc-bindings` executable. Generated SDKs under +`bindings/` are workload dependencies; the generator library lives under +`tooling/extension-bindings/orchestrator`. + +The orchestrator, normalizer, and Go emitter modules must be available as a +resolvable, versioned dependency set. Relative `replace` directives used for +development inside `extensions` do not supply dependencies to a downstream +module. Installed schema and emitter resources must work outside a source +checkout. The common CLI integration will call exported Go normalization and +emission functions directly; Python tooling remains optional for Python targets. + +Standalone generation in `extensions` MUST remain functional. Its entry point, +native-tool options, and ability to select an independently installed Go +profiler executable MUST remain supported. `Execute` preserves the existing +executable-based tooling and artifact identities as the default mode. + +Library callers may set `Options.UseGoLibraries` to call `normalizer.Normalize` +and `goemitter.Emit` directly. `Normalizer` and `GoEmitter` must then be empty; +conflicting executable selections fail instead of being ignored. This option +does not change profiler selection: callers may use `LinkedGoProfiler` or retain +a standalone `GoProfiler` executable. Python emission remains external. + +In library mode, normalizer and Go emitter identities retain their component +names and versions, with the SHA-256 of the actual host executable (`rc` when +integrated). No module checksum is substituted for an executable digest. +Rebuilding the host changes those identities and the normalized model digest. +Artifact locks must pin that host digest with the applicable component name and +version; a lock for a separately built normalizer or emitter fails. These checks +run before generation creates work files. Verification and regeneration use +the selected execution mode and retain the existing schema and identity checks. +Cancellation is checked around direct calls; the component APIs cannot interrupt +an active normalization or emission call. Schema resources are still required +through tooling discovery; packaging them for the common CLI is separate work. + +Linked profiler support MUST be explicit and optional. Its identity must record +the actual linked module version and dependency checksum. Metadata schemas, +lock validation, and profiler schema consumers must support that identity before +new metadata is emitted. An executable SHA-256 and a module checksum identify +different artifacts; a linked identity MUST NOT satisfy an executable lock. +Explicit standalone profiler selection MUST retain executable identity checks. The CLI integration must retain actual executing-tool provenance and all lock checks. A separately installed CLI version is not a substitute for an emitter, @@ -828,13 +868,15 @@ rc profile go generate rc profile python ``` -The common CLI's default installation includes the independently packaged Go -profiler. Other languages are optional profiler add-ons. Each profiler owns its -source parser, language runtime requirements, flags, and semantic validation. +The common CLI links the independently versioned Go profiler as a Go module. +Native Cobra commands bind typed options and invoke its exported functions; +default and explicit Go routes share the same command constructors. Go help and +flags use Cobra, including `--dir` and `--out`. Other languages are optional +profiler add-ons with their own source parsers and runtime requirements. -After selecting the profiler, the CLI forwards its argument vector without -translating language-specific options. Backend help and output remain owned by -the profiler. Launcher diagnostics must not contaminate profile data on stdout. +Optional profiler commands forward their argument vectors without translating +language-specific options. Their backend help and output remain owned by the +profiler. Launcher diagnostics must not contaminate profile data on stdout. Interactive discovery may use managed installations, explicit configuration, and `PATH`, subject to protocol compatibility checks. Binding verification and diff --git a/tooling/extension-bindings/model/runtimeconditions.binding-release.schema.yaml b/tooling/extension-bindings/model/runtimeconditions.binding-release.schema.yaml index 8ab4fa8..b41921b 100644 --- a/tooling/extension-bindings/model/runtimeconditions.binding-release.schema.yaml +++ b/tooling/extension-bindings/model/runtimeconditions.binding-release.schema.yaml @@ -106,6 +106,25 @@ $defs: name: {$ref: '#/$defs/nonEmptyString'} version: {$ref: '#/$defs/nonEmptyString'} sha256: {$ref: '#/$defs/sha256'} + profilerIdentity: + oneOf: + - {$ref: '#/$defs/toolIdentity'} + - {$ref: '#/$defs/linkedGoProfilerIdentity'} + linkedGoProfilerIdentity: + description: Exact linked Go profiler dependency, distinct from an executable digest. + type: object + additionalProperties: false + required: [kind, name, version, module, moduleSum] + properties: + kind: {const: go-module} + name: {const: go-rc-profiler} + version: + type: string + pattern: '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(\.(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*)?(\+incompatible)?$' + module: {const: github.com/runtimeconditions/go-rc-profiler} + moduleSum: + type: string + pattern: '^h1:[A-Za-z0-9+/]{42}[AEIMQUYcgkosw048]=$' package: description: >- Matches the emitter target and runtimeconditions.bindings.yaml package. @@ -256,7 +275,7 @@ $defs: properties: mode: {const: test-fixture} fixtureAssembler: {$ref: '#/$defs/toolIdentity'} - profiler: {$ref: '#/$defs/toolIdentity'} + profiler: {$ref: '#/$defs/profilerIdentity'} productionProvenance: description: >- Phase 5 and later committed package assembly. Source directory and tag @@ -268,7 +287,7 @@ $defs: properties: mode: {const: production} orchestrator: {$ref: '#/$defs/toolIdentity'} - profiler: {$ref: '#/$defs/toolIdentity'} + profiler: {$ref: '#/$defs/profilerIdentity'} sourceDirectory: type: string pattern: '^bindings/[a-z0-9]+(?:-[a-z0-9]+)*/(go|python)$' diff --git a/tooling/extension-bindings/model/runtimeconditions.toolchain-lock.schema.yaml b/tooling/extension-bindings/model/runtimeconditions.toolchain-lock.schema.yaml index 072789a..cbc04cb 100644 --- a/tooling/extension-bindings/model/runtimeconditions.toolchain-lock.schema.yaml +++ b/tooling/extension-bindings/model/runtimeconditions.toolchain-lock.schema.yaml @@ -4,8 +4,10 @@ title: Runtime Conditions toolchain lock description: >- Development may use workspace tools. A released lock pins normalizer, orchestrator, emitters and independently installed profilers by version and - artifact SHA-256. Consumers must check that each selected language has both - an emitter and a profiler and that the actual artifacts match their digests. + artifact SHA-256. Go profiler pins may instead identify a linked module by + exact version and dependency checksum. Consumers must check that each selected + language has an emitter and profiler and that the selected identity form, + version and checksum match the lock without substituting another artifact kind. type: object additionalProperties: false required: [apiVersion, kind, status] @@ -22,7 +24,7 @@ properties: normalizer: {$ref: '#/$defs/toolIdentity'} orchestrator: {$ref: '#/$defs/toolIdentity'} emitters: {$ref: '#/$defs/languageTools'} - profilers: {$ref: '#/$defs/languageTools'} + profilers: {$ref: '#/$defs/profilerTools'} languages: type: object propertyNames: @@ -64,6 +66,33 @@ $defs: description: Optional retrieval URL for the exact checksummed released artifact. type: string pattern: '^https://[^\s]+$' + profilerIdentity: + oneOf: + - {$ref: '#/$defs/toolIdentity'} + - {$ref: '#/$defs/linkedGoProfilerIdentity'} + linkedGoProfilerIdentity: + description: Exact linked Go profiler dependency, distinct from an executable digest. + type: object + additionalProperties: false + required: [kind, name, version, module, moduleSum] + properties: + kind: {const: go-module} + name: {const: go-rc-profiler} + version: + type: string + pattern: '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(\.(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*)?(\+incompatible)?$' + module: {const: github.com/runtimeconditions/go-rc-profiler} + moduleSum: + type: string + pattern: '^h1:[A-Za-z0-9+/]{42}[AEIMQUYcgkosw048]=$' + profilerTools: + type: object + minProperties: 1 + propertyNames: + pattern: '^[a-z][a-z0-9-]*$' + properties: + go: {$ref: '#/$defs/profilerIdentity'} + additionalProperties: {$ref: '#/$defs/toolIdentity'} languageTools: type: object minProperties: 1 diff --git a/tooling/extension-bindings/orchestrator/catalog.go b/tooling/extension-bindings/orchestrator/catalog.go index 7ea54c1..326d419 100644 --- a/tooling/extension-bindings/orchestrator/catalog.go +++ b/tooling/extension-bindings/orchestrator/catalog.go @@ -55,10 +55,10 @@ type Toolchain struct { Kind string `json:"kind" yaml:"kind"` Status string `json:"status" yaml:"status"` Tools struct { - Normalizer Tool `json:"normalizer" yaml:"normalizer"` - Orchestrator Tool `json:"orchestrator" yaml:"orchestrator"` - Emitters map[string]Tool `json:"emitters" yaml:"emitters"` - Profilers map[string]Tool `json:"profilers" yaml:"profilers"` + Normalizer Tool `json:"normalizer" yaml:"normalizer"` + Orchestrator Tool `json:"orchestrator" yaml:"orchestrator"` + Emitters map[string]Tool `json:"emitters" yaml:"emitters"` + Profilers map[string]ProfilerIdentity `json:"profilers" yaml:"profilers"` } `json:"tools" yaml:"tools"` Languages map[string]LanguageToolchain `json:"languages" yaml:"languages"` Python struct { @@ -178,9 +178,13 @@ func DiscoverProject(cwd, packagesPath, toolchainPath, toolingDirectory string) return nil, fmt.Errorf("%s:%s: name is required by the %s native emitter", key, language, language) } if p.Toolchain.Status == "released" { - if p.Toolchain.Tools.Emitters[language].SHA256 == "" || p.Toolchain.Tools.Profilers[language].SHA256 == "" { + profiler, present := p.Toolchain.Tools.Profilers[language] + if p.Toolchain.Tools.Emitters[language].SHA256 == "" || !present { return nil, fmt.Errorf("%s: released lock lacks emitter or profiler", language) } + if err := profiler.Validate(); err != nil { + return nil, fmt.Errorf("%s: invalid released profiler lock: %w", language, err) + } native, ok := p.Toolchain.Languages[language] if !ok || native.LanguageVersion != target.LanguageVersion { return nil, fmt.Errorf("%s:%s: exact language version differs from released lock", key, language) diff --git a/tooling/extension-bindings/orchestrator/cli.go b/tooling/extension-bindings/orchestrator/cli.go index 6bbc941..30ca690 100644 --- a/tooling/extension-bindings/orchestrator/cli.go +++ b/tooling/extension-bindings/orchestrator/cli.go @@ -17,14 +17,35 @@ type repeated []string func (r *repeated) String() string { return strings.Join(*r, ",") } func (r *repeated) Set(value string) error { *r = append(*r, value); return nil } +// Options contains operation inputs. Use DefaultOptions for native tool defaults. type Options struct { Packages, ToolchainLock, Cache, WorkDir, ToolingDir string Output, Input, Report, ClosureOutput, DependencyLockOutput, DependencyLock string CoreSchema, Go, Python, GoEmitter, Normalizer, GoProfiler, PythonProfiler, PythonProfilerArtifact string Targets, ExtensionRoots, Overrides, CatalogRoots repeated + // LinkedGoProfiler opts into library identity; GoProfiler must be empty. + // The caller supplies verified build metadata. Standalone defaults are retained. + LinkedGoProfiler *ProfilerIdentity + // UseGoLibraries calls the linked normalizer and Go emitter directly. + // Normalizer and GoEmitter must be empty; provenance hashes the host executable. + UseGoLibraries bool } -// Run is the reusable command boundary; the rc executable only supplies streams. +var supportedOperations = map[string]bool{"resolve": true, "normalize": true, "generate": true, "verify": true, "package": true, "plan-release": true, "check": true, "update": true} + +// DefaultOptions returns standalone tool defaults from the current environment. +// Explicit overrides, including empty values, may be applied before Execute. +func DefaultOptions() Options { + return Options{ + Go: envOr("RC_BINDINGS_GO", "go"), Python: envOr("RC_BINDINGS_PYTHON", "python3"), + Normalizer: os.Getenv("RC_BINDINGS_NORMALIZER"), GoEmitter: os.Getenv("RC_BINDINGS_GO_EMITTER"), + GoProfiler: envOr("RC_GO_PROFILER_BIN", "go-rc-profiler"), + PythonProfiler: envOr("RC_PYTHON_PROFILER_BIN", "runtimeconditions-python-profiler"), + PythonProfilerArtifact: os.Getenv("RC_PYTHON_PROFILER_ARTIFACT"), + } +} + +// Run parses standalone command arguments and delegates execution to Execute. func Run(ctx context.Context, args []string, stdout, stderr io.Writer) error { if len(args) == 0 || args[0] == "--help" || args[0] == "-h" { fmt.Fprintln(stdout, "Usage: rc bindings [flags]") @@ -38,15 +59,14 @@ func Run(ctx context.Context, args []string, stdout, stderr io.Writer) error { return fmt.Errorf("usage: rc bindings [flags]") } name := args[1] - supported := map[string]bool{"resolve": true, "normalize": true, "generate": true, "verify": true, "package": true, "plan-release": true, "check": true, "update": true} if name == "--help" || name == "-h" { fmt.Fprintln(stdout, "Commands: resolve normalize generate verify package plan-release check update") return nil } - if !supported[name] { + if !supportedOperations[name] { return fmt.Errorf("unknown bindings command %q", name) } - var o Options + o := DefaultOptions() f := flag.NewFlagSet("rc bindings "+name, flag.ContinueOnError) f.SetOutput(stderr) f.StringVar(&o.Packages, "packages", "", "package catalog path") @@ -78,13 +98,13 @@ func Run(ctx context.Context, args []string, stdout, stderr io.Writer) error { f.StringVar(&o.Report, "report", "", "write the YAML verification summary here") } f.StringVar(&o.CoreSchema, "core-schema", "", "explicit core schema file for local development") - f.StringVar(&o.Go, "go", envOr("RC_BINDINGS_GO", "go"), "native Go command") - f.StringVar(&o.Python, "python", envOr("RC_BINDINGS_PYTHON", "python3"), "native Python interpreter") - f.StringVar(&o.Normalizer, "normalizer", os.Getenv("RC_BINDINGS_NORMALIZER"), "installed locked normalizer; development defaults to workspace build") - f.StringVar(&o.GoEmitter, "go-emitter", os.Getenv("RC_BINDINGS_GO_EMITTER"), "installed locked Go emitter; development defaults to workspace build") - f.StringVar(&o.GoProfiler, "go-profiler", envOr("RC_GO_PROFILER_BIN", "go-rc-profiler"), "independently installed Go profiler CLI") - f.StringVar(&o.PythonProfiler, "python-profiler", envOr("RC_PYTHON_PROFILER_BIN", "runtimeconditions-python-profiler"), "independently installed Python profiler CLI") - f.StringVar(&o.PythonProfilerArtifact, "python-profiler-artifact", os.Getenv("RC_PYTHON_PROFILER_ARTIFACT"), "installed Python profiler wheel for artifact identity verification") + f.StringVar(&o.Go, "go", o.Go, "native Go command") + f.StringVar(&o.Python, "python", o.Python, "native Python interpreter") + f.StringVar(&o.Normalizer, "normalizer", o.Normalizer, "installed locked normalizer; development defaults to workspace build") + f.StringVar(&o.GoEmitter, "go-emitter", o.GoEmitter, "installed locked Go emitter; development defaults to workspace build") + f.StringVar(&o.GoProfiler, "go-profiler", o.GoProfiler, "independently installed Go profiler CLI") + f.StringVar(&o.PythonProfiler, "python-profiler", o.PythonProfiler, "independently installed Python profiler CLI") + f.StringVar(&o.PythonProfilerArtifact, "python-profiler-artifact", o.PythonProfilerArtifact, "installed Python profiler wheel for artifact identity verification") if err := f.Parse(args[2:]); err != nil { if err == flag.ErrHelp { return nil @@ -94,213 +114,11 @@ func Run(ctx context.Context, args []string, stdout, stderr io.Writer) error { if f.NArg() != 0 { return fmt.Errorf("unexpected positional arguments: %s", strings.Join(f.Args(), " ")) } - if (name == "generate" || name == "package") && o.Output == "" { - return fmt.Errorf("%s requires --output", name) - } cwd, err := os.Getwd() if err != nil { return err } - for _, ptr := range []*string{&o.Cache, &o.WorkDir, &o.Output, &o.Input, &o.Report, &o.ClosureOutput, &o.DependencyLockOutput, &o.DependencyLock, &o.CoreSchema, &o.PythonProfilerArtifact} { - if *ptr != "" { - *ptr = absolute(cwd, *ptr) - } - } - for i, dir := range o.CatalogRoots { - o.CatalogRoots[i] = absolute(cwd, dir) - } - project, err := DiscoverProject(cwd, o.Packages, o.ToolchainLock, o.ToolingDir) - if err != nil { - return err - } - pipeline, err := NewPipeline(ctx, project, o) - if err != nil { - return err - } - defer pipeline.Close() - outputs := []string{o.Output, o.Report, o.ClosureOutput, o.DependencyLockOutput} - seenOutputs := map[string]bool{} - for _, path := range outputs { - if path == "" { - continue - } - if err = rejectSymlinkPath(path); err != nil { - return err - } - if within(filepath.Join(project.RepositoryRoot, "bindings"), path) { - return fmt.Errorf("explicit outputs must be outside committed bindings; use update to synchronize package source") - } - if seenOutputs[path] { - return fmt.Errorf("output paths must be different: %s", path) - } - seenOutputs[path] = true - if name != "generate" && name != "package" { - if _, statErr := os.Lstat(path); !os.IsNotExist(statErr) { - return fmt.Errorf("output must be a new file: %s", path) - } - } - } - plan, err := project.SelectTargets(o.Targets) - if err != nil { - return err - } - if len(o.ExtensionRoots) > 0 && name != "resolve" && name != "normalize" { - roots := map[string]bool{} - for _, root := range o.ExtensionRoots { - roots[root] = true - } - filtered := BuildPlan{Targets: []Target{}} - foundRoots := map[string]bool{} - for _, target := range plan.Targets { - if roots[target.Set.RootExtension] { - filtered.Targets = append(filtered.Targets, target) - foundRoots[target.Set.RootExtension] = true - } - } - for root := range foundRoots { - delete(roots, root) - } - if len(roots) != 0 { - return fmt.Errorf("extension roots require package delivery metadata in the selected catalog targets: %s", strings.Join(sortedKeys(roots), ", ")) - } - plan = filtered - } - switch name { - case "resolve", "normalize": - documents, err := pipeline.Resolve(plan) - if err != nil { - return err - } - for i, doc := range documents { - if i > 0 { - fmt.Fprintln(stdout, "---") - } - var data []byte - if name == "resolve" { - data, err = yaml.Marshal(doc.SemanticClosure()) - } else { - if err = pipeline.Normalize(doc); err == nil { - data, err = doc.ModelYAML() - } - } - if err != nil { - return err - } - if _, err = stdout.Write(data); err != nil { - return err - } - if name == "resolve" { - lock, err := doc.LockYAML() - if err != nil { - return err - } - fmt.Fprintln(stdout, "---") - if _, err = stdout.Write(lock); err != nil { - return err - } - } - } - if o.ClosureOutput != "" || o.DependencyLockOutput != "" || o.Output != "" { - if len(documents) != 1 { - return fmt.Errorf("file outputs require one extension root; use --extension-root or --target") - } - doc := documents[0] - if o.ClosureOutput != "" { - data, _ := yaml.Marshal(doc.SemanticClosure()) - if err = writeNewFile(o.ClosureOutput, data); err != nil { - return err - } - } - if o.Output != "" { - data, err := doc.ModelYAML() - if err != nil { - return err - } - if err = writeNewFile(o.Output, data); err != nil { - return err - } - } - if o.DependencyLockOutput != "" { - data, err := doc.LockYAML() - if err != nil { - return err - } - if err = writeNewFile(o.DependencyLockOutput, data); err != nil { - return err - } - } - } - return nil - case "generate": - if err = validateEmptyDirectory(o.Output); err != nil { - return err - } - build, err := pipeline.Generate(plan) - if err != nil { - return err - } - if err = prepareEmptyDirectory(o.Output); err != nil { - return err - } - return build.CopySelected(o.Output) - case "plan-release": - release, err := pipeline.PlanRelease(plan) - if err != nil { - return err - } - data, err := yaml.Marshal(release) - if err != nil { - return err - } - if _, err = stdout.Write(data); err != nil { - return err - } - if o.Output != "" { - return writeNewFile(o.Output, data) - } - return nil - case "verify", "check", "package", "update": - var build *Build - if name == "verify" { - build, err = pipeline.LoadBuild(plan, o.Input) - } else { - build, err = pipeline.Generate(plan) - } - if err != nil { - failure := VerificationSummary{Status: "failed", Targets: []TargetResult{}} - failure.Fail("build", err) - return writeSummary(name, o, failure, err, stdout, stderr) - } - summary, err := pipeline.Verify(build) - if err == nil && (name == "check" || name == "package") { - err = build.CompareCommitted() - if err != nil { - summary.Fail("committed-source", err) - } - } - if err == nil && name == "update" { - err = build.Update() - if err != nil { - summary.Fail("update", err) - } - } - if err == nil && name == "package" { - committed, loadErr := pipeline.LoadBuild(plan, "") - if loadErr != nil { - err = loadErr - } else { - _, err = pipeline.Verify(committed) - if err == nil { - err = pipeline.Package(committed, o.Output) - } - } - if err != nil { - summary.Fail("package", err) - } - } - return writeSummary(name, o, summary, err, stdout, stderr) - } - return fmt.Errorf("unhandled bindings command %s", name) + return Execute(ctx, cwd, name, o, stdout, stderr) } func writeSummary(name string, o Options, summary VerificationSummary, result error, stdout, stderr io.Writer) error { diff --git a/tooling/extension-bindings/orchestrator/go_libraries.go b/tooling/extension-bindings/orchestrator/go_libraries.go new file mode 100644 index 0000000..b39d828 --- /dev/null +++ b/tooling/extension-bindings/orchestrator/go_libraries.go @@ -0,0 +1,117 @@ +package orchestrator + +import ( + "crypto/sha256" + "fmt" + "io" + "os" + + goemitter "github.com/runtimeconditions/extensions/tooling/extension-bindings/emitters/go" + "github.com/runtimeconditions/extensions/tooling/extension-bindings/normalizer" +) + +// goLibraryTool identifies the artifact executing a linked component, rather +// than a separately built tool or a module checksum. Its path is never executed. +func (p *Pipeline) goLibraryTool(key string) (string, Tool, error) { + if err := p.Context.Err(); err != nil { + return "", Tool{}, err + } + var identity, expected Tool + switch key { + case "normalizer": + identity = Tool{Name: normalizer.NormalizerName, Version: normalizer.NormalizerVersion} + expected = p.Project.Toolchain.Tools.Normalizer + case "go-emitter": + identity = Tool{Name: goemitter.EmitterName, Version: goemitter.EmitterVersion} + expected = p.Project.Toolchain.Tools.Emitters["go"] + default: + return "", Tool{}, fmt.Errorf("unknown Go library component %s", key) + } + path, cached := p.tools[key] + if cached { + identity.SHA256 = p.identities[key].SHA256 + } else { + var err error + path, err = os.Executable() + if err != nil { + return "", Tool{}, err + } + artifact, err := os.Open(path) + if err != nil { + return "", Tool{}, err + } + digest := sha256.New() + _, err = io.Copy(digest, artifact) + closeErr := artifact.Close() + if err != nil { + return "", Tool{}, err + } + if closeErr != nil { + return "", Tool{}, closeErr + } + identity.SHA256 = fmt.Sprintf("%x", digest.Sum(nil)) + } + if p.Project.Toolchain.Status == "released" && expected.SHA256 == "" { + return "", Tool{}, fmt.Errorf("%s: released lock lacks host executable identity", key) + } + if expected.SHA256 != "" && (expected.Name != identity.Name || expected.Version != identity.Version || expected.SHA256 != identity.SHA256) { + return "", Tool{}, fmt.Errorf("%s host executable identity differs from toolchain lock", key) + } + if err := p.Context.Err(); err != nil { + return "", Tool{}, err + } + p.tools[key], p.identities[key] = path, identity + return path, identity, nil +} + +func (p *Pipeline) normalizeGoLibrary(doc *Document) error { + _, identity, err := p.tool("normalizer") + if err != nil { + return err + } + _, core, err := p.core(doc.CoreVersion) + if err != nil { + return err + } + if err := p.Context.Err(); err != nil { + return err + } + model, err := normalizer.Normalize(doc.Closure, doc.Lock, p.Project.Schemas, normalizer.NormalizeConfig{ + CoreProfileSchema: core, + Normalizer: normalizer.ToolIdentity{Name: identity.Name, Version: identity.Version, SHA256: identity.SHA256}, + }) + if err != nil { + return err + } + if err := p.Context.Err(); err != nil { + return err + } + doc.Model = model + return nil +} + +func (p *Pipeline) emitGoLibrary(bt *BuiltTarget) error { + if _, _, err := p.tool("go-emitter"); err != nil { + return err + } + modelPath, targetPath, err := p.emitterInputs(bt) + if err != nil { + return err + } + // Use the emitter's own strict loaders, matching its standalone entry point. + model, err := goemitter.LoadModel(modelPath) + if err != nil { + return err + } + target, err := goemitter.LoadPackageTarget(targetPath) + if err != nil { + return err + } + if err := p.Context.Err(); err != nil { + return err + } + if err := goemitter.Emit(model, target, bt.Directory); err != nil { + return err + } + return p.Context.Err() +} diff --git a/tooling/extension-bindings/orchestrator/go_libraries_test.go b/tooling/extension-bindings/orchestrator/go_libraries_test.go new file mode 100644 index 0000000..e94eff0 --- /dev/null +++ b/tooling/extension-bindings/orchestrator/go_libraries_test.go @@ -0,0 +1,238 @@ +package orchestrator + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/runtimeconditions/extensions/tooling/extension-bindings/normalizer" +) + +func goLibraryProject(t *testing.T) (*Project, Options) { + t.Helper() + p, o := profilerPipelineProject(t) + o.ToolingDir = t.TempDir() + if err := copyTree(filepath.Join(p.ToolingDirectory, "model"), filepath.Join(o.ToolingDir, "model")); err != nil { + t.Fatal(err) + } + p, err := DiscoverProject(p.Root, "", "", o.ToolingDir) + if err != nil { + t.Fatal(err) + } + _, linked := testProfilerIdentities(t) + o.UseGoLibraries, o.LinkedGoProfiler = true, &linked + return p, o +} + +func TestGoLibrariesGenerateAndVerifyWithoutToolExecutables(t *testing.T) { + standalonePath, standaloneIdentity := standaloneProfilerFixture(t) + for _, mode := range []string{"linked", "standalone"} { + t.Run(mode, func(t *testing.T) { + p, o := goLibraryProject(t) + if mode == "standalone" { + o.GoProfiler, o.LinkedGoProfiler = standalonePath, nil + p.Toolchain.Tools.Profilers = map[string]ProfilerIdentity{"go": standaloneIdentity} + } + goCommand := o.Go + o.Go = filepath.Join(p.Root, "no-build-command") + pipeline, err := NewPipeline(context.Background(), p, o) + if err != nil { + t.Fatal(err) + } + defer pipeline.Close() + plan, err := p.SelectTargets(nil) + if err != nil { + t.Fatal(err) + } + build, err := pipeline.Generate(plan) + if err != nil { + t.Fatal(err) + } + host, err := os.Executable() + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(host) + if err != nil { + t.Fatal(err) + } + digest := normalizer.SHA256Hex(data) + bt := build.Targets[0] + if bt.Document.Model.Metadata.Normalizer.SHA256 != digest || bt.PackageConfig["emitterSha256"] != digest { + t.Fatal("library provenance does not identify the executing host") + } + for _, directory := range []string{"tools", "normalizer-inputs"} { + if _, err := os.Stat(filepath.Join(pipeline.work, directory)); !os.IsNotExist(err) { + t.Fatalf("library mode created %s: %v", directory, err) + } + } + pipeline.Options.Go = goCommand // Native verification still requires Go. + loaded, err := pipeline.LoadBuild(plan, bt.Directory) + if err != nil { + t.Fatal(err) + } + if summary, err := pipeline.Verify(loaded); err != nil || summary.Status != "passed" { + t.Fatalf("verify library-generated bindings: %v %+v", err, summary) + } + }) + } +} + +func TestGoLibrariesMatchStandaloneGeneration(t *testing.T) { + p, o := profilerPipelineProject(t) + _, linked := testProfilerIdentities(t) + o.LinkedGoProfiler = &linked + var builds []*Build + for _, library := range []bool{false, true} { + o.UseGoLibraries = library + pipeline, err := NewPipeline(context.Background(), p, o) + if err != nil { + t.Fatal(err) + } + defer pipeline.Close() + plan, err := p.SelectTargets(nil) + if err != nil { + t.Fatal(err) + } + build, err := pipeline.Generate(plan) + if err != nil { + t.Fatal(err) + } + builds = append(builds, build) + } + standalone, library := builds[0].Targets[0], builds[1].Targets[0] + model := library.Document.Model + model.Metadata.Normalizer = standalone.Document.Model.Metadata.Normalizer + var err error + model.Metadata.SemanticSHA256, err = normalizer.ModelSemanticSHA256(model) + if err != nil { + t.Fatal(err) + } + left, err := normalizer.CanonicalModelYAML(model) + if err != nil { + t.Fatal(err) + } + right, err := standalone.Document.ModelYAML() + if err != nil || !bytes.Equal(left, right) { + t.Fatalf("normalization differs beyond executing-artifact provenance: %v", err) + } + leftTree, err := readTree(library.Directory) + if err != nil { + t.Fatal(err) + } + rightTree, err := readTree(standalone.Directory) + if err != nil { + t.Fatal(err) + } + for name, data := range leftTree { + if strings.HasSuffix(name, ".go") || name == "go.mod" { + data = bytes.ReplaceAll(data, []byte(library.Document.Model.Metadata.SemanticSHA256), []byte(model.Metadata.SemanticSHA256)) + if !bytes.Equal(data, rightTree[name]) { + t.Fatalf("native source differs: %s", name) + } + } + } +} + +func TestGoLibrariesRejectConflictsAndMismatchedLocksBeforeWork(t *testing.T) { + p, o := goLibraryProject(t) + pipeline, err := NewPipeline(context.Background(), p, o) + if err != nil { + t.Fatal(err) + } + defer pipeline.Close() + pins := pipeline.identities + o.WorkDir = filepath.Join(p.Root, "must-not-create") + for _, component := range []string{"normalizer", "go-emitter"} { + for _, field := range []string{"executable", "name", "version", "digest", "missing released pin"} { + t.Run(component+"/"+field, func(t *testing.T) { + project, options := *p, o + project.Toolchain = Toolchain{Status: "development"} + pin := pins[component] + switch field { + case "executable": + if component == "normalizer" { + options.Normalizer = "missing-normalizer" + } else { + options.GoEmitter = "missing-emitter" + } + case "name": + pin.Name = "different-component" + case "version": + pin.Version = "99.0.0" + case "digest": + pin.SHA256 = strings.Repeat("f", 64) + case "missing released pin": + project.Toolchain.Status, pin = "released", Tool{} + } + project.Toolchain.Tools.Normalizer = pins["normalizer"] + project.Toolchain.Tools.Emitters = map[string]Tool{"go": pins["go-emitter"]} + if component == "normalizer" { + project.Toolchain.Tools.Normalizer = pin + } else { + project.Toolchain.Tools.Emitters["go"] = pin + } + if got, err := NewPipeline(context.Background(), &project, options); err == nil { + got.Close() + t.Fatal("invalid library selection accepted") + } + if _, err := os.Stat(options.WorkDir); !os.IsNotExist(err) { + t.Fatal("rejected selection created work files") + } + }) + } + } + p.Toolchain.Status = "released" + p.Toolchain.Tools.Normalizer, p.Toolchain.Tools.Emitters = pins["normalizer"], map[string]Tool{"go": pins["go-emitter"]} + if accepted, err := NewPipeline(context.Background(), p, o); err != nil { + t.Fatalf("matching host pins rejected: %v", err) + } else { + accepted.Close() + } +} + +func TestGoLibrariesHonorCancellation(t *testing.T) { + p, o := goLibraryProject(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + pipeline, err := NewPipeline(ctx, p, o) + if err != nil { + t.Fatal(err) + } + defer pipeline.Close() + cancel() + doc := &Document{} + if err := pipeline.Normalize(doc); !errors.Is(err, context.Canceled) || doc.Model.APIVersion != "" { + t.Fatalf("canceled normalization changed the model: %v", err) + } + if err := pipeline.emit(&BuiltTarget{Target: Target{Language: "go"}}); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled emission: %v", err) + } + if pipeline.work != "" { + t.Fatal("canceled operation created work files") + } +} + +func TestGoLibrariesPythonOnlyCatalogDoesNotRequireGoEmitter(t *testing.T) { + p, o := testProject(t, "python") + o.UseGoLibraries = true + pipeline, err := NewPipeline(context.Background(), p, o) + if err != nil { + t.Fatal(err) + } + defer pipeline.Close() + p.Toolchain.Status = "released" + p.Toolchain.Tools.Normalizer = pipeline.identities["normalizer"] + selected, err := NewPipeline(context.Background(), p, o) + if err != nil { + t.Fatalf("Python-only catalog required a Go emitter pin: %v", err) + } + defer selected.Close() + if _, selectedGo := selected.tools["go-emitter"]; selectedGo { + t.Fatal("Python-only catalog selected a Go emitter") + } +} diff --git a/tooling/extension-bindings/orchestrator/operations.go b/tooling/extension-bindings/orchestrator/operations.go new file mode 100644 index 0000000..a578a1c --- /dev/null +++ b/tooling/extension-bindings/orchestrator/operations.go @@ -0,0 +1,244 @@ +package orchestrator + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +// Execute runs one binding operation without parsing arguments or changing the +// process working directory. cwd must be absolute; relative option paths resolve +// against it. Initialize options with DefaultOptions to use standalone tooling +// defaults, then apply caller overrides. Writers must be non-nil. +func Execute(ctx context.Context, cwd, name string, o Options, stdout, stderr io.Writer) error { + if !supportedOperations[name] { + return fmt.Errorf("unknown bindings command %q", name) + } + if err := ctx.Err(); err != nil { + return err + } + if !filepath.IsAbs(cwd) { + return fmt.Errorf("working directory must be absolute") + } + if (name == "generate" || name == "package") && o.Output == "" { + return fmt.Errorf("%s requires --output", name) + } + // Options is passed by value; copy the slice before resolving its paths. + o.CatalogRoots = append(repeated(nil), o.CatalogRoots...) + o.Overrides = append(repeated(nil), o.Overrides...) + for i, entry := range o.Overrides { + if key, path, ok := strings.Cut(entry, "="); ok && path != "" { + o.Overrides[i] = key + "=" + absolute(cwd, path) + } + } + for _, ptr := range []*string{&o.Go, &o.Python, &o.Normalizer, &o.GoEmitter, &o.GoProfiler, &o.PythonProfiler} { + if strings.ContainsAny(*ptr, "/\\") { + *ptr = absolute(cwd, *ptr) + } + } + for _, ptr := range []*string{&o.Cache, &o.WorkDir, &o.Output, &o.Input, &o.Report, &o.ClosureOutput, &o.DependencyLockOutput, &o.DependencyLock, &o.CoreSchema, &o.PythonProfilerArtifact} { + if *ptr != "" { + *ptr = absolute(cwd, *ptr) + } + } + for i, dir := range o.CatalogRoots { + o.CatalogRoots[i] = absolute(cwd, dir) + } + project, err := DiscoverProject(cwd, o.Packages, o.ToolchainLock, o.ToolingDir) + if err != nil { + return err + } + pipeline, err := NewPipeline(ctx, project, o) + if err != nil { + return err + } + defer pipeline.Close() + outputs := []string{o.Output, o.Report, o.ClosureOutput, o.DependencyLockOutput} + seenOutputs := map[string]bool{} + for _, path := range outputs { + if path == "" { + continue + } + if err = rejectSymlinkPath(path); err != nil { + return err + } + if within(filepath.Join(project.RepositoryRoot, "bindings"), path) { + return fmt.Errorf("explicit outputs must be outside committed bindings; use update to synchronize package source") + } + if seenOutputs[path] { + return fmt.Errorf("output paths must be different: %s", path) + } + seenOutputs[path] = true + if name != "generate" && name != "package" { + if _, statErr := os.Lstat(path); !os.IsNotExist(statErr) { + return fmt.Errorf("output must be a new file: %s", path) + } + } + } + plan, err := project.SelectTargets(o.Targets) + if err != nil { + return err + } + if len(o.ExtensionRoots) > 0 && name != "resolve" && name != "normalize" { + roots := map[string]bool{} + for _, root := range o.ExtensionRoots { + roots[root] = true + } + filtered := BuildPlan{Targets: []Target{}} + foundRoots := map[string]bool{} + for _, target := range plan.Targets { + if roots[target.Set.RootExtension] { + filtered.Targets = append(filtered.Targets, target) + foundRoots[target.Set.RootExtension] = true + } + } + for root := range foundRoots { + delete(roots, root) + } + if len(roots) != 0 { + return fmt.Errorf("extension roots require package delivery metadata in the selected catalog targets: %s", strings.Join(sortedKeys(roots), ", ")) + } + plan = filtered + } + switch name { + case "resolve", "normalize": + documents, err := pipeline.Resolve(plan) + if err != nil { + return err + } + for i, doc := range documents { + if i > 0 { + fmt.Fprintln(stdout, "---") + } + var data []byte + if name == "resolve" { + data, err = yaml.Marshal(doc.SemanticClosure()) + } else { + if err = pipeline.Normalize(doc); err == nil { + data, err = doc.ModelYAML() + } + } + if err != nil { + return err + } + if _, err = stdout.Write(data); err != nil { + return err + } + if name == "resolve" { + lock, err := doc.LockYAML() + if err != nil { + return err + } + fmt.Fprintln(stdout, "---") + if _, err = stdout.Write(lock); err != nil { + return err + } + } + } + if o.ClosureOutput != "" || o.DependencyLockOutput != "" || o.Output != "" { + if len(documents) != 1 { + return fmt.Errorf("file outputs require one extension root; use --extension-root or --target") + } + doc := documents[0] + if o.ClosureOutput != "" { + data, _ := yaml.Marshal(doc.SemanticClosure()) + if err = writeNewFile(o.ClosureOutput, data); err != nil { + return err + } + } + if o.Output != "" { + data, err := doc.ModelYAML() + if err != nil { + return err + } + if err = writeNewFile(o.Output, data); err != nil { + return err + } + } + if o.DependencyLockOutput != "" { + data, err := doc.LockYAML() + if err != nil { + return err + } + if err = writeNewFile(o.DependencyLockOutput, data); err != nil { + return err + } + } + } + return nil + case "generate": + if err = validateEmptyDirectory(o.Output); err != nil { + return err + } + build, err := pipeline.Generate(plan) + if err != nil { + return err + } + if err = prepareEmptyDirectory(o.Output); err != nil { + return err + } + return build.CopySelected(o.Output) + case "plan-release": + release, err := pipeline.PlanRelease(plan) + if err != nil { + return err + } + data, err := yaml.Marshal(release) + if err != nil { + return err + } + if _, err = stdout.Write(data); err != nil { + return err + } + if o.Output != "" { + return writeNewFile(o.Output, data) + } + return nil + case "verify", "check", "package", "update": + var build *Build + if name == "verify" { + build, err = pipeline.LoadBuild(plan, o.Input) + } else { + build, err = pipeline.Generate(plan) + } + if err != nil { + failure := VerificationSummary{Status: "failed", Targets: []TargetResult{}} + failure.Fail("build", err) + return writeSummary(name, o, failure, err, stdout, stderr) + } + summary, err := pipeline.Verify(build) + if err == nil && (name == "check" || name == "package") { + err = build.CompareCommitted() + if err != nil { + summary.Fail("committed-source", err) + } + } + if err == nil && name == "update" { + err = build.Update() + if err != nil { + summary.Fail("update", err) + } + } + if err == nil && name == "package" { + committed, loadErr := pipeline.LoadBuild(plan, "") + if loadErr != nil { + err = loadErr + } else { + _, err = pipeline.Verify(committed) + if err == nil { + err = pipeline.Package(committed, o.Output) + } + } + if err != nil { + summary.Fail("package", err) + } + } + return writeSummary(name, o, summary, err, stdout, stderr) + } + return fmt.Errorf("unhandled bindings command %s", name) +} diff --git a/tooling/extension-bindings/orchestrator/operations_test.go b/tooling/extension-bindings/orchestrator/operations_test.go new file mode 100644 index 0000000..9a7b11f --- /dev/null +++ b/tooling/extension-bindings/orchestrator/operations_test.go @@ -0,0 +1,194 @@ +package orchestrator + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestDefaultOptionsRetainStandaloneToolSelection(t *testing.T) { + for key, value := range map[string]string{ + "RC_BINDINGS_GO": "/tools/go", "RC_BINDINGS_PYTHON": "/tools/python", + "RC_BINDINGS_NORMALIZER": "/tools/normalizer", "RC_BINDINGS_GO_EMITTER": "/tools/emitter", + "RC_GO_PROFILER_BIN": "/tools/go-profiler", "RC_PYTHON_PROFILER_BIN": "/tools/python-profiler", + "RC_PYTHON_PROFILER_ARTIFACT": "/tools/profiler.whl", + } { + t.Setenv(key, value) + } + o := DefaultOptions() + if o.Go != "/tools/go" || o.Python != "/tools/python" || o.Normalizer != "/tools/normalizer" || + o.GoEmitter != "/tools/emitter" || o.GoProfiler != "/tools/go-profiler" || + o.PythonProfiler != "/tools/python-profiler" || o.PythonProfilerArtifact != "/tools/profiler.whl" { + t.Fatalf("standalone defaults: %+v", o) + } + t.Setenv("RC_BINDINGS_GO", "") + t.Setenv("RC_BINDINGS_PYTHON", "") + t.Setenv("RC_GO_PROFILER_BIN", "") + t.Setenv("RC_PYTHON_PROFILER_BIN", "") + o = DefaultOptions() + if o.Go != "go" || o.Python != "python3" || o.GoProfiler != "go-rc-profiler" || o.PythonProfiler != "runtimeconditions-python-profiler" { + t.Fatalf("native command defaults: %+v", o) + } +} + +func TestExecuteResolveAndNormalizeMatchStandalone(t *testing.T) { + p, defaults := testProject(t, "go") + nested := filepath.Join(p.Root, "nested working directory") + if err := os.Mkdir(nested, 0755); err != nil { + t.Fatal(err) + } + core := filepath.Join(p.Root, "core.yaml") + if err := os.WriteFile(core, []byte("$id: https://runtimeconditions.io/schemas/profile/0.2.0/runtimeconditions.profile.schema.yaml\ntype: object\n"), 0644); err != nil { + t.Fatal(err) + } + normalizer := filepath.Join(t.TempDir(), "normalizer") + command := exec.Command(defaults.Go, "build", "-trimpath", "-buildvcs=false", "-o", normalizer, "./cmd/rc-binding-model") + command.Dir = filepath.Join(p.ToolingDirectory, "normalizer") + if data, err := command.CombinedOutput(); err != nil { + t.Fatalf("build normalizer: %v\n%s", err, data) + } + relativeNormalizer, err := filepath.Rel(nested, normalizer) + if err != nil { + t.Fatal(err) + } + processCWD, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + for _, name := range []string{"resolve", "normalize"} { + t.Run(name, func(t *testing.T) { + o := defaults + o.Packages, o.ToolchainLock = "../packages.yaml", "../toolchain.lock.yaml" + o.CoreSchema, o.Normalizer = "../core.yaml", relativeNormalizer + o.CatalogRoots = []string{"../catalog"} + o.Overrides = []string{p.Catalog.Packages["new-channel"].RootExtension + "=../catalog/" + extensionName} + o.Targets = []string{"new-channel:go"} + o.DependencyLockOutput = "../" + name + " API lock.yaml" + apiFile, cliFile := filepath.Join(p.Root, name+" API.yaml"), filepath.Join(p.Root, name+" CLI.yaml") + cliLock := filepath.Join(p.Root, name+" CLI lock.yaml") + args := []string{"bindings", name, "--packages", p.PackagesPath, "--toolchain-lock", p.ToolchainPath, + "--tooling-dir", p.ToolingDirectory, "--catalog-root", filepath.Join(p.Root, "catalog"), + "--target", "new-channel:go", "--go", o.Go, + "--normalizer", normalizer, "--core-schema", core, "--dependency-lock-output", cliLock} + args = append(args, "--extension-override", p.Catalog.Packages["new-channel"].RootExtension+"="+filepath.Join(p.Root, "catalog", extensionName)) + if name == "resolve" { + o.ClosureOutput = "../" + name + " API.yaml" + args = append(args, "--closure-output", cliFile) + } else { + o.Output = "../" + name + " API.yaml" + args = append(args, "--output", cliFile) + } + var apiOut, apiErr, cliOut, cliErr bytes.Buffer + if err := Execute(context.Background(), nested, name, o, &apiOut, &apiErr); err != nil { + t.Fatal(err) + } + if err := Run(context.Background(), args, &cliOut, &cliErr); err != nil { + t.Fatal(err) + } + if !bytes.Equal(apiOut.Bytes(), cliOut.Bytes()) || !bytes.Equal(apiErr.Bytes(), cliErr.Bytes()) { + t.Fatalf("API and standalone streams differ\nAPI: %s\nCLI: %s", apiOut.Bytes(), cliOut.Bytes()) + } + if !strings.Contains(apiOut.String(), "future.channel") { + t.Fatal("resolved vocabulary missing from YAML") + } + for _, pair := range [][2]string{{apiFile, cliFile}, {filepath.Join(nested, o.DependencyLockOutput), cliLock}} { + first, err := os.ReadFile(pair[0]) + if err != nil { + t.Fatal(err) + } + second, err := os.ReadFile(pair[1]) + if err != nil || !bytes.Equal(first, second) { + t.Fatalf("API and standalone files differ: %v", err) + } + } + if o.CatalogRoots[0] != "../catalog" || o.CoreSchema != "../core.yaml" || o.Packages != "../packages.yaml" { + t.Fatalf("Execute mutated caller options: %+v", o) + } + if !strings.Contains(o.Overrides[0], "=../catalog/") || o.Normalizer != relativeNormalizer { + t.Fatalf("Execute mutated caller tool/override paths: %+v", o) + } + }) + } + if current, err := os.Getwd(); err != nil || current != processCWD { + t.Fatalf("process directory changed: %s %v", current, err) + } + if _, err := os.Stat(filepath.Join(p.Root, "bindings")); !os.IsNotExist(err) { + t.Fatal("resolve/normalize created committed binding trees") + } +} + +func TestExecuteValidationMatchesStandalone(t *testing.T) { + p, defaults := testProject(t, "go") + directory := filepath.Join(p.Root, "keep") + if err := os.Mkdir(directory, 0755); err != nil { + t.Fatal(err) + } + sentinel := filepath.Join(directory, "existing.yaml") + if err := os.WriteFile(sentinel, []byte("preserve existing bytes\n"), 0644); err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ name, operation, flag, path, want string }{ + {"existing file", "resolve", "--closure-output", sentinel, "output must be a new file"}, + {"nonempty directory", "generate", "--output", directory, "output must be new or empty"}, + {"committed source", "resolve", "--closure-output", filepath.Join(p.Root, "bindings", "forbidden.yaml"), "explicit outputs must be outside committed bindings"}, + {"duplicate targets", "resolve", "--target", "new-channel:go", "duplicate target new-channel:go"}, + } { + t.Run(tc.name, func(t *testing.T) { + o := defaults + o.Packages, o.ToolchainLock = p.PackagesPath, p.ToolchainPath + if tc.flag == "--output" { + o.Output = tc.path + } else if tc.flag == "--target" { + o.Targets = []string{tc.path, tc.path} + } else { + o.ClosureOutput = tc.path + } + var apiOut, apiErr, cliOut, cliErr bytes.Buffer + apiResult := Execute(context.Background(), p.Root, tc.operation, o, &apiOut, &apiErr) + args := []string{"bindings", tc.operation, "--packages", p.PackagesPath, + "--toolchain-lock", p.ToolchainPath, "--tooling-dir", p.ToolingDirectory, tc.flag, tc.path} + if tc.flag == "--target" { + args = append(args, tc.flag, tc.path) + } + cliResult := Run(context.Background(), args, &cliOut, &cliErr) + if apiResult == nil || cliResult == nil || apiResult.Error() != cliResult.Error() || !strings.Contains(apiResult.Error(), tc.want) { + t.Fatalf("API error %v; standalone error %v; want %s", apiResult, cliResult, tc.want) + } + if apiOut.Len() != 0 || cliOut.Len() != 0 || !bytes.Equal(apiErr.Bytes(), cliErr.Bytes()) { + t.Fatal("output validation changed streams") + } + if data, err := os.ReadFile(sentinel); err != nil || string(data) != "preserve existing bytes\n" { + t.Fatalf("existing output changed: %s %v", data, err) + } + }) + } +} + +func TestExecuteRejectsInvalidRequestsBeforeDiscovery(t *testing.T) { + root := t.TempDir() + for _, tc := range []struct{ cwd, operation, want string }{ + {root, "unknown", "unknown bindings command"}, + {"relative", "resolve", "working directory must be absolute"}, + {root, "generate", "generate requires --output"}, + {root, "package", "package requires --output"}, + } { + err := Execute(context.Background(), tc.cwd, tc.operation, DefaultOptions(), io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("%s: %v; want %s", tc.operation, err, tc.want) + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := Execute(ctx, root, "resolve", DefaultOptions(), io.Discard, io.Discard); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation: %v", err) + } + if entries, err := os.ReadDir(root); err != nil || len(entries) != 0 { + t.Fatalf("invalid requests created files: %v %v", entries, err) + } +} diff --git a/tooling/extension-bindings/orchestrator/pipeline.go b/tooling/extension-bindings/orchestrator/pipeline.go index 00139cd..0c19198 100644 --- a/tooling/extension-bindings/orchestrator/pipeline.go +++ b/tooling/extension-bindings/orchestrator/pipeline.go @@ -62,10 +62,45 @@ type Build struct { } func NewPipeline(ctx context.Context, p *Project, o Options) (*Pipeline, error) { + if o.UseGoLibraries && (o.Normalizer != "" || o.GoEmitter != "") { + return nil, fmt.Errorf("Go library mode conflicts with standalone normalizer or Go emitter selection") + } if p.Toolchain.Status == "released" && len(o.Overrides) > 0 { return nil, fmt.Errorf("released generation forbids extension overrides") } + if o.LinkedGoProfiler != nil { + if o.GoProfiler != "" { + return nil, fmt.Errorf("select either a linked Go profiler identity or a standalone Go profiler executable") + } + identity := *o.LinkedGoProfiler + if identity.Kind != linkedGoProfilerKind { + return nil, fmt.Errorf("LinkedGoProfiler requires a linked Go module identity") + } + if err := identity.Validate(); err != nil { + return nil, err + } + if expected, pinned := p.Toolchain.Tools.Profilers["go"]; pinned { + if err := expected.Match(identity); err != nil { + return nil, fmt.Errorf("go-profiler: %w", err) + } + } + o.LinkedGoProfiler = &identity // Snapshot the caller's supplied identity. + } pipeline := &Pipeline{Context: ctx, Project: p, Options: o, tools: map[string]string{}, identities: map[string]Tool{}, cores: map[string]map[string]any{}, HTTPClient: &http.Client{Timeout: 60 * time.Second, CheckRedirect: func(_ *http.Request, _ []*http.Request) error { return fmt.Errorf("redirects are forbidden") }}} + if o.UseGoLibraries { + keys := []string{"normalizer"} + for _, set := range p.Catalog.Packages { + if _, usesGo := set.Languages["go"]; usesGo { + keys = append(keys, "go-emitter") + break + } + } + for _, key := range keys { + if _, _, err := pipeline.goLibraryTool(key); err != nil { + return nil, err + } + } + } for _, path := range []string{o.WorkDir, o.Cache} { if path != "" { if err := rejectSymlinkPath(path); err != nil { @@ -172,6 +207,9 @@ func (p *Pipeline) python(args ...string) ([]byte, error) { return p.run(p.Project.Root, p.pythonEnv(), p.Options.Python, append(flags, args...)...) } func (p *Pipeline) tool(key string) (string, Tool, error) { + if p.Options.UseGoLibraries && (key == "normalizer" || key == "go-emitter") { + return p.goLibraryTool(key) + } if path, ok := p.tools[key]; ok { return path, p.identities[key], nil } @@ -186,7 +224,14 @@ func (p *Pipeline) tool(key string) (string, Tool, error) { expected = p.Project.Toolchain.Tools.Emitters["go"] case "go-profiler": path = p.Options.GoProfiler - expected = p.Project.Toolchain.Tools.Profilers["go"] + pin := p.Project.Toolchain.Tools.Profilers["go"] + if pin.Kind != "" { + return "", Tool{}, fmt.Errorf("go-profiler identity kind differs from lock") + } + if path == "" { + return "", Tool{}, fmt.Errorf("standalone Go profiler path required; supply --go-profiler or a linked identity") + } + expected = Tool{Name: pin.Name, Version: pin.Version, SHA256: pin.SHA256, ArtifactURL: pin.ArtifactURL} case "orchestrator": path, _ = os.Executable() expected = p.Project.Toolchain.Tools.Orchestrator @@ -314,14 +359,45 @@ func (p *Pipeline) pythonEmitterIdentity() (Tool, error) { p.identities["python-emitter"] = identity return identity, nil } -func (p *Pipeline) profiler(language string) (Tool, error) { - if language == "go" { - _, identity, err := p.tool("go-profiler") - return identity, err +func (p *Pipeline) profiler(language string) (ProfilerIdentity, error) { + var artifact Tool + var err error + switch language { + case "go": + if p.Options.LinkedGoProfiler != nil { + return p.matchProfilerLock(language, *p.Options.LinkedGoProfiler) + } + _, artifact, err = p.tool("go-profiler") + case "python": + artifact, err = p.pythonProfiler() + default: + return ProfilerIdentity{}, fmt.Errorf("unsupported native adapter %s", language) + } + if err != nil { + return ProfilerIdentity{}, err + } + identity, err := NewStandaloneProfilerIdentity(artifact) + if err != nil { + return ProfilerIdentity{}, err + } + return p.matchProfilerLock(language, identity) +} + +func (p *Pipeline) matchProfilerLock(language string, identity ProfilerIdentity) (ProfilerIdentity, error) { + if err := identity.Validate(); err != nil { + return ProfilerIdentity{}, err } - if language != "python" { - return Tool{}, fmt.Errorf("unsupported native adapter %s", language) + if expected, pinned := p.Project.Toolchain.Tools.Profilers[language]; pinned { + if err := expected.Match(identity); err != nil { + return ProfilerIdentity{}, fmt.Errorf("%s profiler: %w", language, err) + } + } else if p.Project.Toolchain.Status == "released" { + return ProfilerIdentity{}, fmt.Errorf("%s: released lock lacks profiler", language) } + return identity, nil +} + +func (p *Pipeline) pythonProfiler() (Tool, error) { if identity, ok := p.identities["python-profiler"]; ok { return identity, nil } @@ -343,10 +419,6 @@ func (p *Pipeline) profiler(language string) (Tool, error) { return Tool{}, err } identity := Tool{Name: "python-rc-profiler", Version: actual.Version, SHA256: actual.SHA256} - expected := p.Project.Toolchain.Tools.Profilers[language] - if expected.SHA256 != "" && (expected.SHA256 != identity.SHA256 || expected.Version != identity.Version) { - return Tool{}, fmt.Errorf("Python profiler identity differs from lock") - } p.identities["python-profiler"] = identity return identity, nil } @@ -526,6 +598,9 @@ func (p *Pipeline) Normalize(doc *Document) error { if doc.Model.APIVersion != "" { return nil } + if p.Options.UseGoLibraries { + return p.normalizeGoLibrary(doc) + } _, core, err := p.core(doc.CoreVersion) if err != nil { return err @@ -733,6 +808,9 @@ func (p *Pipeline) emitterInputs(bt *BuiltTarget) (string, string, error) { return modelPath, targetPath, nil } func (p *Pipeline) emit(bt *BuiltTarget) error { + if p.Options.UseGoLibraries && bt.Target.Language == "go" { + return p.emitGoLibrary(bt) + } modelPath, targetPath, err := p.emitterInputs(bt) if err != nil { return err diff --git a/tooling/extension-bindings/orchestrator/profiler_identity.go b/tooling/extension-bindings/orchestrator/profiler_identity.go new file mode 100644 index 0000000..a4121d2 --- /dev/null +++ b/tooling/extension-bindings/orchestrator/profiler_identity.go @@ -0,0 +1,111 @@ +package orchestrator + +import ( + "fmt" + "regexp" +) + +const linkedGoProfilerModule = "github.com/runtimeconditions/go-rc-profiler" +const linkedGoProfilerKind = "go-module" + +var profilerArtifactDigest = regexp.MustCompile(`^[0-9a-f]{64}$`) +var profilerArtifactURL = regexp.MustCompile(`^https://[^\s]+$`) +var profilerModuleSum = regexp.MustCompile(`^h1:[A-Za-z0-9+/]{42}[AEIMQUYcgkosw048]=$`) +var profilerModuleVersion = regexp.MustCompile(`^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(\.(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*)?(\+incompatible)?$`) + +// ProfilerIdentity describes either a standalone artifact or the linked Go +// profiler module. Standalone identities omit Kind and retain the legacy +// name/version/sha256 representation. Linked identities require kind: go-module +// and module/moduleSum, and cannot contain sha256 or artifactUrl. +// These helpers define the contract; pipeline selection is integrated separately. +type ProfilerIdentity struct { + Name string `json:"name" yaml:"name"` + Version string `json:"version" yaml:"version"` + SHA256 string `json:"sha256,omitempty" yaml:"sha256,omitempty"` + ArtifactURL string `json:"artifactUrl,omitempty" yaml:"artifactUrl,omitempty"` + Kind string `json:"kind,omitempty" yaml:"kind,omitempty"` + Module string `json:"module,omitempty" yaml:"module,omitempty"` + ModuleSum string `json:"moduleSum,omitempty" yaml:"moduleSum,omitempty"` +} + +// NewStandaloneProfilerIdentity validates the existing artifact identity without +// changing its version or digest. ArtifactURL is optional lock retrieval data. +func NewStandaloneProfilerIdentity(tool Tool) (ProfilerIdentity, error) { + identity := ProfilerIdentity{Name: tool.Name, Version: tool.Version, SHA256: tool.SHA256, ArtifactURL: tool.ArtifactURL} + return identity, identity.Validate() +} + +// NewLinkedGoProfilerIdentity validates an exact module version and Go h1 +// dependency checksum supplied by the host's verified build metadata. It does +// not inspect an executable, resolve a module, or install any tooling. +func NewLinkedGoProfilerIdentity(version, moduleSum string) (ProfilerIdentity, error) { + identity := ProfilerIdentity{ + Name: "go-rc-profiler", Version: version, Kind: linkedGoProfilerKind, + Module: linkedGoProfilerModule, ModuleSum: moduleSum, + } + return identity, identity.Validate() +} + +// Validate checks a complete identity, rejecting mixed artifact/module fields. +// Validate document schemas before decoding to reject unknown input fields too. +func (identity ProfilerIdentity) Validate() error { + if identity.Name == "" || identity.Version == "" { + return fmt.Errorf("profiler identity requires name and version") + } + switch identity.Kind { + case "": + if identity.Module != "" || identity.ModuleSum != "" { + return fmt.Errorf("standalone profiler identity cannot contain module fields") + } + if !profilerArtifactDigest.MatchString(identity.SHA256) { + return fmt.Errorf("standalone profiler identity requires a lowercase artifact SHA-256") + } + if identity.ArtifactURL != "" && !profilerArtifactURL.MatchString(identity.ArtifactURL) { + return fmt.Errorf("profiler artifactUrl must be an HTTPS URL without whitespace") + } + case linkedGoProfilerKind: + if identity.SHA256 != "" || identity.ArtifactURL != "" { + return fmt.Errorf("linked profiler identity cannot contain artifact fields") + } + if identity.Name != "go-rc-profiler" || identity.Module != linkedGoProfilerModule { + return fmt.Errorf("linked profiler identity requires the Go profiler name and module path") + } + if !profilerModuleVersion.MatchString(identity.Version) { + return fmt.Errorf("linked profiler identity requires an exact Go module version") + } + if !profilerModuleSum.MatchString(identity.ModuleSum) { + return fmt.Errorf("linked profiler identity requires a canonical Go h1 dependency checksum") + } + default: + return fmt.Errorf("unknown profiler identity kind %q", identity.Kind) + } + return nil +} + +// Match requires the selected profiler to have exactly the identity pinned by +// the receiver. Retrieval URLs do not identify artifact contents. Neither an +// incomplete lock nor a different identity kind can satisfy this comparison. +func (expected ProfilerIdentity) Match(selected ProfilerIdentity) error { + if err := expected.Validate(); err != nil { + return fmt.Errorf("invalid locked profiler identity: %w", err) + } + if err := selected.Validate(); err != nil { + return fmt.Errorf("invalid selected profiler identity: %w", err) + } + if expected.Kind != selected.Kind { + return fmt.Errorf("profiler identity kind differs from lock") + } + if expected.Name != selected.Name { + return fmt.Errorf("profiler name differs from lock") + } + if expected.Version != selected.Version { + return fmt.Errorf("profiler version differs from lock") + } + if expected.SHA256 != selected.SHA256 { + return fmt.Errorf("profiler artifact digest differs from lock") + } + if expected.Module != selected.Module || expected.ModuleSum != selected.ModuleSum { + return fmt.Errorf("profiler module identity differs from lock") + } + return nil +} diff --git a/tooling/extension-bindings/orchestrator/profiler_identity_test.go b/tooling/extension-bindings/orchestrator/profiler_identity_test.go new file mode 100644 index 0000000..0a6483b --- /dev/null +++ b/tooling/extension-bindings/orchestrator/profiler_identity_test.go @@ -0,0 +1,228 @@ +package orchestrator + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func testProfilerIdentities(t *testing.T) (ProfilerIdentity, ProfilerIdentity) { + t.Helper() + standalone, err := NewStandaloneProfilerIdentity(Tool{Name: "go-rc-profiler", Version: "v0.6.0", SHA256: strings.Repeat("0", 64)}) + if err != nil { + t.Fatal(err) + } + linked, err := NewLinkedGoProfilerIdentity("v0.6.0", "h1:"+strings.Repeat("A", 43)+"=") + if err != nil { + t.Fatal(err) + } + return standalone, linked +} + +func changedProfiler(identity ProfilerIdentity, change func(*ProfilerIdentity)) ProfilerIdentity { + change(&identity) + return identity +} + +func profilerIdentityMapping(t *testing.T, identity ProfilerIdentity) map[string]any { + t.Helper() + data, err := json.Marshal(identity) + if err != nil { + t.Fatal(err) + } + var result map[string]any + if err := json.Unmarshal(data, &result); err != nil { + t.Fatal(err) + } + return result +} + +func testProfilerSchemas(t *testing.T) map[string]map[string]any { + t.Helper() + schemas := map[string]map[string]any{} + for name, path := range map[string]string{ + "release": filepath.Join("..", "model", "runtimeconditions.binding-release.schema.yaml"), + "lock": filepath.Join("..", "model", "runtimeconditions.toolchain-lock.schema.yaml"), + "Go profiler": filepath.Join("..", "..", "..", "..", "go-rc-profiler", "extensioncheck", "schema", "runtimeconditions.binding-release.schema.yaml"), + } { + schema, err := readMapping(path) + if name == "Go profiler" && os.IsNotExist(err) { + t.Log("sibling Go profiler checkout absent; embedded schema comparison omitted") + continue + } + if err != nil { + t.Fatal(err) + } + schemas[name] = schema + } + return schemas +} + +func TestStandaloneProfilerIdentityPreservesLegacyEncoding(t *testing.T) { + tool := Tool{Name: "go-rc-profiler", Version: "v0.6.0", SHA256: strings.Repeat("a", 64), ArtifactURL: "https://example.test/profiler.tar.gz"} + identity, err := NewStandaloneProfilerIdentity(tool) + if err != nil { + t.Fatal(err) + } + for _, marshal := range []func(any) ([]byte, error){json.Marshal, yaml.Marshal} { + before, err := marshal(tool) + if err != nil { + t.Fatal(err) + } + after, err := marshal(identity) + if err != nil || !bytes.Equal(before, after) { + t.Fatalf("legacy identity encoding changed: %s -> %s (%v)", before, after, err) + } + } + if _, err := NewStandaloneProfilerIdentity(Tool{}); err == nil { + t.Fatal("empty standalone identity accepted") + } + if _, err := NewLinkedGoProfilerIdentity("unknown", ""); err == nil { + t.Fatal("unknown linked identity accepted") + } +} + +func TestProfilerIdentityValidationAndSchemaAgreement(t *testing.T) { + standalone, linked := testProfilerIdentities(t) + schemas := testProfilerSchemas(t) + for name, schema := range schemas { + for _, definition := range []string{"profilerIdentity", "linkedGoProfilerIdentity"} { + if !reflect.DeepEqual(mapValue(schema["$defs"])[definition], mapValue(schemas["release"]["$defs"])[definition]) { + t.Fatalf("%s disagrees on %s", name, definition) + } + } + } + for _, tc := range []struct { + name string + identity ProfilerIdentity + valid bool + }{ + {"standalone", standalone, true}, {"linked", linked, true}, + {"prerelease", changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "v0.6.1-rc.1" }), true}, + {"pseudo version", changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "v0.6.1-0.20261109120000-0123456789ab" }), true}, + {"empty", ProfilerIdentity{}, false}, + {"missing name", changedProfiler(standalone, func(p *ProfilerIdentity) { p.Name = "" }), false}, + {"missing version", changedProfiler(standalone, func(p *ProfilerIdentity) { p.Version = "" }), false}, + {"uppercase digest", changedProfiler(standalone, func(p *ProfilerIdentity) { p.SHA256 = strings.Repeat("A", 64) }), false}, + {"short digest", changedProfiler(standalone, func(p *ProfilerIdentity) { p.SHA256 = "1234" }), false}, + {"invalid retrieval URL", changedProfiler(standalone, func(p *ProfilerIdentity) { p.ArtifactURL = "http://example.test/profiler" }), false}, + {"artifact with module", changedProfiler(standalone, func(p *ProfilerIdentity) { p.Module = linked.Module }), false}, + {"artifact with module sum", changedProfiler(standalone, func(p *ProfilerIdentity) { p.ModuleSum = linked.ModuleSum }), false}, + {"unknown kind", changedProfiler(linked, func(p *ProfilerIdentity) { p.Kind = "other" }), false}, + {"wrong profiler", changedProfiler(linked, func(p *ProfilerIdentity) { p.Name = "python-rc-profiler" }), false}, + {"wrong module", changedProfiler(linked, func(p *ProfilerIdentity) { p.Module = "example.test/profiler" }), false}, + {"linked with artifact", changedProfiler(linked, func(p *ProfilerIdentity) { p.SHA256 = standalone.SHA256 }), false}, + {"linked with URL", changedProfiler(linked, func(p *ProfilerIdentity) { p.ArtifactURL = "https://example.test/profiler" }), false}, + {"unknown version", changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "unknown" }), false}, + {"short version", changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "v0.6" }), false}, + {"leading zero", changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "v0.6.0-rc.01" }), false}, + {"noncanonical version", changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "v0.6.0+local" }), false}, + {"hex module sum", changedProfiler(linked, func(p *ProfilerIdentity) { p.ModuleSum = standalone.SHA256 }), false}, + {"noncanonical padding", changedProfiler(linked, func(p *ProfilerIdentity) { p.ModuleSum = "h1:" + strings.Repeat("A", 42) + "B=" }), false}, + {"checksum newline", changedProfiler(linked, func(p *ProfilerIdentity) { p.ModuleSum += "\n" }), false}, + {"missing checksum", changedProfiler(linked, func(p *ProfilerIdentity) { p.ModuleSum = "" }), false}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := tc.identity.Validate(); (err == nil) != tc.valid { + t.Fatalf("identity validity: %v; want %t", err, tc.valid) + } + for name, schema := range schemas { + identitySchema := map[string]any{"$ref": "#/$defs/profilerIdentity", "$defs": schema["$defs"]} + if err := validateSchema(identitySchema, profilerIdentityMapping(t, tc.identity)); (err == nil) != tc.valid { + t.Errorf("%s schema validity: %v; want %t", name, err, tc.valid) + } + } + }) + } + for name, schema := range schemas { + value := profilerIdentityMapping(t, linked) + value["unknown"] = "must reject" + if err := validateSchema(map[string]any{"$ref": "#/$defs/profilerIdentity", "$defs": schema["$defs"]}, value); err == nil { + t.Errorf("%s accepted an unknown identity field", name) + } + } +} + +func TestProfilerIdentityMatchRequiresExactKindVersionAndChecksum(t *testing.T) { + standalone, linked := testProfilerIdentities(t) + for _, tc := range []struct { + name string + expected, selected ProfilerIdentity + match bool + }{ + {"artifact", standalone, standalone, true}, {"module", linked, linked, true}, + {"retrieval URL", standalone, changedProfiler(standalone, func(p *ProfilerIdentity) { p.ArtifactURL = "https://example.test/mirror" }), true}, + {"name", standalone, changedProfiler(standalone, func(p *ProfilerIdentity) { p.Name = "other" }), false}, + {"artifact version", standalone, changedProfiler(standalone, func(p *ProfilerIdentity) { p.Version = "v0.6.1" }), false}, + {"module version", linked, changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "v0.6.1" }), false}, + {"digest", standalone, changedProfiler(standalone, func(p *ProfilerIdentity) { p.SHA256 = strings.Repeat("1", 64) }), false}, + {"module sum", linked, changedProfiler(linked, func(p *ProfilerIdentity) { p.ModuleSum = "h1:" + strings.Repeat("B", 42) + "A=" }), false}, + {"module for executable lock", standalone, linked, false}, {"executable for module lock", linked, standalone, false}, + {"incomplete lock", ProfilerIdentity{}, linked, false}, {"incomplete selection", linked, ProfilerIdentity{}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := tc.expected.Match(tc.selected); (err == nil) != tc.match { + t.Fatalf("match: %v; want %t", err, tc.match) + } + }) + } +} + +func TestProfilerIdentityInReleaseAndLockDocuments(t *testing.T) { + standalone, linked := testProfilerIdentities(t) + schemas := testProfilerSchemas(t) + for name, schema := range schemas { + if name == "lock" { + continue + } + doc, err := readMapping(filepath.Join("..", "..", "..", "bindings", "env-configuration", "go", releaseName)) + if err != nil { + t.Fatal(err) + } + if err := validateSchema(schema, doc); err != nil { + t.Fatalf("%s rejected legacy release: %v", name, err) + } + mapValue(doc["provenance"])["profiler"] = profilerIdentityMapping(t, linked) + if err := validateSchema(schema, doc); err != nil { + t.Fatalf("%s rejected linked release: %v", name, err) + } + pkg := mapValue(doc["package"]) + pkg["language"], pkg["minimumPythonVersion"] = "python", "3.11" + delete(pkg, "minimumGoVersion") + for _, dependency := range sequence(doc["packageDependencies"]) { + mapValue(mapValue(dependency)["artifact"])["kind"] = "python-wheel" + } + mapValue(doc["provenance"])["profiler"] = profilerIdentityMapping(t, changedProfiler(standalone, func(p *ProfilerIdentity) { p.Name = "python-rc-profiler" })) + if err := validateSchema(schema, doc); err != nil { + t.Fatalf("%s rejected Python release: %v", name, err) + } + mapValue(doc["provenance"])["profiler"] = profilerIdentityMapping(t, linked) + if err := validateSchema(schema, doc); err == nil { + t.Fatalf("%s accepted linked Go profiler for Python", name) + } + } + artifact := profilerIdentityMapping(t, standalone) + profilers := map[string]any{"go": artifact} + doc := map[string]any{ + "apiVersion": "runtimeconditions.io/toolchain-lock/v1alpha1", "kind": "RuntimeConditionsToolchainLock", "status": "released", + "tools": map[string]any{"normalizer": artifact, "orchestrator": artifact, "emitters": map[string]any{"go": artifact}, "profilers": profilers}, + "languages": map[string]any{"go": map[string]any{"languageVersion": "1.25.0", "packageManager": "go", "tools": map[string]any{"go": "1.25.0"}}}, + } + if err := validateSchema(schemas["lock"], doc); err != nil { + t.Fatal("legacy lock rejected", err) + } + profilers["go"] = profilerIdentityMapping(t, linked) + if err := validateSchema(schemas["lock"], doc); err != nil { + t.Fatal("linked Go lock rejected", err) + } + profilers["python"] = profilerIdentityMapping(t, linked) + if err := validateSchema(schemas["lock"], doc); err == nil { + t.Fatal("linked Go identity accepted for Python lock") + } +} diff --git a/tooling/extension-bindings/orchestrator/profiler_pipeline_test.go b/tooling/extension-bindings/orchestrator/profiler_pipeline_test.go new file mode 100644 index 0000000..7bba18e --- /dev/null +++ b/tooling/extension-bindings/orchestrator/profiler_pipeline_test.go @@ -0,0 +1,243 @@ +package orchestrator + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func writeProfilerFixture(t *testing.T, path string, value any) { + t.Helper() + data, err := yaml.Marshal(value) + if err == nil { + err = os.WriteFile(path, data, 0644) + } + if err != nil { + t.Fatal(err) + } +} + +func profilerPipelineProject(t *testing.T) (*Project, Options) { + t.Helper() + p, o := testProject(t, "go") + o.CoreSchema = filepath.Join(p.Root, "core.yaml") + writeProfilerFixture(t, o.CoreSchema, map[string]any{"$id": "https://runtimeconditions.io/schemas/profile/0.2.0/runtimeconditions.profile.schema.yaml", "type": "object"}) + return p, o +} + +func standaloneProfilerFixture(t *testing.T) (string, ProfilerIdentity) { + t.Helper() + directory := t.TempDir() + for name, content := range map[string]string{ + "go.mod": "module example.test/standalone-profiler\n\ngo 1.22\n", + "main.go": "package main\nfunc main() { panic(\"profiler identity inspection must not execute the binary\") }\n", + } { + if err := os.WriteFile(filepath.Join(directory, name), []byte(content), 0644); err != nil { + t.Fatal(err) + } + } + path := filepath.Join(directory, "profiler") + command := exec.Command(envOr("RC_BINDINGS_GO", "go"), "build", "-trimpath", "-buildvcs=false", "-o", path, ".") + command.Dir = directory + if data, err := command.CombinedOutput(); err != nil { + t.Fatalf("build standalone identity fixture: %v\n%s", err, data) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + identity, err := NewStandaloneProfilerIdentity(Tool{Name: "go-rc-profiler", Version: "v0.6.0", SHA256: fmt.Sprintf("%x", sha256.Sum256(data))}) + if err != nil { + t.Fatal(err) + } + return path, identity +} + +func TestProfilerPipelineGeneratesVerifiesAndRejectsTampering(t *testing.T) { + path, standalone := standaloneProfilerFixture(t) + _, linked := testProfilerIdentities(t) + for _, expected := range []ProfilerIdentity{standalone, linked} { + name := "standalone" + if expected.Kind != "" { + name = "linked" + } + t.Run(name, func(t *testing.T) { + p, o := profilerPipelineProject(t) + p.Toolchain.Tools.Profilers = map[string]ProfilerIdentity{"go": expected} + selected := expected + if expected.Kind == "" { + t.Setenv("RC_GO_PROFILER_BIN", path) + o.GoProfiler = DefaultOptions().GoProfiler + } else { + t.Setenv("RC_GO_PROFILER_BIN", filepath.Join(p.Root, "absent-profiler")) + o.LinkedGoProfiler = &selected + } + pipeline, err := NewPipeline(context.Background(), p, o) + if err != nil { + t.Fatal(err) + } + defer pipeline.Close() + selected.ModuleSum = "caller changed its identity after construction" + plan, err := p.SelectTargets(nil) + if err != nil { + t.Fatal(err) + } + build, err := pipeline.Generate(plan) + if err != nil { + t.Fatal(err) + } + if summary, err := pipeline.Verify(build); err != nil || summary.Status != "passed" { + t.Fatalf("verify generated %s bindings: %v %+v", name, err, summary) + } + releasePath := filepath.Join(build.Targets[0].Directory, releaseName) + document, err := readMapping(releasePath) + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(mapValue(document["provenance"])["profiler"]) + if err != nil { + t.Fatal(err) + } + var recorded ProfilerIdentity + if err := json.Unmarshal(data, &recorded); err != nil { + t.Fatal(err) + } + if err := expected.Match(recorded); err != nil { + t.Fatal("generated profiler identity", err) + } + mutations := []ProfilerIdentity{changedProfiler(expected, func(p *ProfilerIdentity) { p.Version = "v0.6.1" })} + if expected.Kind == "" { + mutations = append(mutations, linked, changedProfiler(expected, func(p *ProfilerIdentity) { p.SHA256 = strings.Repeat("f", 64) })) + } else { + mutations = append(mutations, standalone, changedProfiler(expected, func(p *ProfilerIdentity) { p.ModuleSum = "h1:" + strings.Repeat("B", 42) + "A=" })) + } + for _, tampered := range mutations { + copy := mapValue(jsonCopy(document)) + mapValue(copy["provenance"])["profiler"] = tampered + writeProfilerFixture(t, releasePath, copy) + if _, err := pipeline.Verify(build); err == nil || !strings.Contains(err.Error(), "profiler provenance") { + t.Fatalf("tampered profiler metadata accepted or wrong failure: %+v, %v", tampered, err) + } + } + }) + } +} + +func TestProfilerPipelineReadsReleasedModuleLockAndProtectsCommittedSource(t *testing.T) { + p, o := profilerPipelineProject(t) + standalone, linked := testProfilerIdentities(t) + artifact := Tool{Name: "fixture-tool", Version: "0.1.0", SHA256: strings.Repeat("0", 64)} + lock := map[string]any{ + "apiVersion": "runtimeconditions.io/toolchain-lock/v1alpha1", "kind": "RuntimeConditionsToolchainLock", "status": "released", + "tools": map[string]any{"normalizer": artifact, "orchestrator": artifact, "emitters": map[string]any{"go": artifact}, "profilers": map[string]any{"go": linked}}, + "languages": map[string]any{"go": map[string]any{"languageVersion": p.Catalog.Packages["new-channel"].Languages["go"].LanguageVersion, "packageManager": "go", "tools": map[string]any{"go": "fixture"}}}, + } + writeProfilerFixture(t, p.ToolchainPath, lock) + found, err := DiscoverProject(p.Root, "", "", p.ToolingDirectory) + if err != nil { + t.Fatal(err) + } + if err := linked.Match(found.Toolchain.Tools.Profilers["go"]); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + selected, pin ProfilerIdentity + path, want string + }{ + {"conflict", linked, linked, "explicit-standalone-profiler", "select either"}, + {"wrong option kind", standalone, linked, "", "linked Go module identity"}, + {"unknown identity", changedProfiler(linked, func(p *ProfilerIdentity) { p.ModuleSum = "unknown" }), linked, "", "checksum"}, + {"version mismatch", changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "v0.6.1" }), linked, "", "version differs"}, + {"checksum mismatch", changedProfiler(linked, func(p *ProfilerIdentity) { p.ModuleSum = "h1:" + strings.Repeat("B", 42) + "A=" }), linked, "", "module identity differs"}, + {"artifact lock", linked, standalone, "", "identity kind differs"}, + } { + t.Run(tc.name, func(t *testing.T) { + found.Toolchain.Tools.Profilers["go"] = tc.pin + options := o + options.LinkedGoProfiler, options.GoProfiler = &tc.selected, tc.path + if pipeline, err := NewPipeline(context.Background(), found, options); err == nil || !strings.Contains(err.Error(), tc.want) { + if pipeline != nil { + pipeline.Close() + } + t.Fatalf("invalid selection: %v; want %s", err, tc.want) + } + }) + } + directory := filepath.Join(p.Root, "bindings", "new-channel", "go") + if err := os.MkdirAll(directory, 0755); err != nil { + t.Fatal(err) + } + sentinel := filepath.Join(directory, "keep.go") + if err := os.WriteFile(sentinel, []byte("preserve committed bytes\n"), 0644); err != nil { + t.Fatal(err) + } + wrong := changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "v0.6.1" }) + o.LinkedGoProfiler = &wrong + o.WorkDir = filepath.Join(p.Root, "must-not-create") + var stdout bytes.Buffer + if err := Execute(context.Background(), p.Root, "update", o, &stdout, io.Discard); err == nil || !strings.Contains(err.Error(), "version differs") { + t.Fatalf("update ignored profiler lock: %v", err) + } + if data, err := os.ReadFile(sentinel); err != nil || string(data) != "preserve committed bytes\n" || stdout.Len() != 0 { + t.Fatalf("rejected update affected committed output: %s %v", data, err) + } + if _, err := os.Stat(o.WorkDir); !os.IsNotExist(err) { + t.Fatal("rejected update created work directory") + } +} + +func TestProfilerPipelineNeverFallsBackFromStandaloneSelection(t *testing.T) { + p, o := profilerPipelineProject(t) + _, linked := testProfilerIdentities(t) + for _, pin := range []ProfilerIdentity{{}, linked} { + p.Toolchain.Tools.Profilers = nil + if pin.Kind != "" { + p.Toolchain.Tools.Profilers = map[string]ProfilerIdentity{"go": pin} + } + for _, path := range []string{"", filepath.Join(p.Root, "missing-profiler")} { + o.GoProfiler = path + pipeline, err := NewPipeline(context.Background(), p, o) + if err != nil { + t.Fatal(err) + } + if _, err := pipeline.profiler("go"); err == nil { + t.Fatal("standalone selection silently used another profiler") + } + if pipeline.work != "" { + t.Fatal("missing standalone profiler triggered workspace build") + } + pipeline.Close() + } + } +} + +func TestProfilerPipelineChecksCachedPythonArtifactAgainstLock(t *testing.T) { + p, o := testProject(t, "python") + identity, _ := testProfilerIdentities(t) + identity.Name = "python-rc-profiler" + p.Toolchain.Tools.Profilers = map[string]ProfilerIdentity{"python": identity} + pipeline, err := NewPipeline(context.Background(), p, o) + if err != nil { + t.Fatal(err) + } + defer pipeline.Close() + pipeline.identities["python-profiler"] = Tool{Name: identity.Name, Version: identity.Version, SHA256: identity.SHA256} + if selected, err := pipeline.profiler("python"); err != nil || selected != identity { + t.Fatalf("cached standalone Python identity: %+v %v", selected, err) + } + p.Toolchain.Tools.Profilers["python"] = changedProfiler(identity, func(p *ProfilerIdentity) { p.SHA256 = strings.Repeat("f", 64) }) + if _, err := pipeline.profiler("python"); err == nil { + t.Fatal("cached Python profiler bypassed a mismatched lock") + } +} diff --git a/tooling/extension-bindings/orchestrator/verify.go b/tooling/extension-bindings/orchestrator/verify.go index 5f8da46..94ca77d 100644 --- a/tooling/extension-bindings/orchestrator/verify.go +++ b/tooling/extension-bindings/orchestrator/verify.go @@ -203,9 +203,18 @@ func (p *Pipeline) verifyIdentities(bt *BuiltTarget, build *Build) error { if err != nil { return err } - declared := mapValue(provenance["profiler"]) - if declared["name"] != profiler.Name || declared["version"] != profiler.Version || declared["sha256"] != profiler.SHA256 { - return fmt.Errorf("profiler provenance differs from installed artifact") + data, err := json.Marshal(provenance["profiler"]) + if err != nil { + return err + } + var declaredProfiler ProfilerIdentity + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&declaredProfiler); err != nil { + return fmt.Errorf("invalid profiler provenance: %w", err) + } + if err := declaredProfiler.Match(profiler); err != nil { + return fmt.Errorf("profiler provenance differs from selected profiler: %w", err) } _, tool, err := p.tool("normalizer") if err != nil { @@ -218,7 +227,7 @@ func (p *Pipeline) verifyIdentities(bt *BuiltTarget, build *Build) error { if err != nil { return err } - declared = mapValue(provenance["orchestrator"]) + declared := mapValue(provenance["orchestrator"]) if declared["sha256"] != tool.SHA256 || declared["name"] != tool.Name || declared["version"] != tool.Version { return fmt.Errorf("orchestrator provenance differs from selected artifact") }