From d99f990c800a6ca4f592a278b399108153283e81 Mon Sep 17 00:00:00 2001 From: Colin Lacy Date: Mon, 9 Nov 2026 07:03:47 +0000 Subject: [PATCH 1/7] updates extension listing reference docs --- EXTENSION_RELEASES.md | 104 ++++++++++++++++++++++++--- tooling/extension-bindings/README.md | 19 +++-- 2 files changed, 107 insertions(+), 16 deletions(-) diff --git a/EXTENSION_RELEASES.md b/EXTENSION_RELEASES.md index c9bd1eb..5213175 100644 --- a/EXTENSION_RELEASES.md +++ b/EXTENSION_RELEASES.md @@ -2,17 +2,50 @@ ## Purpose +This contract governs extensions published by Runtime Conditions and the +first-party tools that produce and consume them. It does not add requirements +to the core profile specification or restrict third-party extension identifiers. + An extension release is an immutable semantic vocabulary artifact. Its Runtime Conditions `apiVersion` identifies the document schema understood by validators; it does not identify the extension's semantic release. +The canonical identity rules below are the agreed target for an incremental +tooling migration. Existing definitions and released binding packages still use +artifact URLs as identifiers. Recording this contract does not migrate those +artifacts; the rollout gates below apply before new identities are published. + ## Identity and versioning +- A first-party extension release MUST use a complete, explicitly versioned identifier of the form `https://runtimeconditions.io/extensions/:` or `https://runtimeconditions.io/extensions//:`. +- The complete identifier MUST appear unchanged in the definition's `metadata.id`, profile extension declarations and selectors, dependency references, normalized model identity and ownership fields, and binding or SDK manifest identity fields. +- A generator MUST copy that identifier verbatim. It MUST NOT reconstruct it from a retrieval URL, a native package version, or a separate version annotation. +- `metadata.version` is optional descriptive metadata. When present, it MUST exactly equal the version in the canonical identifier. Tools MUST NOT append it to `metadata.id`. - A published extension identifier MUST resolve permanently to the same semantic definition. - A semantic change MUST produce a new extension identifier. Existing identifiers are never repointed or overwritten. -- Provider extensions SHOULD encode a Semantic Versioning 2.0.0 release in the identifier and repeat it as `metadata.version` so humans and automation can distinguish document schema version from extension semantic version. +- Provider extensions SHOULD use a Semantic Versioning 2.0.0 release. Existing labels such as `v1alpha1` remain valid first-party release versions. - An additive vocabulary or validation change increments the minor version, a meaning-changing removal or incompatible reclassification increments the major version, and a correction that does not change accepted vocabulary or profile meaning increments the patch version. -- During pre-adoption development, `0.x.y` releases may replace earlier experimental conventions without compatibility promises. Before public adoption, the same immutability and compatibility rules apply to every published identifier. +- During pre-adoption development, `0.x.y` releases may introduce incompatible conventions through new identifiers and package versions. Published identifiers, tags, and assets remain immutable. + +Canonical reference examples: + +```yaml +extensions: + - https://runtimeconditions.io/extensions/common-integrations:v1alpha1 + - https://runtimeconditions.io/extensions/env-configuration:v1alpha1 + - https://runtimeconditions.io/extensions/aws/s3:0.2.0 +``` + +A definition for the first reference declares: + +```yaml +metadata: + id: https://runtimeconditions.io/extensions/common-integrations:v1alpha1 + version: v1alpha1 +``` -The AWS S3 candidate therefore uses `https://runtimeconditions.io/aws/aws-s3:0.1.0` while retaining `apiVersion: runtimeconditions.io/v1alpha1` as the Runtime Conditions document schema version. +The `version` annotation may be omitted when authoring a release; `metadata.id` +still supplies its complete identity. Optionality does not permit changing an +already published definition. Native binding package versions and the Runtime +Conditions document `apiVersion` are versioned independently. ## Authoritative provenance @@ -43,10 +76,61 @@ Humans review semantic bridge changes, added or removed operations, resource cla ## HTTPS catalog lookup -A definition uses `metadata.uri` and `metadata.version`; its identifier is -`:`. The canonical URI has `//` or `/`. -The versioned artifact is retrieved from -`https:///extensions////runtimeconditions.extension.yaml`. -Providerless identities use `rc` for lookup without changing the declared URI. -A missing definition is not found; no alternate path or provider is tried. -Only HTTPS retrieval is currently supported. `file:` and `oci:` are deferred. +The canonical identifier and the artifact retrieval URL are separate values. +A first-party catalog or configured resolver maps the complete identifier to +the location of its definition. The `rc` namespace used for providerless entries +inside this repository is not inserted into their canonical identifiers. + +Resolvers MUST preserve the requested identifier for equality, dependency +deduplication, cache identity, cycle detection, and returned catalog keys. A +retrieval mapping MUST NOT replace that identifier with the artifact URL. The +retrieved first-party definition's `metadata.id` MUST match the requested +identifier exactly. Missing mappings, unavailable artifacts, identity mismatches, +and inconsistent version annotations MUST fail resolution. + +The familiar artifact layout is +`https://runtimeconditions.io/extensions///runtimeconditions.extension.yaml` +or +`https://runtimeconditions.io/extensions////runtimeconditions.extension.yaml`. +An existing URL that already identifies a published legacy definition MUST NOT +be overwritten with a definition declaring the new canonical identifier. The +migration must allocate a separate retrieval location in that case and record +its mapping explicitly. + +Support for other retrieval backends remains an implementation capability. This +first-party identity convention does not impose a path rewrite on third-party +identifiers. + +## Incremental migration gates + +Each implementation increment must identify its files and validation before +editing. Consumer support must precede changes to published definition identities. + +1. **Record the tooling contract.** Establish the complete canonical identifier, + optional version annotation, retrieval mapping, and legacy-artifact rules in + this document. This step changes documentation only. +2. **Prepare consumers with synthetic fixtures.** Audit the shared resolver, + binding normalizer and emitters, site publishing, and language profilers. + Verify that canonical identifiers survive resolution, ownership checks, and + profile generation unchanged, including when a version annotation is present + or absent. Existing released URL identifiers must continue to work. Keep + cross-repository compatibility checks outside the shared binding-tooling suite. +3. **Migrate the common/env dependency pair.** Select retrieval locations that + preserve published legacy artifacts. Update the definitions, dependency edge, + catalog entries, generated metadata, and native package versions in bounded + increments. Regenerate bindings through the tooling; do not hand-edit generated + identities or digests. Verify installed consumers before publication. +4. **Update the CLI integration.** After compatible profiler and binding releases + are available, update the required dependency pins and fixtures. Require the + installed CLI to emit the canonical references exactly, while retaining + coverage for workloads using the previous binding releases. +5. **Expand one dependency group at a time.** Apply the same checks to remaining + extensions, SDK mappings, language bindings, profilers, demos, and adapters. + Each group must resolve, generate, and validate before the next group moves. + +For each migrated group, record the previous and canonical identifiers, retrieval +locations, affected dependency edges, artifact digests, package versions, and +consumer release requirements. A new definition identity changes its semantic +digest even when its vocabulary is unchanged; dependent models and bindings must +be regenerated. Publication continues through the existing release verification +and dry-run process. diff --git a/tooling/extension-bindings/README.md b/tooling/extension-bindings/README.md index 363de28..b6afeae 100644 --- a/tooling/extension-bindings/README.md +++ b/tooling/extension-bindings/README.md @@ -9,6 +9,13 @@ GitHub Actions generates and commits Go and Python bindings to `main`, then a separate manual workflow publishes verified Go module tags and release assets. Registry publication of Python bindings remains later work. +The first-party identity contract and migration gates are in +[`EXTENSION_RELEASES.md`](../../EXTENSION_RELEASES.md). The target contract uses +the complete versioned `metadata.id` unchanged throughout resolution, models, +manifests, and generated profiles. Its migration is incremental; existing +definitions and released bindings still use artifact URL identities. The core +profile specification is unchanged by this tooling convention. + ## Automatic Go and Python binding generation `.github/workflows/binding-generate.yml` runs on relevant `main` changes to the @@ -378,12 +385,12 @@ go run ./cmd/rc-binding-model \ --dependency-lock-output ``` -Output paths must not already exist. HTTPS retrieval requires a dependency lock -with an exact source digest and the URL derived from the requested identifier: -`https:///extensions////runtimeconditions.extension.yaml`. -Providerless identities use provider `rc` for retrieval while retaining their -declared identity. HTTPS redirects are rejected; `file:` and `oci:` retrieval -are unsupported. A previously written lock is supplied with `--dependency-lock`. +Output paths must not already exist. HTTPS retrieval records the artifact URL +and exact source digest in the dependency lock. Canonical identity support and +its retrieval mappings must pass the migration gates in +[`EXTENSION_RELEASES.md`](../../EXTENSION_RELEASES.md) before catalog identities +change. An artifact URL used for transport must not replace the requested +identifier. A previously written lock is supplied with `--dependency-lock`. The extension cache is content-addressed. Each extension entry must be named ``, `.yaml`, or From f29f7e7798f976a7fd18d8e8b64a67bed5493d1e Mon Sep 17 00:00:00 2001 From: Colin Lacy Date: Mon, 9 Nov 2026 07:22:22 +0000 Subject: [PATCH 2/7] allows go lib leverage for binding generation --- tooling/extension-bindings/IMPLEMENTATION.md | 77 ++++-- .../extension-bindings/orchestrator/cli.go | 240 ++--------------- .../orchestrator/operations.go | 244 ++++++++++++++++++ .../orchestrator/operations_test.go | 194 ++++++++++++++ 4 files changed, 515 insertions(+), 240 deletions(-) create mode 100644 tooling/extension-bindings/orchestrator/operations.go create mode 100644 tooling/extension-bindings/orchestrator/operations_test.go diff --git a/tooling/extension-bindings/IMPLEMENTATION.md b/tooling/extension-bindings/IMPLEMENTATION.md index 4fed09e..3d6a8e2 100644 --- a/tooling/extension-bindings/IMPLEMENTATION.md +++ b/tooling/extension-bindings/IMPLEMENTATION.md @@ -19,8 +19,8 @@ publication are outside the enabled Go workflows. The binding implementation remains in `extensions`. The downstream common `rc` executable belongs to `rc-cli` and will expose this implementation through a -thin adapter. Language-specific profilers remain independently implemented and -released programs. +thin library adapter. Language-specific profilers remain independently +implemented and released; the Go profiler also exposes a library API. The following requirements apply throughout the pipeline: @@ -716,9 +716,22 @@ rc bindings check rc bindings update ``` -The executable is built from `orchestrator/cmd/rc`. The reusable command -boundary is `orchestrator.Run(context.Context, []string, io.Writer, io.Writer)`; -it accepts the `bindings` command group and returns errors to its caller. +The standalone executable is built from `orchestrator/cmd/rc` and remains +supported independently of the common CLI. Its argument parser is +`orchestrator.Run(context.Context, []string, io.Writer, io.Writer)`. +It accepts the `bindings` command group and delegates to the library boundary: + +```go +orchestrator.Execute(ctx, cwd, operation, options, stdout, stderr) +``` + +`Execute` accepts a context, an absolute working directory, an operation name, +typed `Options`, and non-nil output writers. It parses no command arguments and +does not change the process working directory. Initialize options using +`orchestrator.DefaultOptions()` to obtain the standalone native-tool defaults +from the current environment, then apply explicit overrides. Explicit empty +values are preserved. Relative input, output, override, and executable paths +resolve against the supplied directory; bare tool names retain PATH lookup. The executable supplies process arguments and streams and chooses the exit code. | Command | Behavior and output | @@ -792,21 +805,31 @@ NOT maintain a second binding generator or duplicate compatibility rules. ### 13.2 Binding adapter and distribution -The reusable Go `orchestrator.Run` boundary and standalone command interface -provide integration boundaries. An adapter may invoke a versioned Go API or an -explicitly selected standalone tooling artifact, while preserving the command -contract defined here. - -For an in-process adapter, the orchestrator, normalizer, and Go emitter modules -must be available as a resolvable, versioned dependency set. Relative `replace` -directives used for development inside `extensions` do not supply dependencies -to a downstream module. The schema bundle and required emitter resources must -also be located through an explicit installed tooling path. - -For a subprocess adapter, invoke the selected binding tool with an argument -vector, preserve the caller's directory and streams, and propagate cancellation -and failure. Select its executable by an explicit managed path; dispatch MUST -NOT recursively resolve the common `rc` command as its own backend. +The common CLI MUST import the Go orchestrator and call `Execute` with typed +options from native Cobra commands. It MUST NOT rebuild an argument vector for +`Run` or dispatch to a separate `rc-bindings` executable. Generated SDKs under +`bindings/` are workload dependencies; the generator library lives under +`tooling/extension-bindings/orchestrator`. + +The orchestrator, normalizer, and Go emitter modules must be available as a +resolvable, versioned dependency set. Relative `replace` directives used for +development inside `extensions` do not supply dependencies to a downstream +module. Installed schema and emitter resources must work outside a source +checkout. The common CLI integration will call exported Go normalization and +emission functions directly; Python tooling remains optional for Python targets. + +Standalone generation in `extensions` MUST remain functional. Its entry point, +native-tool options, and ability to select an independently installed Go +profiler executable MUST remain supported. Extracting `Execute` preserves the +existing executable-based tooling and artifact identities. Direct Go tool +execution and linked identities are subsequent integration work. + +Linked profiler support MUST be explicit and optional. Its identity must record +the actual linked module version and dependency checksum. Metadata schemas, +lock validation, and profiler schema consumers must support that identity before +new metadata is emitted. An executable SHA-256 and a module checksum identify +different artifacts; a linked identity MUST NOT satisfy an executable lock. +Explicit standalone profiler selection MUST retain executable identity checks. The CLI integration must retain actual executing-tool provenance and all lock checks. A separately installed CLI version is not a substitute for an emitter, @@ -828,13 +851,15 @@ rc profile go generate rc profile python ``` -The common CLI's default installation includes the independently packaged Go -profiler. Other languages are optional profiler add-ons. Each profiler owns its -source parser, language runtime requirements, flags, and semantic validation. +The common CLI links the independently versioned Go profiler as a Go module. +Native Cobra commands bind typed options and invoke its exported functions; +default and explicit Go routes share the same command constructors. Go help and +flags use Cobra, including `--dir` and `--out`. Other languages are optional +profiler add-ons with their own source parsers and runtime requirements. -After selecting the profiler, the CLI forwards its argument vector without -translating language-specific options. Backend help and output remain owned by -the profiler. Launcher diagnostics must not contaminate profile data on stdout. +Optional profiler commands forward their argument vectors without translating +language-specific options. Their backend help and output remain owned by the +profiler. Launcher diagnostics must not contaminate profile data on stdout. Interactive discovery may use managed installations, explicit configuration, and `PATH`, subject to protocol compatibility checks. Binding verification and diff --git a/tooling/extension-bindings/orchestrator/cli.go b/tooling/extension-bindings/orchestrator/cli.go index 6bbc941..19ac087 100644 --- a/tooling/extension-bindings/orchestrator/cli.go +++ b/tooling/extension-bindings/orchestrator/cli.go @@ -17,6 +17,7 @@ type repeated []string func (r *repeated) String() string { return strings.Join(*r, ",") } func (r *repeated) Set(value string) error { *r = append(*r, value); return nil } +// Options contains operation inputs. Use DefaultOptions for native tool defaults. type Options struct { Packages, ToolchainLock, Cache, WorkDir, ToolingDir string Output, Input, Report, ClosureOutput, DependencyLockOutput, DependencyLock string @@ -24,7 +25,21 @@ type Options struct { Targets, ExtensionRoots, Overrides, CatalogRoots repeated } -// Run is the reusable command boundary; the rc executable only supplies streams. +var supportedOperations = map[string]bool{"resolve": true, "normalize": true, "generate": true, "verify": true, "package": true, "plan-release": true, "check": true, "update": true} + +// DefaultOptions returns standalone tool defaults from the current environment. +// Explicit overrides, including empty values, may be applied before Execute. +func DefaultOptions() Options { + return Options{ + Go: envOr("RC_BINDINGS_GO", "go"), Python: envOr("RC_BINDINGS_PYTHON", "python3"), + Normalizer: os.Getenv("RC_BINDINGS_NORMALIZER"), GoEmitter: os.Getenv("RC_BINDINGS_GO_EMITTER"), + GoProfiler: envOr("RC_GO_PROFILER_BIN", "go-rc-profiler"), + PythonProfiler: envOr("RC_PYTHON_PROFILER_BIN", "runtimeconditions-python-profiler"), + PythonProfilerArtifact: os.Getenv("RC_PYTHON_PROFILER_ARTIFACT"), + } +} + +// Run parses standalone command arguments and delegates execution to Execute. func Run(ctx context.Context, args []string, stdout, stderr io.Writer) error { if len(args) == 0 || args[0] == "--help" || args[0] == "-h" { fmt.Fprintln(stdout, "Usage: rc bindings [flags]") @@ -38,15 +53,14 @@ func Run(ctx context.Context, args []string, stdout, stderr io.Writer) error { return fmt.Errorf("usage: rc bindings [flags]") } name := args[1] - supported := map[string]bool{"resolve": true, "normalize": true, "generate": true, "verify": true, "package": true, "plan-release": true, "check": true, "update": true} if name == "--help" || name == "-h" { fmt.Fprintln(stdout, "Commands: resolve normalize generate verify package plan-release check update") return nil } - if !supported[name] { + if !supportedOperations[name] { return fmt.Errorf("unknown bindings command %q", name) } - var o Options + o := DefaultOptions() f := flag.NewFlagSet("rc bindings "+name, flag.ContinueOnError) f.SetOutput(stderr) f.StringVar(&o.Packages, "packages", "", "package catalog path") @@ -78,13 +92,13 @@ func Run(ctx context.Context, args []string, stdout, stderr io.Writer) error { f.StringVar(&o.Report, "report", "", "write the YAML verification summary here") } f.StringVar(&o.CoreSchema, "core-schema", "", "explicit core schema file for local development") - f.StringVar(&o.Go, "go", envOr("RC_BINDINGS_GO", "go"), "native Go command") - f.StringVar(&o.Python, "python", envOr("RC_BINDINGS_PYTHON", "python3"), "native Python interpreter") - f.StringVar(&o.Normalizer, "normalizer", os.Getenv("RC_BINDINGS_NORMALIZER"), "installed locked normalizer; development defaults to workspace build") - f.StringVar(&o.GoEmitter, "go-emitter", os.Getenv("RC_BINDINGS_GO_EMITTER"), "installed locked Go emitter; development defaults to workspace build") - f.StringVar(&o.GoProfiler, "go-profiler", envOr("RC_GO_PROFILER_BIN", "go-rc-profiler"), "independently installed Go profiler CLI") - f.StringVar(&o.PythonProfiler, "python-profiler", envOr("RC_PYTHON_PROFILER_BIN", "runtimeconditions-python-profiler"), "independently installed Python profiler CLI") - f.StringVar(&o.PythonProfilerArtifact, "python-profiler-artifact", os.Getenv("RC_PYTHON_PROFILER_ARTIFACT"), "installed Python profiler wheel for artifact identity verification") + f.StringVar(&o.Go, "go", o.Go, "native Go command") + f.StringVar(&o.Python, "python", o.Python, "native Python interpreter") + f.StringVar(&o.Normalizer, "normalizer", o.Normalizer, "installed locked normalizer; development defaults to workspace build") + f.StringVar(&o.GoEmitter, "go-emitter", o.GoEmitter, "installed locked Go emitter; development defaults to workspace build") + f.StringVar(&o.GoProfiler, "go-profiler", o.GoProfiler, "independently installed Go profiler CLI") + f.StringVar(&o.PythonProfiler, "python-profiler", o.PythonProfiler, "independently installed Python profiler CLI") + f.StringVar(&o.PythonProfilerArtifact, "python-profiler-artifact", o.PythonProfilerArtifact, "installed Python profiler wheel for artifact identity verification") if err := f.Parse(args[2:]); err != nil { if err == flag.ErrHelp { return nil @@ -94,213 +108,11 @@ func Run(ctx context.Context, args []string, stdout, stderr io.Writer) error { if f.NArg() != 0 { return fmt.Errorf("unexpected positional arguments: %s", strings.Join(f.Args(), " ")) } - if (name == "generate" || name == "package") && o.Output == "" { - return fmt.Errorf("%s requires --output", name) - } cwd, err := os.Getwd() if err != nil { return err } - for _, ptr := range []*string{&o.Cache, &o.WorkDir, &o.Output, &o.Input, &o.Report, &o.ClosureOutput, &o.DependencyLockOutput, &o.DependencyLock, &o.CoreSchema, &o.PythonProfilerArtifact} { - if *ptr != "" { - *ptr = absolute(cwd, *ptr) - } - } - for i, dir := range o.CatalogRoots { - o.CatalogRoots[i] = absolute(cwd, dir) - } - project, err := DiscoverProject(cwd, o.Packages, o.ToolchainLock, o.ToolingDir) - if err != nil { - return err - } - pipeline, err := NewPipeline(ctx, project, o) - if err != nil { - return err - } - defer pipeline.Close() - outputs := []string{o.Output, o.Report, o.ClosureOutput, o.DependencyLockOutput} - seenOutputs := map[string]bool{} - for _, path := range outputs { - if path == "" { - continue - } - if err = rejectSymlinkPath(path); err != nil { - return err - } - if within(filepath.Join(project.RepositoryRoot, "bindings"), path) { - return fmt.Errorf("explicit outputs must be outside committed bindings; use update to synchronize package source") - } - if seenOutputs[path] { - return fmt.Errorf("output paths must be different: %s", path) - } - seenOutputs[path] = true - if name != "generate" && name != "package" { - if _, statErr := os.Lstat(path); !os.IsNotExist(statErr) { - return fmt.Errorf("output must be a new file: %s", path) - } - } - } - plan, err := project.SelectTargets(o.Targets) - if err != nil { - return err - } - if len(o.ExtensionRoots) > 0 && name != "resolve" && name != "normalize" { - roots := map[string]bool{} - for _, root := range o.ExtensionRoots { - roots[root] = true - } - filtered := BuildPlan{Targets: []Target{}} - foundRoots := map[string]bool{} - for _, target := range plan.Targets { - if roots[target.Set.RootExtension] { - filtered.Targets = append(filtered.Targets, target) - foundRoots[target.Set.RootExtension] = true - } - } - for root := range foundRoots { - delete(roots, root) - } - if len(roots) != 0 { - return fmt.Errorf("extension roots require package delivery metadata in the selected catalog targets: %s", strings.Join(sortedKeys(roots), ", ")) - } - plan = filtered - } - switch name { - case "resolve", "normalize": - documents, err := pipeline.Resolve(plan) - if err != nil { - return err - } - for i, doc := range documents { - if i > 0 { - fmt.Fprintln(stdout, "---") - } - var data []byte - if name == "resolve" { - data, err = yaml.Marshal(doc.SemanticClosure()) - } else { - if err = pipeline.Normalize(doc); err == nil { - data, err = doc.ModelYAML() - } - } - if err != nil { - return err - } - if _, err = stdout.Write(data); err != nil { - return err - } - if name == "resolve" { - lock, err := doc.LockYAML() - if err != nil { - return err - } - fmt.Fprintln(stdout, "---") - if _, err = stdout.Write(lock); err != nil { - return err - } - } - } - if o.ClosureOutput != "" || o.DependencyLockOutput != "" || o.Output != "" { - if len(documents) != 1 { - return fmt.Errorf("file outputs require one extension root; use --extension-root or --target") - } - doc := documents[0] - if o.ClosureOutput != "" { - data, _ := yaml.Marshal(doc.SemanticClosure()) - if err = writeNewFile(o.ClosureOutput, data); err != nil { - return err - } - } - if o.Output != "" { - data, err := doc.ModelYAML() - if err != nil { - return err - } - if err = writeNewFile(o.Output, data); err != nil { - return err - } - } - if o.DependencyLockOutput != "" { - data, err := doc.LockYAML() - if err != nil { - return err - } - if err = writeNewFile(o.DependencyLockOutput, data); err != nil { - return err - } - } - } - return nil - case "generate": - if err = validateEmptyDirectory(o.Output); err != nil { - return err - } - build, err := pipeline.Generate(plan) - if err != nil { - return err - } - if err = prepareEmptyDirectory(o.Output); err != nil { - return err - } - return build.CopySelected(o.Output) - case "plan-release": - release, err := pipeline.PlanRelease(plan) - if err != nil { - return err - } - data, err := yaml.Marshal(release) - if err != nil { - return err - } - if _, err = stdout.Write(data); err != nil { - return err - } - if o.Output != "" { - return writeNewFile(o.Output, data) - } - return nil - case "verify", "check", "package", "update": - var build *Build - if name == "verify" { - build, err = pipeline.LoadBuild(plan, o.Input) - } else { - build, err = pipeline.Generate(plan) - } - if err != nil { - failure := VerificationSummary{Status: "failed", Targets: []TargetResult{}} - failure.Fail("build", err) - return writeSummary(name, o, failure, err, stdout, stderr) - } - summary, err := pipeline.Verify(build) - if err == nil && (name == "check" || name == "package") { - err = build.CompareCommitted() - if err != nil { - summary.Fail("committed-source", err) - } - } - if err == nil && name == "update" { - err = build.Update() - if err != nil { - summary.Fail("update", err) - } - } - if err == nil && name == "package" { - committed, loadErr := pipeline.LoadBuild(plan, "") - if loadErr != nil { - err = loadErr - } else { - _, err = pipeline.Verify(committed) - if err == nil { - err = pipeline.Package(committed, o.Output) - } - } - if err != nil { - summary.Fail("package", err) - } - } - return writeSummary(name, o, summary, err, stdout, stderr) - } - return fmt.Errorf("unhandled bindings command %s", name) + return Execute(ctx, cwd, name, o, stdout, stderr) } func writeSummary(name string, o Options, summary VerificationSummary, result error, stdout, stderr io.Writer) error { diff --git a/tooling/extension-bindings/orchestrator/operations.go b/tooling/extension-bindings/orchestrator/operations.go new file mode 100644 index 0000000..a578a1c --- /dev/null +++ b/tooling/extension-bindings/orchestrator/operations.go @@ -0,0 +1,244 @@ +package orchestrator + +import ( + "context" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "gopkg.in/yaml.v3" +) + +// Execute runs one binding operation without parsing arguments or changing the +// process working directory. cwd must be absolute; relative option paths resolve +// against it. Initialize options with DefaultOptions to use standalone tooling +// defaults, then apply caller overrides. Writers must be non-nil. +func Execute(ctx context.Context, cwd, name string, o Options, stdout, stderr io.Writer) error { + if !supportedOperations[name] { + return fmt.Errorf("unknown bindings command %q", name) + } + if err := ctx.Err(); err != nil { + return err + } + if !filepath.IsAbs(cwd) { + return fmt.Errorf("working directory must be absolute") + } + if (name == "generate" || name == "package") && o.Output == "" { + return fmt.Errorf("%s requires --output", name) + } + // Options is passed by value; copy the slice before resolving its paths. + o.CatalogRoots = append(repeated(nil), o.CatalogRoots...) + o.Overrides = append(repeated(nil), o.Overrides...) + for i, entry := range o.Overrides { + if key, path, ok := strings.Cut(entry, "="); ok && path != "" { + o.Overrides[i] = key + "=" + absolute(cwd, path) + } + } + for _, ptr := range []*string{&o.Go, &o.Python, &o.Normalizer, &o.GoEmitter, &o.GoProfiler, &o.PythonProfiler} { + if strings.ContainsAny(*ptr, "/\\") { + *ptr = absolute(cwd, *ptr) + } + } + for _, ptr := range []*string{&o.Cache, &o.WorkDir, &o.Output, &o.Input, &o.Report, &o.ClosureOutput, &o.DependencyLockOutput, &o.DependencyLock, &o.CoreSchema, &o.PythonProfilerArtifact} { + if *ptr != "" { + *ptr = absolute(cwd, *ptr) + } + } + for i, dir := range o.CatalogRoots { + o.CatalogRoots[i] = absolute(cwd, dir) + } + project, err := DiscoverProject(cwd, o.Packages, o.ToolchainLock, o.ToolingDir) + if err != nil { + return err + } + pipeline, err := NewPipeline(ctx, project, o) + if err != nil { + return err + } + defer pipeline.Close() + outputs := []string{o.Output, o.Report, o.ClosureOutput, o.DependencyLockOutput} + seenOutputs := map[string]bool{} + for _, path := range outputs { + if path == "" { + continue + } + if err = rejectSymlinkPath(path); err != nil { + return err + } + if within(filepath.Join(project.RepositoryRoot, "bindings"), path) { + return fmt.Errorf("explicit outputs must be outside committed bindings; use update to synchronize package source") + } + if seenOutputs[path] { + return fmt.Errorf("output paths must be different: %s", path) + } + seenOutputs[path] = true + if name != "generate" && name != "package" { + if _, statErr := os.Lstat(path); !os.IsNotExist(statErr) { + return fmt.Errorf("output must be a new file: %s", path) + } + } + } + plan, err := project.SelectTargets(o.Targets) + if err != nil { + return err + } + if len(o.ExtensionRoots) > 0 && name != "resolve" && name != "normalize" { + roots := map[string]bool{} + for _, root := range o.ExtensionRoots { + roots[root] = true + } + filtered := BuildPlan{Targets: []Target{}} + foundRoots := map[string]bool{} + for _, target := range plan.Targets { + if roots[target.Set.RootExtension] { + filtered.Targets = append(filtered.Targets, target) + foundRoots[target.Set.RootExtension] = true + } + } + for root := range foundRoots { + delete(roots, root) + } + if len(roots) != 0 { + return fmt.Errorf("extension roots require package delivery metadata in the selected catalog targets: %s", strings.Join(sortedKeys(roots), ", ")) + } + plan = filtered + } + switch name { + case "resolve", "normalize": + documents, err := pipeline.Resolve(plan) + if err != nil { + return err + } + for i, doc := range documents { + if i > 0 { + fmt.Fprintln(stdout, "---") + } + var data []byte + if name == "resolve" { + data, err = yaml.Marshal(doc.SemanticClosure()) + } else { + if err = pipeline.Normalize(doc); err == nil { + data, err = doc.ModelYAML() + } + } + if err != nil { + return err + } + if _, err = stdout.Write(data); err != nil { + return err + } + if name == "resolve" { + lock, err := doc.LockYAML() + if err != nil { + return err + } + fmt.Fprintln(stdout, "---") + if _, err = stdout.Write(lock); err != nil { + return err + } + } + } + if o.ClosureOutput != "" || o.DependencyLockOutput != "" || o.Output != "" { + if len(documents) != 1 { + return fmt.Errorf("file outputs require one extension root; use --extension-root or --target") + } + doc := documents[0] + if o.ClosureOutput != "" { + data, _ := yaml.Marshal(doc.SemanticClosure()) + if err = writeNewFile(o.ClosureOutput, data); err != nil { + return err + } + } + if o.Output != "" { + data, err := doc.ModelYAML() + if err != nil { + return err + } + if err = writeNewFile(o.Output, data); err != nil { + return err + } + } + if o.DependencyLockOutput != "" { + data, err := doc.LockYAML() + if err != nil { + return err + } + if err = writeNewFile(o.DependencyLockOutput, data); err != nil { + return err + } + } + } + return nil + case "generate": + if err = validateEmptyDirectory(o.Output); err != nil { + return err + } + build, err := pipeline.Generate(plan) + if err != nil { + return err + } + if err = prepareEmptyDirectory(o.Output); err != nil { + return err + } + return build.CopySelected(o.Output) + case "plan-release": + release, err := pipeline.PlanRelease(plan) + if err != nil { + return err + } + data, err := yaml.Marshal(release) + if err != nil { + return err + } + if _, err = stdout.Write(data); err != nil { + return err + } + if o.Output != "" { + return writeNewFile(o.Output, data) + } + return nil + case "verify", "check", "package", "update": + var build *Build + if name == "verify" { + build, err = pipeline.LoadBuild(plan, o.Input) + } else { + build, err = pipeline.Generate(plan) + } + if err != nil { + failure := VerificationSummary{Status: "failed", Targets: []TargetResult{}} + failure.Fail("build", err) + return writeSummary(name, o, failure, err, stdout, stderr) + } + summary, err := pipeline.Verify(build) + if err == nil && (name == "check" || name == "package") { + err = build.CompareCommitted() + if err != nil { + summary.Fail("committed-source", err) + } + } + if err == nil && name == "update" { + err = build.Update() + if err != nil { + summary.Fail("update", err) + } + } + if err == nil && name == "package" { + committed, loadErr := pipeline.LoadBuild(plan, "") + if loadErr != nil { + err = loadErr + } else { + _, err = pipeline.Verify(committed) + if err == nil { + err = pipeline.Package(committed, o.Output) + } + } + if err != nil { + summary.Fail("package", err) + } + } + return writeSummary(name, o, summary, err, stdout, stderr) + } + return fmt.Errorf("unhandled bindings command %s", name) +} diff --git a/tooling/extension-bindings/orchestrator/operations_test.go b/tooling/extension-bindings/orchestrator/operations_test.go new file mode 100644 index 0000000..9a7b11f --- /dev/null +++ b/tooling/extension-bindings/orchestrator/operations_test.go @@ -0,0 +1,194 @@ +package orchestrator + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" +) + +func TestDefaultOptionsRetainStandaloneToolSelection(t *testing.T) { + for key, value := range map[string]string{ + "RC_BINDINGS_GO": "/tools/go", "RC_BINDINGS_PYTHON": "/tools/python", + "RC_BINDINGS_NORMALIZER": "/tools/normalizer", "RC_BINDINGS_GO_EMITTER": "/tools/emitter", + "RC_GO_PROFILER_BIN": "/tools/go-profiler", "RC_PYTHON_PROFILER_BIN": "/tools/python-profiler", + "RC_PYTHON_PROFILER_ARTIFACT": "/tools/profiler.whl", + } { + t.Setenv(key, value) + } + o := DefaultOptions() + if o.Go != "/tools/go" || o.Python != "/tools/python" || o.Normalizer != "/tools/normalizer" || + o.GoEmitter != "/tools/emitter" || o.GoProfiler != "/tools/go-profiler" || + o.PythonProfiler != "/tools/python-profiler" || o.PythonProfilerArtifact != "/tools/profiler.whl" { + t.Fatalf("standalone defaults: %+v", o) + } + t.Setenv("RC_BINDINGS_GO", "") + t.Setenv("RC_BINDINGS_PYTHON", "") + t.Setenv("RC_GO_PROFILER_BIN", "") + t.Setenv("RC_PYTHON_PROFILER_BIN", "") + o = DefaultOptions() + if o.Go != "go" || o.Python != "python3" || o.GoProfiler != "go-rc-profiler" || o.PythonProfiler != "runtimeconditions-python-profiler" { + t.Fatalf("native command defaults: %+v", o) + } +} + +func TestExecuteResolveAndNormalizeMatchStandalone(t *testing.T) { + p, defaults := testProject(t, "go") + nested := filepath.Join(p.Root, "nested working directory") + if err := os.Mkdir(nested, 0755); err != nil { + t.Fatal(err) + } + core := filepath.Join(p.Root, "core.yaml") + if err := os.WriteFile(core, []byte("$id: https://runtimeconditions.io/schemas/profile/0.2.0/runtimeconditions.profile.schema.yaml\ntype: object\n"), 0644); err != nil { + t.Fatal(err) + } + normalizer := filepath.Join(t.TempDir(), "normalizer") + command := exec.Command(defaults.Go, "build", "-trimpath", "-buildvcs=false", "-o", normalizer, "./cmd/rc-binding-model") + command.Dir = filepath.Join(p.ToolingDirectory, "normalizer") + if data, err := command.CombinedOutput(); err != nil { + t.Fatalf("build normalizer: %v\n%s", err, data) + } + relativeNormalizer, err := filepath.Rel(nested, normalizer) + if err != nil { + t.Fatal(err) + } + processCWD, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + for _, name := range []string{"resolve", "normalize"} { + t.Run(name, func(t *testing.T) { + o := defaults + o.Packages, o.ToolchainLock = "../packages.yaml", "../toolchain.lock.yaml" + o.CoreSchema, o.Normalizer = "../core.yaml", relativeNormalizer + o.CatalogRoots = []string{"../catalog"} + o.Overrides = []string{p.Catalog.Packages["new-channel"].RootExtension + "=../catalog/" + extensionName} + o.Targets = []string{"new-channel:go"} + o.DependencyLockOutput = "../" + name + " API lock.yaml" + apiFile, cliFile := filepath.Join(p.Root, name+" API.yaml"), filepath.Join(p.Root, name+" CLI.yaml") + cliLock := filepath.Join(p.Root, name+" CLI lock.yaml") + args := []string{"bindings", name, "--packages", p.PackagesPath, "--toolchain-lock", p.ToolchainPath, + "--tooling-dir", p.ToolingDirectory, "--catalog-root", filepath.Join(p.Root, "catalog"), + "--target", "new-channel:go", "--go", o.Go, + "--normalizer", normalizer, "--core-schema", core, "--dependency-lock-output", cliLock} + args = append(args, "--extension-override", p.Catalog.Packages["new-channel"].RootExtension+"="+filepath.Join(p.Root, "catalog", extensionName)) + if name == "resolve" { + o.ClosureOutput = "../" + name + " API.yaml" + args = append(args, "--closure-output", cliFile) + } else { + o.Output = "../" + name + " API.yaml" + args = append(args, "--output", cliFile) + } + var apiOut, apiErr, cliOut, cliErr bytes.Buffer + if err := Execute(context.Background(), nested, name, o, &apiOut, &apiErr); err != nil { + t.Fatal(err) + } + if err := Run(context.Background(), args, &cliOut, &cliErr); err != nil { + t.Fatal(err) + } + if !bytes.Equal(apiOut.Bytes(), cliOut.Bytes()) || !bytes.Equal(apiErr.Bytes(), cliErr.Bytes()) { + t.Fatalf("API and standalone streams differ\nAPI: %s\nCLI: %s", apiOut.Bytes(), cliOut.Bytes()) + } + if !strings.Contains(apiOut.String(), "future.channel") { + t.Fatal("resolved vocabulary missing from YAML") + } + for _, pair := range [][2]string{{apiFile, cliFile}, {filepath.Join(nested, o.DependencyLockOutput), cliLock}} { + first, err := os.ReadFile(pair[0]) + if err != nil { + t.Fatal(err) + } + second, err := os.ReadFile(pair[1]) + if err != nil || !bytes.Equal(first, second) { + t.Fatalf("API and standalone files differ: %v", err) + } + } + if o.CatalogRoots[0] != "../catalog" || o.CoreSchema != "../core.yaml" || o.Packages != "../packages.yaml" { + t.Fatalf("Execute mutated caller options: %+v", o) + } + if !strings.Contains(o.Overrides[0], "=../catalog/") || o.Normalizer != relativeNormalizer { + t.Fatalf("Execute mutated caller tool/override paths: %+v", o) + } + }) + } + if current, err := os.Getwd(); err != nil || current != processCWD { + t.Fatalf("process directory changed: %s %v", current, err) + } + if _, err := os.Stat(filepath.Join(p.Root, "bindings")); !os.IsNotExist(err) { + t.Fatal("resolve/normalize created committed binding trees") + } +} + +func TestExecuteValidationMatchesStandalone(t *testing.T) { + p, defaults := testProject(t, "go") + directory := filepath.Join(p.Root, "keep") + if err := os.Mkdir(directory, 0755); err != nil { + t.Fatal(err) + } + sentinel := filepath.Join(directory, "existing.yaml") + if err := os.WriteFile(sentinel, []byte("preserve existing bytes\n"), 0644); err != nil { + t.Fatal(err) + } + for _, tc := range []struct{ name, operation, flag, path, want string }{ + {"existing file", "resolve", "--closure-output", sentinel, "output must be a new file"}, + {"nonempty directory", "generate", "--output", directory, "output must be new or empty"}, + {"committed source", "resolve", "--closure-output", filepath.Join(p.Root, "bindings", "forbidden.yaml"), "explicit outputs must be outside committed bindings"}, + {"duplicate targets", "resolve", "--target", "new-channel:go", "duplicate target new-channel:go"}, + } { + t.Run(tc.name, func(t *testing.T) { + o := defaults + o.Packages, o.ToolchainLock = p.PackagesPath, p.ToolchainPath + if tc.flag == "--output" { + o.Output = tc.path + } else if tc.flag == "--target" { + o.Targets = []string{tc.path, tc.path} + } else { + o.ClosureOutput = tc.path + } + var apiOut, apiErr, cliOut, cliErr bytes.Buffer + apiResult := Execute(context.Background(), p.Root, tc.operation, o, &apiOut, &apiErr) + args := []string{"bindings", tc.operation, "--packages", p.PackagesPath, + "--toolchain-lock", p.ToolchainPath, "--tooling-dir", p.ToolingDirectory, tc.flag, tc.path} + if tc.flag == "--target" { + args = append(args, tc.flag, tc.path) + } + cliResult := Run(context.Background(), args, &cliOut, &cliErr) + if apiResult == nil || cliResult == nil || apiResult.Error() != cliResult.Error() || !strings.Contains(apiResult.Error(), tc.want) { + t.Fatalf("API error %v; standalone error %v; want %s", apiResult, cliResult, tc.want) + } + if apiOut.Len() != 0 || cliOut.Len() != 0 || !bytes.Equal(apiErr.Bytes(), cliErr.Bytes()) { + t.Fatal("output validation changed streams") + } + if data, err := os.ReadFile(sentinel); err != nil || string(data) != "preserve existing bytes\n" { + t.Fatalf("existing output changed: %s %v", data, err) + } + }) + } +} + +func TestExecuteRejectsInvalidRequestsBeforeDiscovery(t *testing.T) { + root := t.TempDir() + for _, tc := range []struct{ cwd, operation, want string }{ + {root, "unknown", "unknown bindings command"}, + {"relative", "resolve", "working directory must be absolute"}, + {root, "generate", "generate requires --output"}, + {root, "package", "package requires --output"}, + } { + err := Execute(context.Background(), tc.cwd, tc.operation, DefaultOptions(), io.Discard, io.Discard) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("%s: %v; want %s", tc.operation, err, tc.want) + } + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if err := Execute(ctx, root, "resolve", DefaultOptions(), io.Discard, io.Discard); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation: %v", err) + } + if entries, err := os.ReadDir(root); err != nil || len(entries) != 0 { + t.Fatalf("invalid requests created files: %v %v", entries, err) + } +} From b1615cdfbf0e869aa3ba0344a1c51ff45c21d3b5 Mon Sep 17 00:00:00 2001 From: Colin Lacy Date: Mon, 9 Nov 2026 07:45:14 +0000 Subject: [PATCH 3/7] allows bindings to identify profiler, modifies ext resolve --- ...timeconditions.binding-release.schema.yaml | 23 +- ...ntimeconditions.toolchain-lock.schema.yaml | 35 ++- .../normalizer/resolve_test.go | 128 +++++++++- .../orchestrator/profiler_identity.go | 111 +++++++++ .../orchestrator/profiler_identity_test.go | 228 ++++++++++++++++++ 5 files changed, 519 insertions(+), 6 deletions(-) create mode 100644 tooling/extension-bindings/orchestrator/profiler_identity.go create mode 100644 tooling/extension-bindings/orchestrator/profiler_identity_test.go diff --git a/tooling/extension-bindings/model/runtimeconditions.binding-release.schema.yaml b/tooling/extension-bindings/model/runtimeconditions.binding-release.schema.yaml index 8ab4fa8..b41921b 100644 --- a/tooling/extension-bindings/model/runtimeconditions.binding-release.schema.yaml +++ b/tooling/extension-bindings/model/runtimeconditions.binding-release.schema.yaml @@ -106,6 +106,25 @@ $defs: name: {$ref: '#/$defs/nonEmptyString'} version: {$ref: '#/$defs/nonEmptyString'} sha256: {$ref: '#/$defs/sha256'} + profilerIdentity: + oneOf: + - {$ref: '#/$defs/toolIdentity'} + - {$ref: '#/$defs/linkedGoProfilerIdentity'} + linkedGoProfilerIdentity: + description: Exact linked Go profiler dependency, distinct from an executable digest. + type: object + additionalProperties: false + required: [kind, name, version, module, moduleSum] + properties: + kind: {const: go-module} + name: {const: go-rc-profiler} + version: + type: string + pattern: '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(\.(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*)?(\+incompatible)?$' + module: {const: github.com/runtimeconditions/go-rc-profiler} + moduleSum: + type: string + pattern: '^h1:[A-Za-z0-9+/]{42}[AEIMQUYcgkosw048]=$' package: description: >- Matches the emitter target and runtimeconditions.bindings.yaml package. @@ -256,7 +275,7 @@ $defs: properties: mode: {const: test-fixture} fixtureAssembler: {$ref: '#/$defs/toolIdentity'} - profiler: {$ref: '#/$defs/toolIdentity'} + profiler: {$ref: '#/$defs/profilerIdentity'} productionProvenance: description: >- Phase 5 and later committed package assembly. Source directory and tag @@ -268,7 +287,7 @@ $defs: properties: mode: {const: production} orchestrator: {$ref: '#/$defs/toolIdentity'} - profiler: {$ref: '#/$defs/toolIdentity'} + profiler: {$ref: '#/$defs/profilerIdentity'} sourceDirectory: type: string pattern: '^bindings/[a-z0-9]+(?:-[a-z0-9]+)*/(go|python)$' diff --git a/tooling/extension-bindings/model/runtimeconditions.toolchain-lock.schema.yaml b/tooling/extension-bindings/model/runtimeconditions.toolchain-lock.schema.yaml index 072789a..cbc04cb 100644 --- a/tooling/extension-bindings/model/runtimeconditions.toolchain-lock.schema.yaml +++ b/tooling/extension-bindings/model/runtimeconditions.toolchain-lock.schema.yaml @@ -4,8 +4,10 @@ title: Runtime Conditions toolchain lock description: >- Development may use workspace tools. A released lock pins normalizer, orchestrator, emitters and independently installed profilers by version and - artifact SHA-256. Consumers must check that each selected language has both - an emitter and a profiler and that the actual artifacts match their digests. + artifact SHA-256. Go profiler pins may instead identify a linked module by + exact version and dependency checksum. Consumers must check that each selected + language has an emitter and profiler and that the selected identity form, + version and checksum match the lock without substituting another artifact kind. type: object additionalProperties: false required: [apiVersion, kind, status] @@ -22,7 +24,7 @@ properties: normalizer: {$ref: '#/$defs/toolIdentity'} orchestrator: {$ref: '#/$defs/toolIdentity'} emitters: {$ref: '#/$defs/languageTools'} - profilers: {$ref: '#/$defs/languageTools'} + profilers: {$ref: '#/$defs/profilerTools'} languages: type: object propertyNames: @@ -64,6 +66,33 @@ $defs: description: Optional retrieval URL for the exact checksummed released artifact. type: string pattern: '^https://[^\s]+$' + profilerIdentity: + oneOf: + - {$ref: '#/$defs/toolIdentity'} + - {$ref: '#/$defs/linkedGoProfilerIdentity'} + linkedGoProfilerIdentity: + description: Exact linked Go profiler dependency, distinct from an executable digest. + type: object + additionalProperties: false + required: [kind, name, version, module, moduleSum] + properties: + kind: {const: go-module} + name: {const: go-rc-profiler} + version: + type: string + pattern: '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(\.(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*)?(\+incompatible)?$' + module: {const: github.com/runtimeconditions/go-rc-profiler} + moduleSum: + type: string + pattern: '^h1:[A-Za-z0-9+/]{42}[AEIMQUYcgkosw048]=$' + profilerTools: + type: object + minProperties: 1 + propertyNames: + pattern: '^[a-z][a-z0-9-]*$' + properties: + go: {$ref: '#/$defs/profilerIdentity'} + additionalProperties: {$ref: '#/$defs/toolIdentity'} languageTools: type: object minProperties: 1 diff --git a/tooling/extension-bindings/normalizer/resolve_test.go b/tooling/extension-bindings/normalizer/resolve_test.go index 52490d0..f949a68 100644 --- a/tooling/extension-bindings/normalizer/resolve_test.go +++ b/tooling/extension-bindings/normalizer/resolve_test.go @@ -348,7 +348,7 @@ func TestResolverBackendsProduceOneSemanticModel(t *testing.T) { writeFixture(t, cache, sourceDigest+".yaml", data) schemas := testSchemas(t) - httpsLocator := id + httpsLocator := "https://artifacts.example.test/definitions/backend.yaml" configs := map[string]ResolverConfig{ "override": {Schemas: schemas, Overrides: map[string]string{id: path}}, @@ -438,6 +438,132 @@ func TestDevelopmentHTTPSRecordsLockAndExplicitCache(t *testing.T) { } } +type mappedHTTPTransport struct { + documents map[string][]byte + requests []string +} + +func (transport *mappedHTTPTransport) RoundTrip(request *http.Request) (*http.Response, error) { + locator := request.URL.String() + transport.requests = append(transport.requests, locator) + data, exists := transport.documents[locator] + if !exists { + return nil, fmt.Errorf("unexpected retrieval URL %q", locator) + } + return (staticHTTPTransport{data: data}).RoundTrip(request) +} + +func TestLockedHTTPSCanonicalIDsSurviveNormalization(t *testing.T) { + const rootID = "https://runtimeconditions.io/extensions/env-configuration:v1alpha1" + const dependencyID = "https://runtimeconditions.io/extensions/common-integrations:v1alpha1" + const rootURL = "https://artifacts.example.test/canonical/env/definition.yaml" + const dependencyURL = "https://artifacts.example.test/canonical/common/definition.yaml" + root := bytes.Replace(fixtureExtension(rootID, []string{dependencyID}, kindSpec("fixture.env")), []byte("version: 1.0.0"), []byte("version: v1alpha1"), 1) + dependency := bytes.Replace(fixtureExtension(dependencyID, nil, kindSpec("fixture.common")), []byte(" version: 1.0.0\n"), nil, 1) + transport := &mappedHTTPTransport{documents: map[string][]byte{rootURL: root, dependencyURL: dependency}} + config := ResolverConfig{ + Schemas: testSchemas(t), Network: true, HTTPClient: &http.Client{Transport: transport}, + Locks: map[string]LockEntry{ + rootID: {SourceSHA256: SHA256Hex(root), SourceBackend: "https", Locator: rootURL}, + dependencyID: {SourceSHA256: SHA256Hex(dependency), SourceBackend: "https", Locator: dependencyURL}, + }, + } + resolver, err := NewResolver(config) + if err != nil { + t.Fatal(err) + } + closure, err := resolver.Resolve(context.Background(), rootID) + if err != nil { + t.Fatal(err) + } + if got := strings.Join(transport.requests, "\n"); got != rootURL+"\n"+dependencyURL { + t.Fatalf("retrieved URLs = %q", got) + } + if closure.Root != rootID || closure.ByID[rootID].Definition.Metadata.ID != rootID || closure.ByID[dependencyID].Definition.Metadata.ID != dependencyID { + t.Fatal("retrieval URLs replaced canonical closure identifiers") + } + lock := BuildDependencyLock(closure) + model, err := Normalize(closure, lock, config.Schemas, testNormalizeConfig()) + if err != nil { + t.Fatal(err) + } + if model.RootExtension.ID != rootID || model.RootExtension.Version != "v1alpha1" || len(model.Extensions) != 2 || len(lock.Extensions) != 2 { + t.Fatal("canonical root or extension inventory changed during normalization") + } + for i, id := range []string{dependencyID, rootID} { + if model.Extensions[i].ID != id || lock.Extensions[i].ID != id || lock.Extensions[i].SourceLocator != config.Locks[id].Locator { + t.Fatalf("model or lock lost identity/location separation for %s", id) + } + } + if model.Extensions[0].Version != "" || len(model.DependencyEdges) != 1 || model.DependencyEdges[0] != (DependencyEdge{From: rootID, To: dependencyID}) { + t.Fatal("dependency identity, omitted version, or dependency edge changed") + } + owners := map[string]string{"fixture.env": rootID, "fixture.common": dependencyID} + if len(model.Vocabulary.Owners) != len(owners) { + t.Fatal("normalized vocabulary owner inventory changed") + } + for _, owner := range model.Vocabulary.Owners { + if owner.Owner != owners[owner.Kind] { + t.Fatalf("vocabulary owner changed to %q", owner.Owner) + } + } +} + +func TestLockedHTTPSMappedDefinitionValidation(t *testing.T) { + const id = "https://runtimeconditions.io/extensions/example:1.0.0" + const locator = "https://artifacts.example.test/example/definition.yaml" + for _, test := range []struct { + name, definitionID, sourceDigest, semanticDigest, code string + }{ + {"identity mismatch", "https://runtimeconditions.io/extensions/other:1.0.0", "", "", "RCB1220"}, + {"source digest mismatch", id, strings.Repeat("0", 64), "", "RCB1212"}, + {"semantic digest mismatch", id, "", strings.Repeat("f", 64), "RCB1213"}, + } { + t.Run(test.name, func(t *testing.T) { + data := fixtureExtension(test.definitionID, nil, kindSpec("fixture.example")) + digest := test.sourceDigest + if digest == "" { + digest = SHA256Hex(data) + } + transport := &mappedHTTPTransport{documents: map[string][]byte{locator: data}} + resolver, err := NewResolver(ResolverConfig{ + Schemas: testSchemas(t), Network: true, HTTPClient: &http.Client{Transport: transport}, + Locks: map[string]LockEntry{id: {SourceSHA256: digest, SemanticSHA256: test.semanticDigest, Locator: locator}}, + }) + if err != nil { + t.Fatal(err) + } + _, err = resolver.Resolve(context.Background(), id) + if err == nil || diagnosticCode(t, err) != test.code { + t.Fatalf("expected %s, got %v", test.code, err) + } + if len(transport.requests) != 1 || transport.requests[0] != locator { + t.Fatalf("retrieved URLs = %v", transport.requests) + } + }) + } +} + +func TestLockedHTTPSRejectsInvalidRetrievalURLs(t *testing.T) { + const id = "https://runtimeconditions.io/extensions/example:1.0.0" + for _, locator := range []string{"http://artifacts.example.test/definition.yaml", "file:///definition.yaml", "https:///missing-host.yaml", "https://user:password@artifacts.example.test/definition.yaml", "https://artifacts.example.test/%invalid"} { + t.Run(locator, func(t *testing.T) { + transport := &countingTransport{} + resolver, err := NewResolver(ResolverConfig{ + Schemas: testSchemas(t), Network: true, HTTPClient: &http.Client{Transport: transport}, + Locks: map[string]LockEntry{id: {SourceSHA256: strings.Repeat("a", 64), Locator: locator}}, + }) + if err != nil { + t.Fatal(err) + } + _, err = resolver.Resolve(context.Background(), id) + if err == nil || diagnosticCode(t, err) != "RCB1207" || transport.requests.Load() != 0 { + t.Fatalf("invalid retrieval URL reached transport or lacked RCB1207: %v", err) + } + }) + } +} + type countingTransport struct { requests atomic.Int64 } diff --git a/tooling/extension-bindings/orchestrator/profiler_identity.go b/tooling/extension-bindings/orchestrator/profiler_identity.go new file mode 100644 index 0000000..a4121d2 --- /dev/null +++ b/tooling/extension-bindings/orchestrator/profiler_identity.go @@ -0,0 +1,111 @@ +package orchestrator + +import ( + "fmt" + "regexp" +) + +const linkedGoProfilerModule = "github.com/runtimeconditions/go-rc-profiler" +const linkedGoProfilerKind = "go-module" + +var profilerArtifactDigest = regexp.MustCompile(`^[0-9a-f]{64}$`) +var profilerArtifactURL = regexp.MustCompile(`^https://[^\s]+$`) +var profilerModuleSum = regexp.MustCompile(`^h1:[A-Za-z0-9+/]{42}[AEIMQUYcgkosw048]=$`) +var profilerModuleVersion = regexp.MustCompile(`^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*)(\.(0|[1-9][0-9]*|[0-9]*[A-Za-z-][0-9A-Za-z-]*))*)?(\+incompatible)?$`) + +// ProfilerIdentity describes either a standalone artifact or the linked Go +// profiler module. Standalone identities omit Kind and retain the legacy +// name/version/sha256 representation. Linked identities require kind: go-module +// and module/moduleSum, and cannot contain sha256 or artifactUrl. +// These helpers define the contract; pipeline selection is integrated separately. +type ProfilerIdentity struct { + Name string `json:"name" yaml:"name"` + Version string `json:"version" yaml:"version"` + SHA256 string `json:"sha256,omitempty" yaml:"sha256,omitempty"` + ArtifactURL string `json:"artifactUrl,omitempty" yaml:"artifactUrl,omitempty"` + Kind string `json:"kind,omitempty" yaml:"kind,omitempty"` + Module string `json:"module,omitempty" yaml:"module,omitempty"` + ModuleSum string `json:"moduleSum,omitempty" yaml:"moduleSum,omitempty"` +} + +// NewStandaloneProfilerIdentity validates the existing artifact identity without +// changing its version or digest. ArtifactURL is optional lock retrieval data. +func NewStandaloneProfilerIdentity(tool Tool) (ProfilerIdentity, error) { + identity := ProfilerIdentity{Name: tool.Name, Version: tool.Version, SHA256: tool.SHA256, ArtifactURL: tool.ArtifactURL} + return identity, identity.Validate() +} + +// NewLinkedGoProfilerIdentity validates an exact module version and Go h1 +// dependency checksum supplied by the host's verified build metadata. It does +// not inspect an executable, resolve a module, or install any tooling. +func NewLinkedGoProfilerIdentity(version, moduleSum string) (ProfilerIdentity, error) { + identity := ProfilerIdentity{ + Name: "go-rc-profiler", Version: version, Kind: linkedGoProfilerKind, + Module: linkedGoProfilerModule, ModuleSum: moduleSum, + } + return identity, identity.Validate() +} + +// Validate checks a complete identity, rejecting mixed artifact/module fields. +// Validate document schemas before decoding to reject unknown input fields too. +func (identity ProfilerIdentity) Validate() error { + if identity.Name == "" || identity.Version == "" { + return fmt.Errorf("profiler identity requires name and version") + } + switch identity.Kind { + case "": + if identity.Module != "" || identity.ModuleSum != "" { + return fmt.Errorf("standalone profiler identity cannot contain module fields") + } + if !profilerArtifactDigest.MatchString(identity.SHA256) { + return fmt.Errorf("standalone profiler identity requires a lowercase artifact SHA-256") + } + if identity.ArtifactURL != "" && !profilerArtifactURL.MatchString(identity.ArtifactURL) { + return fmt.Errorf("profiler artifactUrl must be an HTTPS URL without whitespace") + } + case linkedGoProfilerKind: + if identity.SHA256 != "" || identity.ArtifactURL != "" { + return fmt.Errorf("linked profiler identity cannot contain artifact fields") + } + if identity.Name != "go-rc-profiler" || identity.Module != linkedGoProfilerModule { + return fmt.Errorf("linked profiler identity requires the Go profiler name and module path") + } + if !profilerModuleVersion.MatchString(identity.Version) { + return fmt.Errorf("linked profiler identity requires an exact Go module version") + } + if !profilerModuleSum.MatchString(identity.ModuleSum) { + return fmt.Errorf("linked profiler identity requires a canonical Go h1 dependency checksum") + } + default: + return fmt.Errorf("unknown profiler identity kind %q", identity.Kind) + } + return nil +} + +// Match requires the selected profiler to have exactly the identity pinned by +// the receiver. Retrieval URLs do not identify artifact contents. Neither an +// incomplete lock nor a different identity kind can satisfy this comparison. +func (expected ProfilerIdentity) Match(selected ProfilerIdentity) error { + if err := expected.Validate(); err != nil { + return fmt.Errorf("invalid locked profiler identity: %w", err) + } + if err := selected.Validate(); err != nil { + return fmt.Errorf("invalid selected profiler identity: %w", err) + } + if expected.Kind != selected.Kind { + return fmt.Errorf("profiler identity kind differs from lock") + } + if expected.Name != selected.Name { + return fmt.Errorf("profiler name differs from lock") + } + if expected.Version != selected.Version { + return fmt.Errorf("profiler version differs from lock") + } + if expected.SHA256 != selected.SHA256 { + return fmt.Errorf("profiler artifact digest differs from lock") + } + if expected.Module != selected.Module || expected.ModuleSum != selected.ModuleSum { + return fmt.Errorf("profiler module identity differs from lock") + } + return nil +} diff --git a/tooling/extension-bindings/orchestrator/profiler_identity_test.go b/tooling/extension-bindings/orchestrator/profiler_identity_test.go new file mode 100644 index 0000000..0a6483b --- /dev/null +++ b/tooling/extension-bindings/orchestrator/profiler_identity_test.go @@ -0,0 +1,228 @@ +package orchestrator + +import ( + "bytes" + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func testProfilerIdentities(t *testing.T) (ProfilerIdentity, ProfilerIdentity) { + t.Helper() + standalone, err := NewStandaloneProfilerIdentity(Tool{Name: "go-rc-profiler", Version: "v0.6.0", SHA256: strings.Repeat("0", 64)}) + if err != nil { + t.Fatal(err) + } + linked, err := NewLinkedGoProfilerIdentity("v0.6.0", "h1:"+strings.Repeat("A", 43)+"=") + if err != nil { + t.Fatal(err) + } + return standalone, linked +} + +func changedProfiler(identity ProfilerIdentity, change func(*ProfilerIdentity)) ProfilerIdentity { + change(&identity) + return identity +} + +func profilerIdentityMapping(t *testing.T, identity ProfilerIdentity) map[string]any { + t.Helper() + data, err := json.Marshal(identity) + if err != nil { + t.Fatal(err) + } + var result map[string]any + if err := json.Unmarshal(data, &result); err != nil { + t.Fatal(err) + } + return result +} + +func testProfilerSchemas(t *testing.T) map[string]map[string]any { + t.Helper() + schemas := map[string]map[string]any{} + for name, path := range map[string]string{ + "release": filepath.Join("..", "model", "runtimeconditions.binding-release.schema.yaml"), + "lock": filepath.Join("..", "model", "runtimeconditions.toolchain-lock.schema.yaml"), + "Go profiler": filepath.Join("..", "..", "..", "..", "go-rc-profiler", "extensioncheck", "schema", "runtimeconditions.binding-release.schema.yaml"), + } { + schema, err := readMapping(path) + if name == "Go profiler" && os.IsNotExist(err) { + t.Log("sibling Go profiler checkout absent; embedded schema comparison omitted") + continue + } + if err != nil { + t.Fatal(err) + } + schemas[name] = schema + } + return schemas +} + +func TestStandaloneProfilerIdentityPreservesLegacyEncoding(t *testing.T) { + tool := Tool{Name: "go-rc-profiler", Version: "v0.6.0", SHA256: strings.Repeat("a", 64), ArtifactURL: "https://example.test/profiler.tar.gz"} + identity, err := NewStandaloneProfilerIdentity(tool) + if err != nil { + t.Fatal(err) + } + for _, marshal := range []func(any) ([]byte, error){json.Marshal, yaml.Marshal} { + before, err := marshal(tool) + if err != nil { + t.Fatal(err) + } + after, err := marshal(identity) + if err != nil || !bytes.Equal(before, after) { + t.Fatalf("legacy identity encoding changed: %s -> %s (%v)", before, after, err) + } + } + if _, err := NewStandaloneProfilerIdentity(Tool{}); err == nil { + t.Fatal("empty standalone identity accepted") + } + if _, err := NewLinkedGoProfilerIdentity("unknown", ""); err == nil { + t.Fatal("unknown linked identity accepted") + } +} + +func TestProfilerIdentityValidationAndSchemaAgreement(t *testing.T) { + standalone, linked := testProfilerIdentities(t) + schemas := testProfilerSchemas(t) + for name, schema := range schemas { + for _, definition := range []string{"profilerIdentity", "linkedGoProfilerIdentity"} { + if !reflect.DeepEqual(mapValue(schema["$defs"])[definition], mapValue(schemas["release"]["$defs"])[definition]) { + t.Fatalf("%s disagrees on %s", name, definition) + } + } + } + for _, tc := range []struct { + name string + identity ProfilerIdentity + valid bool + }{ + {"standalone", standalone, true}, {"linked", linked, true}, + {"prerelease", changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "v0.6.1-rc.1" }), true}, + {"pseudo version", changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "v0.6.1-0.20261109120000-0123456789ab" }), true}, + {"empty", ProfilerIdentity{}, false}, + {"missing name", changedProfiler(standalone, func(p *ProfilerIdentity) { p.Name = "" }), false}, + {"missing version", changedProfiler(standalone, func(p *ProfilerIdentity) { p.Version = "" }), false}, + {"uppercase digest", changedProfiler(standalone, func(p *ProfilerIdentity) { p.SHA256 = strings.Repeat("A", 64) }), false}, + {"short digest", changedProfiler(standalone, func(p *ProfilerIdentity) { p.SHA256 = "1234" }), false}, + {"invalid retrieval URL", changedProfiler(standalone, func(p *ProfilerIdentity) { p.ArtifactURL = "http://example.test/profiler" }), false}, + {"artifact with module", changedProfiler(standalone, func(p *ProfilerIdentity) { p.Module = linked.Module }), false}, + {"artifact with module sum", changedProfiler(standalone, func(p *ProfilerIdentity) { p.ModuleSum = linked.ModuleSum }), false}, + {"unknown kind", changedProfiler(linked, func(p *ProfilerIdentity) { p.Kind = "other" }), false}, + {"wrong profiler", changedProfiler(linked, func(p *ProfilerIdentity) { p.Name = "python-rc-profiler" }), false}, + {"wrong module", changedProfiler(linked, func(p *ProfilerIdentity) { p.Module = "example.test/profiler" }), false}, + {"linked with artifact", changedProfiler(linked, func(p *ProfilerIdentity) { p.SHA256 = standalone.SHA256 }), false}, + {"linked with URL", changedProfiler(linked, func(p *ProfilerIdentity) { p.ArtifactURL = "https://example.test/profiler" }), false}, + {"unknown version", changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "unknown" }), false}, + {"short version", changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "v0.6" }), false}, + {"leading zero", changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "v0.6.0-rc.01" }), false}, + {"noncanonical version", changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "v0.6.0+local" }), false}, + {"hex module sum", changedProfiler(linked, func(p *ProfilerIdentity) { p.ModuleSum = standalone.SHA256 }), false}, + {"noncanonical padding", changedProfiler(linked, func(p *ProfilerIdentity) { p.ModuleSum = "h1:" + strings.Repeat("A", 42) + "B=" }), false}, + {"checksum newline", changedProfiler(linked, func(p *ProfilerIdentity) { p.ModuleSum += "\n" }), false}, + {"missing checksum", changedProfiler(linked, func(p *ProfilerIdentity) { p.ModuleSum = "" }), false}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := tc.identity.Validate(); (err == nil) != tc.valid { + t.Fatalf("identity validity: %v; want %t", err, tc.valid) + } + for name, schema := range schemas { + identitySchema := map[string]any{"$ref": "#/$defs/profilerIdentity", "$defs": schema["$defs"]} + if err := validateSchema(identitySchema, profilerIdentityMapping(t, tc.identity)); (err == nil) != tc.valid { + t.Errorf("%s schema validity: %v; want %t", name, err, tc.valid) + } + } + }) + } + for name, schema := range schemas { + value := profilerIdentityMapping(t, linked) + value["unknown"] = "must reject" + if err := validateSchema(map[string]any{"$ref": "#/$defs/profilerIdentity", "$defs": schema["$defs"]}, value); err == nil { + t.Errorf("%s accepted an unknown identity field", name) + } + } +} + +func TestProfilerIdentityMatchRequiresExactKindVersionAndChecksum(t *testing.T) { + standalone, linked := testProfilerIdentities(t) + for _, tc := range []struct { + name string + expected, selected ProfilerIdentity + match bool + }{ + {"artifact", standalone, standalone, true}, {"module", linked, linked, true}, + {"retrieval URL", standalone, changedProfiler(standalone, func(p *ProfilerIdentity) { p.ArtifactURL = "https://example.test/mirror" }), true}, + {"name", standalone, changedProfiler(standalone, func(p *ProfilerIdentity) { p.Name = "other" }), false}, + {"artifact version", standalone, changedProfiler(standalone, func(p *ProfilerIdentity) { p.Version = "v0.6.1" }), false}, + {"module version", linked, changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "v0.6.1" }), false}, + {"digest", standalone, changedProfiler(standalone, func(p *ProfilerIdentity) { p.SHA256 = strings.Repeat("1", 64) }), false}, + {"module sum", linked, changedProfiler(linked, func(p *ProfilerIdentity) { p.ModuleSum = "h1:" + strings.Repeat("B", 42) + "A=" }), false}, + {"module for executable lock", standalone, linked, false}, {"executable for module lock", linked, standalone, false}, + {"incomplete lock", ProfilerIdentity{}, linked, false}, {"incomplete selection", linked, ProfilerIdentity{}, false}, + } { + t.Run(tc.name, func(t *testing.T) { + if err := tc.expected.Match(tc.selected); (err == nil) != tc.match { + t.Fatalf("match: %v; want %t", err, tc.match) + } + }) + } +} + +func TestProfilerIdentityInReleaseAndLockDocuments(t *testing.T) { + standalone, linked := testProfilerIdentities(t) + schemas := testProfilerSchemas(t) + for name, schema := range schemas { + if name == "lock" { + continue + } + doc, err := readMapping(filepath.Join("..", "..", "..", "bindings", "env-configuration", "go", releaseName)) + if err != nil { + t.Fatal(err) + } + if err := validateSchema(schema, doc); err != nil { + t.Fatalf("%s rejected legacy release: %v", name, err) + } + mapValue(doc["provenance"])["profiler"] = profilerIdentityMapping(t, linked) + if err := validateSchema(schema, doc); err != nil { + t.Fatalf("%s rejected linked release: %v", name, err) + } + pkg := mapValue(doc["package"]) + pkg["language"], pkg["minimumPythonVersion"] = "python", "3.11" + delete(pkg, "minimumGoVersion") + for _, dependency := range sequence(doc["packageDependencies"]) { + mapValue(mapValue(dependency)["artifact"])["kind"] = "python-wheel" + } + mapValue(doc["provenance"])["profiler"] = profilerIdentityMapping(t, changedProfiler(standalone, func(p *ProfilerIdentity) { p.Name = "python-rc-profiler" })) + if err := validateSchema(schema, doc); err != nil { + t.Fatalf("%s rejected Python release: %v", name, err) + } + mapValue(doc["provenance"])["profiler"] = profilerIdentityMapping(t, linked) + if err := validateSchema(schema, doc); err == nil { + t.Fatalf("%s accepted linked Go profiler for Python", name) + } + } + artifact := profilerIdentityMapping(t, standalone) + profilers := map[string]any{"go": artifact} + doc := map[string]any{ + "apiVersion": "runtimeconditions.io/toolchain-lock/v1alpha1", "kind": "RuntimeConditionsToolchainLock", "status": "released", + "tools": map[string]any{"normalizer": artifact, "orchestrator": artifact, "emitters": map[string]any{"go": artifact}, "profilers": profilers}, + "languages": map[string]any{"go": map[string]any{"languageVersion": "1.25.0", "packageManager": "go", "tools": map[string]any{"go": "1.25.0"}}}, + } + if err := validateSchema(schemas["lock"], doc); err != nil { + t.Fatal("legacy lock rejected", err) + } + profilers["go"] = profilerIdentityMapping(t, linked) + if err := validateSchema(schemas["lock"], doc); err != nil { + t.Fatal("linked Go lock rejected", err) + } + profilers["python"] = profilerIdentityMapping(t, linked) + if err := validateSchema(schemas["lock"], doc); err == nil { + t.Fatal("linked Go identity accepted for Python lock") + } +} From b6aa090c87e67eeed18f8def62570cde816f1663 Mon Sep 17 00:00:00 2001 From: Colin Lacy Date: Mon, 9 Nov 2026 08:06:46 +0000 Subject: [PATCH 4/7] sets strict extension path id resolution --- EXTENSION_RELEASES.md | 90 ++++++++++----- tooling/common/build_extensions_site.py | 104 ++++++++++++----- tooling/common/test_build_extensions_site.py | 112 ++++++++++++++++++- 3 files changed, 252 insertions(+), 54 deletions(-) diff --git a/EXTENSION_RELEASES.md b/EXTENSION_RELEASES.md index 5213175..7d446b8 100644 --- a/EXTENSION_RELEASES.md +++ b/EXTENSION_RELEASES.md @@ -1,20 +1,21 @@ -# Extension release and compatibility contract +# First-party extension release contract ## Purpose This contract governs extensions published by Runtime Conditions and the -first-party tools that produce and consume them. It does not add requirements -to the core profile specification or restrict third-party extension identifiers. +first-party tools that produce and consume them. It defines the reference forms +supported by that tooling; the core profile specification remains unchanged. An extension release is an immutable semantic vocabulary artifact. Its Runtime Conditions `apiVersion` identifies the document schema understood by validators; it does not identify the extension's semantic release. -The canonical identity rules below are the agreed target for an incremental -tooling migration. Existing definitions and released binding packages still use -artifact URLs as identifiers. Recording this contract does not migrate those -artifacts; the rollout gates below apply before new identities are published. +The canonical identity rules below are the target for an incremental tooling +migration. Inputs that still use artifact URLs as identifiers must be migrated. +First-party tooling MUST reject unsupported reference forms rather than retain +legacy acceptance, aliases, automatic conversion, or fallback behavior. ## Identity and versioning +- First-party tooling MUST accept and emit only complete `:` extension references. Unversioned identifiers and artifact-URL extension references MUST be rejected. - A first-party extension release MUST use a complete, explicitly versioned identifier of the form `https://runtimeconditions.io/extensions/:` or `https://runtimeconditions.io/extensions//:`. - The complete identifier MUST appear unchanged in the definition's `metadata.id`, profile extension declarations and selectors, dependency references, normalized model identity and ownership fields, and binding or SDK manifest identity fields. - A generator MUST copy that identifier verbatim. It MUST NOT reconstruct it from a retrieval URL, a native package version, or a separate version annotation. @@ -88,18 +89,52 @@ retrieved first-party definition's `metadata.id` MUST match the requested identifier exactly. Missing mappings, unavailable artifacts, identity mismatches, and inconsistent version annotations MUST fail resolution. -The familiar artifact layout is -`https://runtimeconditions.io/extensions///runtimeconditions.extension.yaml` +The first-party publisher accepts only the canonical identifier forms in this +contract. A providerless identifier omits the catalog's internal `rc` namespace. +The artifact layout is +`https://runtimeconditions.io/extensions//releases//runtimeconditions.extension.yaml` or -`https://runtimeconditions.io/extensions////runtimeconditions.extension.yaml`. -An existing URL that already identifies a published legacy definition MUST NOT -be overwritten with a definition declaring the new canonical identifier. The -migration must allocate a separate retrieval location in that case and record -its mapping explicitly. +`https://runtimeconditions.io/extensions///releases//runtimeconditions.extension.yaml`. +Each version occupies one URL path segment and is percent-encoded there when +necessary. Its exact spelling in the canonical identifier remains unchanged. -Support for other retrieval backends remains an implementation capability. This -first-party identity convention does not impose a path rewrite on third-party -identifiers. +For example, the identifier +`https://runtimeconditions.io/extensions/common-integrations:v1alpha1` maps to +`https://runtimeconditions.io/extensions/common-integrations/releases/v1alpha1/runtimeconditions.extension.yaml`. +The publisher copies the definition bytes unchanged. It does not create artifact +URL identities, legacy routes, or aliases. Definition IDs and dependency +references must be canonical; an optional `metadata.version` must match the +version in the definition ID. + +### Generated lookup index + +The publisher writes `https://runtimeconditions.io/extensions/index.yaml`: + +```yaml +apiVersion: runtimeconditions.io/extension-index/v1alpha1 +kind: RuntimeConditionsExtensionIndex +extensions: + - id: https://runtimeconditions.io/extensions/common-integrations:v1alpha1 + url: https://runtimeconditions.io/extensions/common-integrations/releases/v1alpha1/runtimeconditions.extension.yaml + sourceSha256: +``` + +Entries are sorted by their exact `id` strings. Each canonical ID appears once; +`url` is its retrieval location and `sourceSha256` is the lowercase hexadecimal +SHA-256 of its published bytes. This generated first-party lookup contract does +not extend the core profile format. Consumers must explicitly integrate the +lookup entries into their retrieval mappings or dependency locks; writing the +index does not change consumer discovery behavior by itself. + +The publisher MUST reject unsupported IDs or dependency reference forms, +inconsistent version annotations, duplicate IDs, and conflicting output paths +before writing artifacts. Canonical definitions and the lookup index are the +only extension publication outputs. Existing Go package discovery pages remain +a separate publication output. + +Support for other retrieval backends remains an implementation capability. +First-party tools must keep the complete versioned reference as the identity +regardless of how the artifact is retrieved. ## Incremental migration gates @@ -107,30 +142,35 @@ Each implementation increment must identify its files and validation before editing. Consumer support must precede changes to published definition identities. 1. **Record the tooling contract.** Establish the complete canonical identifier, - optional version annotation, retrieval mapping, and legacy-artifact rules in + optional version annotation, retrieval mapping, and strict rejection rules in this document. This step changes documentation only. 2. **Prepare consumers with synthetic fixtures.** Audit the shared resolver, binding normalizer and emitters, site publishing, and language profilers. Verify that canonical identifiers survive resolution, ownership checks, and profile generation unchanged, including when a version annotation is present - or absent. Existing released URL identifiers must continue to work. Keep + or absent. Verify that artifact-URL extension references are rejected. Keep cross-repository compatibility checks outside the shared binding-tooling suite. -3. **Migrate the common/env dependency pair.** Select retrieval locations that - preserve published legacy artifacts. Update the definitions, dependency edge, - catalog entries, generated metadata, and native package versions in bounded +3. **Migrate the common/env dependency pair.** Replace artifact-URL identities + with canonical references and use the canonical publication layout. Update + the definitions, dependency edge, catalog entries, generated metadata, and native package versions in bounded increments. Regenerate bindings through the tooling; do not hand-edit generated identities or digests. Verify installed consumers before publication. 4. **Update the CLI integration.** After compatible profiler and binding releases are available, update the required dependency pins and fixtures. Require the - installed CLI to emit the canonical references exactly, while retaining - coverage for workloads using the previous binding releases. + installed CLI to emit the canonical references exactly and reject inputs + that still expose artifact-URL extension references. 5. **Expand one dependency group at a time.** Apply the same checks to remaining extensions, SDK mappings, language bindings, profilers, demos, and adapters. Each group must resolve, generate, and validate before the next group moves. -For each migrated group, record the previous and canonical identifiers, retrieval +For each migrated group, record the canonical identifiers, retrieval locations, affected dependency edges, artifact digests, package versions, and consumer release requirements. A new definition identity changes its semantic digest even when its vocabulary is unchanged; dependent models and bindings must be regenerated. Publication continues through the existing release verification and dry-run process. + +Incremental implementation does not authorize compatibility branches. The +current catalog cannot pass strict site assembly until its definitions and +dependency references have been migrated. Complete that migration and the +required consumer updates before deploying the resulting site. diff --git a/tooling/common/build_extensions_site.py b/tooling/common/build_extensions_site.py index d6d9e11..001e36a 100644 --- a/tooling/common/build_extensions_site.py +++ b/tooling/common/build_extensions_site.py @@ -1,29 +1,43 @@ -"""Build a static site tree so every catalog extension is reachable at its -own `metadata.id` URL. - -Each extension's id is the URL it's meant to be resolved at, e.g. -https://runtimeconditions.io/extensions/aws-s3/0.1.0/runtimeconditions.extension.yaml. -This copies each extension.yaml to the output directory at the path portion -of that URL, so publishing the output directory as a static site (GitHub -Pages or otherwise) makes every id resolvable by itself - no separate -mapping to keep in sync. Generated Go modules also receive discovery pages -at their public module coordinates from the package catalog. +"""Publish canonical extension definitions, their lookup index, and Go discovery. + +Only complete : extension references are supported. Definition +bytes are published unchanged beneath release paths; extensions/index.yaml maps +their exact identities to retrieval URLs and source checksums. """ from __future__ import annotations import argparse -import shutil +import re import sys +from hashlib import sha256 from pathlib import Path from html import escape -from urllib.parse import urlparse +from urllib.parse import quote + +import yaml from go_bindings import GoBinding, load_bindings, validate_source from serialization import read_document REPO_ROOT = Path(__file__).resolve().parents[2] EXPECTED_HOST = "runtimeconditions.io" +CANONICAL_REFERENCE = re.compile( + rf"https://{re.escape(EXPECTED_HOST)}/extensions/" + r"([A-Za-z0-9][A-Za-z0-9._-]*(?:/[A-Za-z0-9][A-Za-z0-9._-]*)?):([^/?#\s]+)" +) + + +def canonical_parts(reference: str) -> tuple[str, str]: + match = CANONICAL_REFERENCE.fullmatch(reference) if isinstance(reference, str) else None + if match is None: + raise ValueError(f"expected canonical extension reference :, got {reference!r}") + location, version = match.groups() + if location.startswith("rc/"): + raise ValueError("canonical providerless references must omit the internal rc namespace") + if version in {".", ".."} or "\\" in version or any(ord(char) < 32 or ord(char) == 127 for char in version): + raise ValueError(f"invalid canonical extension version in {reference!r}") + return location, version def build_go_pages(output_dir: Path, root: Path) -> None: @@ -96,31 +110,69 @@ def build(output_dir: Path, root: Path = REPO_ROOT) -> int: return 1 errors = [] + entries = {} + outputs = {} for extension_path in extension_paths: - doc = read_document(extension_path) - extension_id = doc.get("metadata", {}).get("id", "") - parsed = urlparse(extension_id) relative = extension_path.relative_to(root) - - if parsed.scheme != "https" or parsed.netloc != EXPECTED_HOST: - errors.append(f"{relative}: metadata.id {extension_id!r} is not an https://{EXPECTED_HOST}/... URL") - continue - - destination = output_dir / parsed.path.lstrip("/") - destination.parent.mkdir(parents=True, exist_ok=True) - shutil.copyfile(extension_path, destination) - print(f"{relative} -> {parsed.path}") + try: + doc = read_document(extension_path) + metadata = doc.get("metadata") + if doc.get("kind") != "RuntimeConditionsExtensionDefinition" or not isinstance(metadata, dict): + raise ValueError("expected extension definition with metadata object") + extension_id = metadata.get("id") + location, version = canonical_parts(extension_id) + if "version" in metadata and metadata["version"] != version: + raise ValueError("metadata.version must exactly match the canonical reference version") + spec = doc.get("spec", {}) + if not isinstance(spec, dict) or not isinstance(spec.get("dependencies", []), list): + raise ValueError("expected spec object with a dependencies array when present") + for dependency in spec.get("dependencies", []): + canonical_parts(dependency) + destination = Path("extensions", *location.split("/"), "releases", version, "runtimeconditions.extension.yaml") + if extension_id in entries: + raise ValueError(f"duplicate extension id {extension_id!r}") + if destination in outputs: + raise ValueError(f"conflicting output path {destination}") + source = extension_path.read_bytes() + outputs[destination] = source + entries[extension_id] = { + "id": extension_id, + "url": f"https://{EXPECTED_HOST}/" + quote(destination.as_posix(), safe="/"), + "sourceSha256": sha256(source).hexdigest(), + } + except (ValueError, OSError) as error: + errors.append(f"{relative}: {error}") + + index = { + "apiVersion": "runtimeconditions.io/extension-index/v1alpha1", + "kind": "RuntimeConditionsExtensionIndex", + "extensions": [entries[identifier] for identifier in sorted(entries)], + } + outputs[Path("extensions/index.yaml")] = yaml.safe_dump( + index, sort_keys=False, allow_unicode=True, width=4096 + ).encode("utf-8") + for relative, source in outputs.items(): + destination = output_dir / relative + if (not destination.resolve().is_relative_to(output_dir.resolve()) + or any(parent.exists() and not parent.is_dir() for parent in destination.parents) + or destination.exists() and (not destination.is_file() or destination.read_bytes() != source)): + errors.append(f"conflicting output path {relative}") if errors: for error in errors: print(error, file=sys.stderr) - print(f"\n{len(errors)} extension(s) have an id that can't be published this way.", file=sys.stderr) + print(f"\nExtension publication failed with {len(errors)} error(s).", file=sys.stderr) return 1 try: build_go_pages(output_dir, root) + for relative, source in outputs.items(): + destination = output_dir / relative + destination.parent.mkdir(parents=True, exist_ok=True) + destination.write_bytes(source) + print(f"extension publication -> /{relative.as_posix()}") except (ValueError, KeyError, OSError) as error: - print(f"Go binding discovery failed: {error}", file=sys.stderr) + print(f"Site assembly failed: {error}", file=sys.stderr) return 1 return 0 diff --git a/tooling/common/test_build_extensions_site.py b/tooling/common/test_build_extensions_site.py index c590006..241fd3c 100644 --- a/tooling/common/test_build_extensions_site.py +++ b/tooling/common/test_build_extensions_site.py @@ -2,7 +2,10 @@ from __future__ import annotations +from contextlib import redirect_stderr +from hashlib import sha256 from html.parser import HTMLParser +from io import StringIO from pathlib import Path import tempfile import unittest @@ -43,10 +46,20 @@ def setUp(self) -> None: self.write_catalog() self.definition = self.root / "catalog/vendor/service/releases/0.1.0/runtimeconditions.extension.yaml" self.definition.parent.mkdir(parents=True) - self.definition.write_text(yaml.safe_dump({"kind": "RuntimeConditionsExtensionDefinition", "metadata": { - "id": "https://runtimeconditions.io/extensions/vendor/service/0.1.0/runtimeconditions.extension.yaml"}})) + self.document = {"kind": "RuntimeConditionsExtensionDefinition", "metadata": { + "id": "https://runtimeconditions.io/extensions/vendor/service:0.1.0", "version": "0.1.0"}} + self.write_definition() self.write_source() + def write_definition(self) -> None: + self.definition.write_text(yaml.safe_dump(self.document), encoding="utf-8") + + def assert_rejected(self, message: str) -> None: + errors = StringIO() + with redirect_stderr(errors): + self.assertEqual(build(self.output, self.root), 1) + self.assertIn(message, errors.getvalue()) + def write_catalog(self) -> None: path = self.root / "tooling/extension-bindings/packages.yaml" path.parent.mkdir(parents=True, exist_ok=True) @@ -69,8 +82,101 @@ def test_site_contains_discovery_install_and_unaliased_import(self) -> None: self.assertIn("go get runtimeconditions.io/x/vendor/service@v0.1.0", page) self.assertFalse((self.output / "x/vendor/pending").exists()) self.assertIn("runtimeconditions.io/x/vendor/service", (self.output / "x/index.html").read_text()) - published = self.output / "extensions/vendor/service/0.1.0/runtimeconditions.extension.yaml" + published = self.output / "extensions/vendor/service/releases/0.1.0/runtimeconditions.extension.yaml" self.assertEqual(published.read_bytes(), self.definition.read_bytes()) + index = yaml.safe_load((self.output / "extensions/index.yaml").read_text()) + self.assertEqual(index, { + "apiVersion": "runtimeconditions.io/extension-index/v1alpha1", + "kind": "RuntimeConditionsExtensionIndex", + "extensions": [{ + "id": self.document["metadata"]["id"], + "url": "https://runtimeconditions.io/extensions/vendor/service/releases/0.1.0/runtimeconditions.extension.yaml", + "sourceSha256": sha256(published.read_bytes()).hexdigest(), + }], + }) + self.assertFalse((self.output / "extensions/vendor/service:0.1.0").exists()) + self.assertFalse((self.output / "extensions/vendor/service/0.1.0").exists()) + + def test_providerless_and_digest_versions_publish_exact_ids(self) -> None: + for version in ("v1alpha1", "sha256:abc123"): + with self.subTest(version=version): + reference = f"https://runtimeconditions.io/extensions/common-integrations:{version}" + self.document["metadata"] = {"id": reference} + self.write_definition() + output = self.output / version + self.assertEqual(build(output, self.root), 0) + artifact = output / f"extensions/common-integrations/releases/{version}/runtimeconditions.extension.yaml" + self.assertEqual(artifact.read_bytes(), self.definition.read_bytes()) + entry = yaml.safe_load((output / "extensions/index.yaml").read_text())["extensions"][0] + self.assertEqual(entry["id"], reference) + encoded = version.replace(":", "%3A") + self.assertEqual(entry["url"], f"https://runtimeconditions.io/extensions/common-integrations/releases/{encoded}/runtimeconditions.extension.yaml") + + def test_legacy_and_invalid_references_are_rejected_before_writing(self) -> None: + for reference in ( + "https://runtimeconditions.io/extensions/vendor/service/0.1.0/runtimeconditions.extension.yaml", + "https://runtimeconditions.io/extensions/vendor/service", + "https://runtimeconditions.io/extensions/vendor/service:", + "https://runtimeconditions.io/extensions/rc/service:0.1.0", + "https://runtimeconditions.io/extensions/vendor/service:..", + "https://runtimeconditions.io/extensions/vendor/../escape:1", + "https://runtimeconditions.io/extensions/vendor/service:1?query=value", + "https://runtimeconditions.io/extensions/vendor/service:1#fragment", + "https://runtimeconditions.io/extensions/vendor/service:1\x00", + "http://runtimeconditions.io/extensions/vendor/service:1", + "https://other.example/extensions/vendor/service:1", + None, + ): + with self.subTest(reference=reference): + self.document["metadata"]["id"] = reference + self.write_definition() + self.assert_rejected("canonical") + self.assertFalse(self.output.exists()) + + def test_version_annotation_must_match_canonical_id(self) -> None: + for version in ("0.2.0", "", None, 1): + with self.subTest(version=version): + self.document["metadata"]["version"] = version + self.write_definition() + self.assert_rejected("metadata.version") + self.assertFalse(self.output.exists()) + + def test_legacy_dependency_reference_is_rejected(self) -> None: + self.document["spec"] = {"dependencies": [ + "https://runtimeconditions.io/extensions/common-integrations/v1alpha1/runtimeconditions.extension.yaml"]} + self.write_definition() + self.assert_rejected("canonical") + self.assertFalse(self.output.exists()) + + def test_duplicate_id_is_rejected_before_writing(self) -> None: + duplicate = self.root / "catalog/vendor/duplicate/releases/0.1.0/runtimeconditions.extension.yaml" + duplicate.parent.mkdir(parents=True) + duplicate.write_bytes(self.definition.read_bytes()) + self.assert_rejected("duplicate extension id") + self.assertFalse(self.output.exists()) + + def test_conflicting_artifact_path_is_rejected_before_writing(self) -> None: + artifact = self.output / "extensions/vendor/service/releases/0.1.0/runtimeconditions.extension.yaml" + artifact.parent.mkdir(parents=True) + artifact.write_bytes(b"unrelated content\n") + self.assert_rejected("conflicting output path") + self.assertEqual(artifact.read_bytes(), b"unrelated content\n") + self.assertFalse((self.output / "extensions/index.yaml").exists()) + self.assertFalse((self.output / "x").exists()) + + def test_lookup_entries_are_sorted_and_reproducible(self) -> None: + additional = self.root / "catalog/rc/common-integrations/common-v1alpha1.yaml" + additional.parent.mkdir(parents=True) + reference = "https://runtimeconditions.io/extensions/common-integrations:v1alpha1" + additional.write_text(yaml.safe_dump({"kind": "RuntimeConditionsExtensionDefinition", "metadata": {"id": reference}})) + self.assertEqual(build(self.output, self.root), 0) + index = (self.output / "extensions/index.yaml").read_bytes() + entries = yaml.safe_load(index)["extensions"] + self.assertEqual([entry["id"] for entry in entries], [reference, self.document["metadata"]["id"]]) + self.assertEqual(entries[0]["sourceSha256"], sha256(additional.read_bytes()).hexdigest()) + other_output = self.output.parent / "second-site" + self.assertEqual(build(other_output, self.root), 0) + self.assertEqual((other_output / "extensions/index.yaml").read_bytes(), index) def test_mismatched_generated_module_fails_site_build(self) -> None: (self.root / "bindings/service/go/go.mod").write_text("module github.com/old/module\n") From 700af7ba8c8cca2847920d53beeff2617665fc2d Mon Sep 17 00:00:00 2001 From: Colin Lacy Date: Mon, 9 Nov 2026 08:10:46 +0000 Subject: [PATCH 5/7] allows for linked go profile generator --- .../orchestrator/catalog.go | 14 +- .../extension-bindings/orchestrator/cli.go | 3 + .../orchestrator/pipeline.go | 74 +++++- .../orchestrator/profiler_pipeline_test.go | 243 ++++++++++++++++++ .../extension-bindings/orchestrator/verify.go | 17 +- 5 files changed, 331 insertions(+), 20 deletions(-) create mode 100644 tooling/extension-bindings/orchestrator/profiler_pipeline_test.go diff --git a/tooling/extension-bindings/orchestrator/catalog.go b/tooling/extension-bindings/orchestrator/catalog.go index 7ea54c1..326d419 100644 --- a/tooling/extension-bindings/orchestrator/catalog.go +++ b/tooling/extension-bindings/orchestrator/catalog.go @@ -55,10 +55,10 @@ type Toolchain struct { Kind string `json:"kind" yaml:"kind"` Status string `json:"status" yaml:"status"` Tools struct { - Normalizer Tool `json:"normalizer" yaml:"normalizer"` - Orchestrator Tool `json:"orchestrator" yaml:"orchestrator"` - Emitters map[string]Tool `json:"emitters" yaml:"emitters"` - Profilers map[string]Tool `json:"profilers" yaml:"profilers"` + Normalizer Tool `json:"normalizer" yaml:"normalizer"` + Orchestrator Tool `json:"orchestrator" yaml:"orchestrator"` + Emitters map[string]Tool `json:"emitters" yaml:"emitters"` + Profilers map[string]ProfilerIdentity `json:"profilers" yaml:"profilers"` } `json:"tools" yaml:"tools"` Languages map[string]LanguageToolchain `json:"languages" yaml:"languages"` Python struct { @@ -178,9 +178,13 @@ func DiscoverProject(cwd, packagesPath, toolchainPath, toolingDirectory string) return nil, fmt.Errorf("%s:%s: name is required by the %s native emitter", key, language, language) } if p.Toolchain.Status == "released" { - if p.Toolchain.Tools.Emitters[language].SHA256 == "" || p.Toolchain.Tools.Profilers[language].SHA256 == "" { + profiler, present := p.Toolchain.Tools.Profilers[language] + if p.Toolchain.Tools.Emitters[language].SHA256 == "" || !present { return nil, fmt.Errorf("%s: released lock lacks emitter or profiler", language) } + if err := profiler.Validate(); err != nil { + return nil, fmt.Errorf("%s: invalid released profiler lock: %w", language, err) + } native, ok := p.Toolchain.Languages[language] if !ok || native.LanguageVersion != target.LanguageVersion { return nil, fmt.Errorf("%s:%s: exact language version differs from released lock", key, language) diff --git a/tooling/extension-bindings/orchestrator/cli.go b/tooling/extension-bindings/orchestrator/cli.go index 19ac087..a3eb7ac 100644 --- a/tooling/extension-bindings/orchestrator/cli.go +++ b/tooling/extension-bindings/orchestrator/cli.go @@ -23,6 +23,9 @@ type Options struct { Output, Input, Report, ClosureOutput, DependencyLockOutput, DependencyLock string CoreSchema, Go, Python, GoEmitter, Normalizer, GoProfiler, PythonProfiler, PythonProfilerArtifact string Targets, ExtensionRoots, Overrides, CatalogRoots repeated + // LinkedGoProfiler opts into library identity; GoProfiler must be empty. + // The caller supplies verified build metadata. Standalone defaults are retained. + LinkedGoProfiler *ProfilerIdentity } var supportedOperations = map[string]bool{"resolve": true, "normalize": true, "generate": true, "verify": true, "package": true, "plan-release": true, "check": true, "update": true} diff --git a/tooling/extension-bindings/orchestrator/pipeline.go b/tooling/extension-bindings/orchestrator/pipeline.go index 00139cd..77a19ea 100644 --- a/tooling/extension-bindings/orchestrator/pipeline.go +++ b/tooling/extension-bindings/orchestrator/pipeline.go @@ -65,6 +65,24 @@ func NewPipeline(ctx context.Context, p *Project, o Options) (*Pipeline, error) if p.Toolchain.Status == "released" && len(o.Overrides) > 0 { return nil, fmt.Errorf("released generation forbids extension overrides") } + if o.LinkedGoProfiler != nil { + if o.GoProfiler != "" { + return nil, fmt.Errorf("select either a linked Go profiler identity or a standalone Go profiler executable") + } + identity := *o.LinkedGoProfiler + if identity.Kind != linkedGoProfilerKind { + return nil, fmt.Errorf("LinkedGoProfiler requires a linked Go module identity") + } + if err := identity.Validate(); err != nil { + return nil, err + } + if expected, pinned := p.Toolchain.Tools.Profilers["go"]; pinned { + if err := expected.Match(identity); err != nil { + return nil, fmt.Errorf("go-profiler: %w", err) + } + } + o.LinkedGoProfiler = &identity // Snapshot the caller's supplied identity. + } pipeline := &Pipeline{Context: ctx, Project: p, Options: o, tools: map[string]string{}, identities: map[string]Tool{}, cores: map[string]map[string]any{}, HTTPClient: &http.Client{Timeout: 60 * time.Second, CheckRedirect: func(_ *http.Request, _ []*http.Request) error { return fmt.Errorf("redirects are forbidden") }}} for _, path := range []string{o.WorkDir, o.Cache} { if path != "" { @@ -186,7 +204,14 @@ func (p *Pipeline) tool(key string) (string, Tool, error) { expected = p.Project.Toolchain.Tools.Emitters["go"] case "go-profiler": path = p.Options.GoProfiler - expected = p.Project.Toolchain.Tools.Profilers["go"] + pin := p.Project.Toolchain.Tools.Profilers["go"] + if pin.Kind != "" { + return "", Tool{}, fmt.Errorf("go-profiler identity kind differs from lock") + } + if path == "" { + return "", Tool{}, fmt.Errorf("standalone Go profiler path required; supply --go-profiler or a linked identity") + } + expected = Tool{Name: pin.Name, Version: pin.Version, SHA256: pin.SHA256, ArtifactURL: pin.ArtifactURL} case "orchestrator": path, _ = os.Executable() expected = p.Project.Toolchain.Tools.Orchestrator @@ -314,14 +339,45 @@ func (p *Pipeline) pythonEmitterIdentity() (Tool, error) { p.identities["python-emitter"] = identity return identity, nil } -func (p *Pipeline) profiler(language string) (Tool, error) { - if language == "go" { - _, identity, err := p.tool("go-profiler") - return identity, err +func (p *Pipeline) profiler(language string) (ProfilerIdentity, error) { + var artifact Tool + var err error + switch language { + case "go": + if p.Options.LinkedGoProfiler != nil { + return p.matchProfilerLock(language, *p.Options.LinkedGoProfiler) + } + _, artifact, err = p.tool("go-profiler") + case "python": + artifact, err = p.pythonProfiler() + default: + return ProfilerIdentity{}, fmt.Errorf("unsupported native adapter %s", language) } - if language != "python" { - return Tool{}, fmt.Errorf("unsupported native adapter %s", language) + if err != nil { + return ProfilerIdentity{}, err + } + identity, err := NewStandaloneProfilerIdentity(artifact) + if err != nil { + return ProfilerIdentity{}, err } + return p.matchProfilerLock(language, identity) +} + +func (p *Pipeline) matchProfilerLock(language string, identity ProfilerIdentity) (ProfilerIdentity, error) { + if err := identity.Validate(); err != nil { + return ProfilerIdentity{}, err + } + if expected, pinned := p.Project.Toolchain.Tools.Profilers[language]; pinned { + if err := expected.Match(identity); err != nil { + return ProfilerIdentity{}, fmt.Errorf("%s profiler: %w", language, err) + } + } else if p.Project.Toolchain.Status == "released" { + return ProfilerIdentity{}, fmt.Errorf("%s: released lock lacks profiler", language) + } + return identity, nil +} + +func (p *Pipeline) pythonProfiler() (Tool, error) { if identity, ok := p.identities["python-profiler"]; ok { return identity, nil } @@ -343,10 +399,6 @@ func (p *Pipeline) profiler(language string) (Tool, error) { return Tool{}, err } identity := Tool{Name: "python-rc-profiler", Version: actual.Version, SHA256: actual.SHA256} - expected := p.Project.Toolchain.Tools.Profilers[language] - if expected.SHA256 != "" && (expected.SHA256 != identity.SHA256 || expected.Version != identity.Version) { - return Tool{}, fmt.Errorf("Python profiler identity differs from lock") - } p.identities["python-profiler"] = identity return identity, nil } diff --git a/tooling/extension-bindings/orchestrator/profiler_pipeline_test.go b/tooling/extension-bindings/orchestrator/profiler_pipeline_test.go new file mode 100644 index 0000000..7bba18e --- /dev/null +++ b/tooling/extension-bindings/orchestrator/profiler_pipeline_test.go @@ -0,0 +1,243 @@ +package orchestrator + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func writeProfilerFixture(t *testing.T, path string, value any) { + t.Helper() + data, err := yaml.Marshal(value) + if err == nil { + err = os.WriteFile(path, data, 0644) + } + if err != nil { + t.Fatal(err) + } +} + +func profilerPipelineProject(t *testing.T) (*Project, Options) { + t.Helper() + p, o := testProject(t, "go") + o.CoreSchema = filepath.Join(p.Root, "core.yaml") + writeProfilerFixture(t, o.CoreSchema, map[string]any{"$id": "https://runtimeconditions.io/schemas/profile/0.2.0/runtimeconditions.profile.schema.yaml", "type": "object"}) + return p, o +} + +func standaloneProfilerFixture(t *testing.T) (string, ProfilerIdentity) { + t.Helper() + directory := t.TempDir() + for name, content := range map[string]string{ + "go.mod": "module example.test/standalone-profiler\n\ngo 1.22\n", + "main.go": "package main\nfunc main() { panic(\"profiler identity inspection must not execute the binary\") }\n", + } { + if err := os.WriteFile(filepath.Join(directory, name), []byte(content), 0644); err != nil { + t.Fatal(err) + } + } + path := filepath.Join(directory, "profiler") + command := exec.Command(envOr("RC_BINDINGS_GO", "go"), "build", "-trimpath", "-buildvcs=false", "-o", path, ".") + command.Dir = directory + if data, err := command.CombinedOutput(); err != nil { + t.Fatalf("build standalone identity fixture: %v\n%s", err, data) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + identity, err := NewStandaloneProfilerIdentity(Tool{Name: "go-rc-profiler", Version: "v0.6.0", SHA256: fmt.Sprintf("%x", sha256.Sum256(data))}) + if err != nil { + t.Fatal(err) + } + return path, identity +} + +func TestProfilerPipelineGeneratesVerifiesAndRejectsTampering(t *testing.T) { + path, standalone := standaloneProfilerFixture(t) + _, linked := testProfilerIdentities(t) + for _, expected := range []ProfilerIdentity{standalone, linked} { + name := "standalone" + if expected.Kind != "" { + name = "linked" + } + t.Run(name, func(t *testing.T) { + p, o := profilerPipelineProject(t) + p.Toolchain.Tools.Profilers = map[string]ProfilerIdentity{"go": expected} + selected := expected + if expected.Kind == "" { + t.Setenv("RC_GO_PROFILER_BIN", path) + o.GoProfiler = DefaultOptions().GoProfiler + } else { + t.Setenv("RC_GO_PROFILER_BIN", filepath.Join(p.Root, "absent-profiler")) + o.LinkedGoProfiler = &selected + } + pipeline, err := NewPipeline(context.Background(), p, o) + if err != nil { + t.Fatal(err) + } + defer pipeline.Close() + selected.ModuleSum = "caller changed its identity after construction" + plan, err := p.SelectTargets(nil) + if err != nil { + t.Fatal(err) + } + build, err := pipeline.Generate(plan) + if err != nil { + t.Fatal(err) + } + if summary, err := pipeline.Verify(build); err != nil || summary.Status != "passed" { + t.Fatalf("verify generated %s bindings: %v %+v", name, err, summary) + } + releasePath := filepath.Join(build.Targets[0].Directory, releaseName) + document, err := readMapping(releasePath) + if err != nil { + t.Fatal(err) + } + data, err := json.Marshal(mapValue(document["provenance"])["profiler"]) + if err != nil { + t.Fatal(err) + } + var recorded ProfilerIdentity + if err := json.Unmarshal(data, &recorded); err != nil { + t.Fatal(err) + } + if err := expected.Match(recorded); err != nil { + t.Fatal("generated profiler identity", err) + } + mutations := []ProfilerIdentity{changedProfiler(expected, func(p *ProfilerIdentity) { p.Version = "v0.6.1" })} + if expected.Kind == "" { + mutations = append(mutations, linked, changedProfiler(expected, func(p *ProfilerIdentity) { p.SHA256 = strings.Repeat("f", 64) })) + } else { + mutations = append(mutations, standalone, changedProfiler(expected, func(p *ProfilerIdentity) { p.ModuleSum = "h1:" + strings.Repeat("B", 42) + "A=" })) + } + for _, tampered := range mutations { + copy := mapValue(jsonCopy(document)) + mapValue(copy["provenance"])["profiler"] = tampered + writeProfilerFixture(t, releasePath, copy) + if _, err := pipeline.Verify(build); err == nil || !strings.Contains(err.Error(), "profiler provenance") { + t.Fatalf("tampered profiler metadata accepted or wrong failure: %+v, %v", tampered, err) + } + } + }) + } +} + +func TestProfilerPipelineReadsReleasedModuleLockAndProtectsCommittedSource(t *testing.T) { + p, o := profilerPipelineProject(t) + standalone, linked := testProfilerIdentities(t) + artifact := Tool{Name: "fixture-tool", Version: "0.1.0", SHA256: strings.Repeat("0", 64)} + lock := map[string]any{ + "apiVersion": "runtimeconditions.io/toolchain-lock/v1alpha1", "kind": "RuntimeConditionsToolchainLock", "status": "released", + "tools": map[string]any{"normalizer": artifact, "orchestrator": artifact, "emitters": map[string]any{"go": artifact}, "profilers": map[string]any{"go": linked}}, + "languages": map[string]any{"go": map[string]any{"languageVersion": p.Catalog.Packages["new-channel"].Languages["go"].LanguageVersion, "packageManager": "go", "tools": map[string]any{"go": "fixture"}}}, + } + writeProfilerFixture(t, p.ToolchainPath, lock) + found, err := DiscoverProject(p.Root, "", "", p.ToolingDirectory) + if err != nil { + t.Fatal(err) + } + if err := linked.Match(found.Toolchain.Tools.Profilers["go"]); err != nil { + t.Fatal(err) + } + for _, tc := range []struct { + name string + selected, pin ProfilerIdentity + path, want string + }{ + {"conflict", linked, linked, "explicit-standalone-profiler", "select either"}, + {"wrong option kind", standalone, linked, "", "linked Go module identity"}, + {"unknown identity", changedProfiler(linked, func(p *ProfilerIdentity) { p.ModuleSum = "unknown" }), linked, "", "checksum"}, + {"version mismatch", changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "v0.6.1" }), linked, "", "version differs"}, + {"checksum mismatch", changedProfiler(linked, func(p *ProfilerIdentity) { p.ModuleSum = "h1:" + strings.Repeat("B", 42) + "A=" }), linked, "", "module identity differs"}, + {"artifact lock", linked, standalone, "", "identity kind differs"}, + } { + t.Run(tc.name, func(t *testing.T) { + found.Toolchain.Tools.Profilers["go"] = tc.pin + options := o + options.LinkedGoProfiler, options.GoProfiler = &tc.selected, tc.path + if pipeline, err := NewPipeline(context.Background(), found, options); err == nil || !strings.Contains(err.Error(), tc.want) { + if pipeline != nil { + pipeline.Close() + } + t.Fatalf("invalid selection: %v; want %s", err, tc.want) + } + }) + } + directory := filepath.Join(p.Root, "bindings", "new-channel", "go") + if err := os.MkdirAll(directory, 0755); err != nil { + t.Fatal(err) + } + sentinel := filepath.Join(directory, "keep.go") + if err := os.WriteFile(sentinel, []byte("preserve committed bytes\n"), 0644); err != nil { + t.Fatal(err) + } + wrong := changedProfiler(linked, func(p *ProfilerIdentity) { p.Version = "v0.6.1" }) + o.LinkedGoProfiler = &wrong + o.WorkDir = filepath.Join(p.Root, "must-not-create") + var stdout bytes.Buffer + if err := Execute(context.Background(), p.Root, "update", o, &stdout, io.Discard); err == nil || !strings.Contains(err.Error(), "version differs") { + t.Fatalf("update ignored profiler lock: %v", err) + } + if data, err := os.ReadFile(sentinel); err != nil || string(data) != "preserve committed bytes\n" || stdout.Len() != 0 { + t.Fatalf("rejected update affected committed output: %s %v", data, err) + } + if _, err := os.Stat(o.WorkDir); !os.IsNotExist(err) { + t.Fatal("rejected update created work directory") + } +} + +func TestProfilerPipelineNeverFallsBackFromStandaloneSelection(t *testing.T) { + p, o := profilerPipelineProject(t) + _, linked := testProfilerIdentities(t) + for _, pin := range []ProfilerIdentity{{}, linked} { + p.Toolchain.Tools.Profilers = nil + if pin.Kind != "" { + p.Toolchain.Tools.Profilers = map[string]ProfilerIdentity{"go": pin} + } + for _, path := range []string{"", filepath.Join(p.Root, "missing-profiler")} { + o.GoProfiler = path + pipeline, err := NewPipeline(context.Background(), p, o) + if err != nil { + t.Fatal(err) + } + if _, err := pipeline.profiler("go"); err == nil { + t.Fatal("standalone selection silently used another profiler") + } + if pipeline.work != "" { + t.Fatal("missing standalone profiler triggered workspace build") + } + pipeline.Close() + } + } +} + +func TestProfilerPipelineChecksCachedPythonArtifactAgainstLock(t *testing.T) { + p, o := testProject(t, "python") + identity, _ := testProfilerIdentities(t) + identity.Name = "python-rc-profiler" + p.Toolchain.Tools.Profilers = map[string]ProfilerIdentity{"python": identity} + pipeline, err := NewPipeline(context.Background(), p, o) + if err != nil { + t.Fatal(err) + } + defer pipeline.Close() + pipeline.identities["python-profiler"] = Tool{Name: identity.Name, Version: identity.Version, SHA256: identity.SHA256} + if selected, err := pipeline.profiler("python"); err != nil || selected != identity { + t.Fatalf("cached standalone Python identity: %+v %v", selected, err) + } + p.Toolchain.Tools.Profilers["python"] = changedProfiler(identity, func(p *ProfilerIdentity) { p.SHA256 = strings.Repeat("f", 64) }) + if _, err := pipeline.profiler("python"); err == nil { + t.Fatal("cached Python profiler bypassed a mismatched lock") + } +} diff --git a/tooling/extension-bindings/orchestrator/verify.go b/tooling/extension-bindings/orchestrator/verify.go index 5f8da46..94ca77d 100644 --- a/tooling/extension-bindings/orchestrator/verify.go +++ b/tooling/extension-bindings/orchestrator/verify.go @@ -203,9 +203,18 @@ func (p *Pipeline) verifyIdentities(bt *BuiltTarget, build *Build) error { if err != nil { return err } - declared := mapValue(provenance["profiler"]) - if declared["name"] != profiler.Name || declared["version"] != profiler.Version || declared["sha256"] != profiler.SHA256 { - return fmt.Errorf("profiler provenance differs from installed artifact") + data, err := json.Marshal(provenance["profiler"]) + if err != nil { + return err + } + var declaredProfiler ProfilerIdentity + decoder := json.NewDecoder(bytes.NewReader(data)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&declaredProfiler); err != nil { + return fmt.Errorf("invalid profiler provenance: %w", err) + } + if err := declaredProfiler.Match(profiler); err != nil { + return fmt.Errorf("profiler provenance differs from selected profiler: %w", err) } _, tool, err := p.tool("normalizer") if err != nil { @@ -218,7 +227,7 @@ func (p *Pipeline) verifyIdentities(bt *BuiltTarget, build *Build) error { if err != nil { return err } - declared = mapValue(provenance["orchestrator"]) + declared := mapValue(provenance["orchestrator"]) if declared["sha256"] != tool.SHA256 || declared["name"] != tool.Name || declared["version"] != tool.Version { return fmt.Errorf("orchestrator provenance differs from selected artifact") } From e8e74d5132a18fa5aea265e065f050813c9a4ee0 Mon Sep 17 00:00:00 2001 From: Colin Lacy Date: Tue, 10 Nov 2026 07:29:37 +0000 Subject: [PATCH 6/7] normalizes extension ids, allows go profiler by lib import --- tooling/extension-bindings/IMPLEMENTATION.md | 23 +- .../extension-bindings/normalizer/resolve.go | 57 +++-- .../normalizer/resolve_test.go | 103 ++++++-- .../extension-bindings/normalizer/validate.go | 8 +- .../normalizer/validate_test.go | 54 +++- .../extension-bindings/orchestrator/cli.go | 3 + .../orchestrator/go_libraries.go | 117 +++++++++ .../orchestrator/go_libraries_test.go | 238 ++++++++++++++++++ .../orchestrator/pipeline.go | 26 ++ 9 files changed, 587 insertions(+), 42 deletions(-) create mode 100644 tooling/extension-bindings/orchestrator/go_libraries.go create mode 100644 tooling/extension-bindings/orchestrator/go_libraries_test.go diff --git a/tooling/extension-bindings/IMPLEMENTATION.md b/tooling/extension-bindings/IMPLEMENTATION.md index 3d6a8e2..14424de 100644 --- a/tooling/extension-bindings/IMPLEMENTATION.md +++ b/tooling/extension-bindings/IMPLEMENTATION.md @@ -820,9 +820,26 @@ emission functions directly; Python tooling remains optional for Python targets. Standalone generation in `extensions` MUST remain functional. Its entry point, native-tool options, and ability to select an independently installed Go -profiler executable MUST remain supported. Extracting `Execute` preserves the -existing executable-based tooling and artifact identities. Direct Go tool -execution and linked identities are subsequent integration work. +profiler executable MUST remain supported. `Execute` preserves the existing +executable-based tooling and artifact identities as the default mode. + +Library callers may set `Options.UseGoLibraries` to call `normalizer.Normalize` +and `goemitter.Emit` directly. `Normalizer` and `GoEmitter` must then be empty; +conflicting executable selections fail instead of being ignored. This option +does not change profiler selection: callers may use `LinkedGoProfiler` or retain +a standalone `GoProfiler` executable. Python emission remains external. + +In library mode, normalizer and Go emitter identities retain their component +names and versions, with the SHA-256 of the actual host executable (`rc` when +integrated). No module checksum is substituted for an executable digest. +Rebuilding the host changes those identities and the normalized model digest. +Artifact locks must pin that host digest with the applicable component name and +version; a lock for a separately built normalizer or emitter fails. These checks +run before generation creates work files. Verification and regeneration use +the selected execution mode and retain the existing schema and identity checks. +Cancellation is checked around direct calls; the component APIs cannot interrupt +an active normalization or emission call. Schema resources are still required +through tooling discovery; packaging them for the common CLI is separate work. Linked profiler support MUST be explicit and optional. Its identity must record the actual linked module version and dependency checksum. Metadata schemas, diff --git a/tooling/extension-bindings/normalizer/resolve.go b/tooling/extension-bindings/normalizer/resolve.go index 24d4acf..d8028ff 100644 --- a/tooling/extension-bindings/normalizer/resolve.go +++ b/tooling/extension-bindings/normalizer/resolve.go @@ -339,7 +339,7 @@ func backendRank(backend string) int { } func (r *Resolver) Resolve(ctx context.Context, rootID string) (ResolvedClosure, error) { - if _, err := parseExtensionIdentifier(rootID); err != nil { + if _, _, err := parseExtensionIdentifier(rootID); err != nil { return ResolvedClosure{}, diagnostic("unknown-extension", "RCB1203", rootID, "", err.Error()) } state := visitState{activeIndex: map[string]int{}, complete: map[string]bool{}} @@ -451,7 +451,7 @@ func (r *Resolver) visit(ctx context.Context, id string, state *visitState) erro } func (r *Resolver) load(ctx context.Context, id string) (ResolvedDocument, error) { - parsed, err := parseExtensionIdentifier(id) + parsed, _, err := parseExtensionIdentifier(id) if err != nil { return ResolvedDocument{}, diagnostic("unknown-extension", "RCB1207", id, "", err.Error()) } @@ -467,31 +467,54 @@ func (r *Resolver) load(ctx context.Context, id string) (ResolvedDocument, error return r.finishLoad(id, data, candidate.backend, candidate.locator) } if strings.EqualFold(parsed.Scheme, "https") { - if parsed.Hostname() == "" || parsed.User != nil { - return ResolvedDocument{}, diagnostic("unknown-extension", "RCB1207", id, "", "HTTPS retrieval requires a host and forbids credentials") - } - return r.loadHTTPS(ctx, id, id) + return r.loadHTTPS(ctx, id, r.config.Locks[id].Locator) } return ResolvedDocument{}, diagnostic("unknown-extension", "RCB1207", id, "", fmt.Sprintf("automatic retrieval for scheme %q is unavailable; supply the exact identifier through a catalog, package, cache, or override", parsed.Scheme)) } -// parseExtensionIdentifier validates identity without selecting a retrieval -// backend or imposing a provider/service/version layout. The exact spelling is -// retained throughout resolution, model provenance, and dependency locks. -func parseExtensionIdentifier(id string) (*url.URL, error) { +// parseExtensionIdentifier requires :. The separator is the +// first literal colon in the final path component (or opaque URI body), excluding +// scheme and authority colons. Version labels may themselves contain colons. +// The complete reference retains its exact spelling throughout resolution. +func parseExtensionIdentifier(id string) (*url.URL, string, error) { parsed, err := url.Parse(id) - if err != nil || !parsed.IsAbs() || strings.ContainsFunc(id, func(r rune) bool { - return unicode.IsSpace(r) || r < 0x20 || r == 0x7f + if err != nil || !parsed.IsAbs() || strings.ContainsAny(id, "?#\\") || strings.ContainsFunc(id, func(r rune) bool { + return unicode.IsSpace(r) || unicode.IsControl(r) }) { - return nil, fmt.Errorf("extension identifier must be a non-empty absolute URI with a scheme") + return nil, "", fmt.Errorf("extension reference must be an absolute : URI without whitespace, query, or fragment") + } + component := parsed.Opaque + if component == "" { + path := parsed.EscapedPath() + component = path[strings.LastIndex(path, "/")+1:] + } + location, version, versioned := strings.Cut(component, ":") + if !versioned || location == "" || version == "" || version == "." || version == ".." || strings.Contains(version, "/") { + return nil, "", fmt.Errorf("extension reference must include a non-empty location and explicit version as :") + } + name, err := url.PathUnescape(location) + if err != nil || strings.HasSuffix(strings.ToLower(name), ".yaml") || strings.HasSuffix(strings.ToLower(name), ".yml") || strings.HasSuffix(strings.ToLower(name), ".json") { + return nil, "", fmt.Errorf("extension reference location must identify an extension, not a definition artifact URL") + } + if parsed.User != nil || (strings.EqualFold(parsed.Scheme, "https") && (parsed.Hostname() == "" || parsed.Opaque != "")) { + return nil, "", fmt.Errorf("extension reference forbids credentials and HTTPS references require a host") } - return parsed, nil + return parsed, version, nil } func (r *Resolver) loadHTTPS(ctx context.Context, id, locator string) (ResolvedDocument, error) { if !r.config.Network { return ResolvedDocument{}, diagnostic("unknown-extension", "RCB1208", id, "", "network resolution is disabled") } + if locator == "" { + return ResolvedDocument{}, diagnostic("extension-dependency", "RCB1209", id, "", "HTTPS resolution requires an explicit retrieval locator") + } + parsed, err := url.Parse(locator) + if err != nil || !strings.EqualFold(parsed.Scheme, "https") || parsed.Hostname() == "" || parsed.User != nil || strings.ContainsAny(locator, "#\\") || strings.ContainsFunc(locator, func(r rune) bool { + return unicode.IsSpace(r) || unicode.IsControl(r) + }) { + return ResolvedDocument{}, diagnostic("unknown-extension", "RCB1207", id, "", "retrieval locator must be an HTTPS URL with a host and without credentials or a fragment") + } lock, locked := r.config.Locks[id] if (!locked || lock.SourceSHA256 == "" || lock.Locator == "") && !r.config.RecordNetworkLocks { return ResolvedDocument{}, diagnostic("extension-dependency", "RCB1209", id, "", "HTTPS resolution requires source digest and locator locks") @@ -518,7 +541,7 @@ func (r *Resolver) loadHTTPS(ctx context.Context, id, locator string) (ResolvedD if err != nil { return ResolvedDocument{}, err } - if locked && SHA256Hex(data) != lock.SourceSHA256 { + if locked && lock.SourceSHA256 != "" && SHA256Hex(data) != lock.SourceSHA256 { return ResolvedDocument{}, diagnostic("extension-dependency", "RCB1212", id, "", "HTTPS content does not match the dependency lock") } document, err := r.finishLoad(id, data, "https", response.Request.URL.String()) @@ -528,11 +551,11 @@ func (r *Resolver) loadHTTPS(ctx context.Context, id, locator string) (ResolvedD if lock.SemanticSHA256 != "" && document.SemanticSHA256 != lock.SemanticSHA256 { return ResolvedDocument{}, diagnostic("extension-dependency", "RCB1213", id, "", "HTTPS semantic digest does not match the dependency lock") } - if !locked { + if !locked || lock.SourceSHA256 == "" { if r.config.Locks == nil { r.config.Locks = map[string]LockEntry{} } - r.config.Locks[id] = LockEntry{SourceSHA256: document.SourceSHA256, SemanticSHA256: document.SemanticSHA256, Locator: locator} + r.config.Locks[id] = LockEntry{SourceSHA256: document.SourceSHA256, SemanticSHA256: document.SemanticSHA256, SourceBackend: "https", Locator: locator} } if r.config.CacheDir != "" { if err := os.MkdirAll(r.config.CacheDir, 0755); err != nil { diff --git a/tooling/extension-bindings/normalizer/resolve_test.go b/tooling/extension-bindings/normalizer/resolve_test.go index f949a68..e30c32f 100644 --- a/tooling/extension-bindings/normalizer/resolve_test.go +++ b/tooling/extension-bindings/normalizer/resolve_test.go @@ -25,7 +25,7 @@ func fixtureExtension(id string, dependencies []string, specBody string) []byte dependencyYAML.WriteByte('\n') } } - return []byte(fmt.Sprintf("apiVersion: runtimeconditions.io/v1alpha1\nkind: RuntimeConditionsExtensionDefinition\nmetadata:\n id: %s\n version: 1.0.0\nspec:\n%s%s", id, dependencyYAML.String(), specBody)) + return []byte(fmt.Sprintf("apiVersion: runtimeconditions.io/v1alpha1\nkind: RuntimeConditionsExtensionDefinition\nmetadata:\n id: %q\n version: 1.0.0\nspec:\n%s%s", id, dependencyYAML.String(), specBody)) } func kindSpec(name string) string { @@ -72,25 +72,35 @@ func closureIDs(closure ResolvedClosure) []string { func TestExtensionIdentifierSchemesAndExactSpelling(t *testing.T) { for _, id := range []string{ - "https://example.test/arbitrary/nested/path/release.yaml?revision=abc", - "https://EXAMPLE.test:8443/CaseSensitive/%73ervice/V1+build", - "file:///opt/extensions/release.yaml", "oci://example.test/extensions@sha256:abc", - "urn:example:extension:release-2026", "custom+resolver:immutable-release", + "https://runtimeconditions.io/extensions/common-integrations:v1alpha1", + "https://example.test/arbitrary/nested/service:sha256:abc", + "https://EXAMPLE.test:8443/CaseSensitive/%73ervice:V1+build", + "file:///opt/extensions/service:1.0.0", "oci://example.test/extensions:1.0.0", + "urn:service:release-2026", "custom+resolver:service:1.0.0", } { - parsed, err := parseExtensionIdentifier(id) - if err != nil || parsed.String() != id { + parsed, version, err := parseExtensionIdentifier(id) + if err != nil || parsed.String() != id || version == "" { t.Errorf("identifier %q changed or was rejected: %v", id, err) } } - for _, id := range []string{"", "relative/path", "/absolute/path", "//example.test/path", "1bad:release", "urn:bad version", "https://example.test/%invalid"} { - if _, err := parseExtensionIdentifier(id); err == nil { + for _, id := range []string{ + "", "relative/path:1", "/absolute/path:1", "//example.test/path:1", "1bad:release:1", + "urn:bad version:1", "https://example.test/%invalid:1", "https://example.test:8443", + "https://example.test/service:", "https://example.test/:1", "https://example.test/service:.", + "https://example.test/service:..", "https:///service:1", "https://user@example.test/service:1", + "https://example.test/service:1?revision=abc", "https://example.test/service:1#fragment", + "https://example.test/service:bad\\version", "https://example.test/service:bad\u0085version", + "https://example.test/definition.yaml:1.0.0", "file:///opt/definition.yml:1.0.0", + "https://example.test/definition.json:1.0.0", "https://example.test/definition%2Eyaml:1.0.0", + } { + if _, _, err := parseExtensionIdentifier(id); err == nil { t.Errorf("invalid extension identifier accepted: %q", id) } } } -func TestVersionlessIdentifiersResolveFromEveryLocalBackend(t *testing.T) { - for _, id := range []string{"urn:example:release-a", "file:///opt/extensions/release.yaml", "oci://registry.example/extensions@sha256:abc", "custom+resolver:release-a"} { +func TestVersionedIdentifiersWithoutAnnotationResolveFromEveryLocalBackend(t *testing.T) { + for _, id := range []string{"https://example.test/service:1.0.0", "urn:service:1.0.0", "file:///opt/extensions/service:1.0.0", "oci://registry.example/extensions:1.0.0", "custom+resolver:service:1.0.0"} { t.Run(id, func(t *testing.T) { data := bytes.ReplaceAll(fixtureExtension(id, nil, kindSpec("future.service")), []byte(" version: 1.0.0\n"), nil) directory := t.TempDir() @@ -131,8 +141,44 @@ func TestVersionlessIdentifiersResolveFromEveryLocalBackend(t *testing.T) { } } +func TestUnversionedReferencesAreRejectedAcrossBackends(t *testing.T) { + for _, id := range []string{ + "https://runtimeconditions.io/extensions/common-integrations/v1alpha1/runtimeconditions.extension.yaml", + "https://runtimeconditions.io/extensions/common-integrations/v1alpha1/runtimeconditions.extension.yaml:v1alpha1", + "https://example.test/service", "https://example.test:8443", "urn:release-a", + "file:///opt/extensions/release.yaml", "oci://registry.example/extensions", "custom+resolver:release-a", + } { + t.Run(id, func(t *testing.T) { + data := fixtureExtension(id, nil, kindSpec("fixture.service")) + directory, cache := t.TempDir(), t.TempDir() + path := writeFixture(t, directory, "definition.yaml", data) + writeFixture(t, cache, SHA256Hex(data)+".yaml", data) + transport := &countingTransport{} + for name, config := range map[string]ResolverConfig{ + "catalog": {CatalogRoots: []string{directory}}, "package": {PackageRoots: []string{directory}}, + "cache": {CacheDir: cache}, "override": {Overrides: map[string]string{id: path}}, + "https": {Network: true, HTTPClient: &http.Client{Transport: transport}, Locks: map[string]LockEntry{id: {SourceSHA256: SHA256Hex(data), Locator: "https://artifacts.example.test/definition.yaml"}}}, + } { + t.Run(name, func(t *testing.T) { + config.Schemas = testSchemas(t) + resolver, err := NewResolver(config) + if err != nil { + t.Fatal(err) + } + if _, err = resolver.Resolve(context.Background(), id); err == nil || diagnosticCode(t, err) != "RCB1203" { + t.Fatalf("expected reference rejection, got %v", err) + } + }) + } + if transport.requests.Load() != 0 { + t.Fatal("invalid reference reached HTTP transport") + } + }) + } +} + func TestUnavailableRetrievalDoesNotInvalidateIdentifier(t *testing.T) { - for _, id := range []string{"urn:example:release", "oci://example.test/release", "file:///opt/extensions/release.yaml"} { + for _, id := range []string{"urn:service:1.0.0", "oci://example.test/service:1.0.0", "file:///opt/extensions/service:1.0.0"} { resolver := fixtureResolver(t) _, err := resolver.Resolve(context.Background(), id) if err == nil || diagnosticCode(t, err) != "RCB1207" || !strings.Contains(err.Error(), "automatic retrieval") { @@ -142,14 +188,14 @@ func TestUnavailableRetrievalDoesNotInvalidateIdentifier(t *testing.T) { } func TestRequestedIdentifierMustExactlyMatchDefinition(t *testing.T) { - id := "urn:example:ReleaseA" + id := "https://example.test/ReleaseA:1.0.0" path := writeFixture(t, t.TempDir(), "release.yaml", fixtureExtension(id, nil, kindSpec("future.service"))) - _, err := NewResolver(ResolverConfig{Schemas: testSchemas(t), Overrides: map[string]string{"urn:example:releasea": path}}) + _, err := NewResolver(ResolverConfig{Schemas: testSchemas(t), Overrides: map[string]string{"https://example.test/releasea:1.0.0": path}}) if err == nil || diagnosticCode(t, err) != "RCB1201" { t.Fatalf("case-sensitive identity mismatch accepted: %v", err) } resolver := fixtureResolver(t) - _, err = resolver.finishLoad("urn:example:releasea", fixtureExtension(id, nil, kindSpec("future.service")), "https", "https://example.test/release.yaml") + _, err = resolver.finishLoad("https://example.test/releasea:1.0.0", fixtureExtension(id, nil, kindSpec("future.service")), "https", "https://example.test/release.yaml") if err == nil || diagnosticCode(t, err) != "RCB1220" { t.Fatalf("retrieved identity mismatch accepted: %v", err) } @@ -399,7 +445,7 @@ func TestDevelopmentHTTPSRecordsLockAndExplicitCache(t *testing.T) { id := "https://new.example.test/future/service:1.0.0" data := fixtureExtension(id, nil, kindSpec("future.kind")) cache := filepath.Join(t.TempDir(), "explicit-cache") - config := ResolverConfig{Schemas: testSchemas(t), Network: true, RecordNetworkLocks: true, CacheDir: cache, HTTPClient: &http.Client{Transport: staticHTTPTransport{data: data}}} + config := ResolverConfig{Schemas: testSchemas(t), Network: true, RecordNetworkLocks: true, CacheDir: cache, HTTPClient: &http.Client{Transport: staticHTTPTransport{data: data}}, Locks: map[string]LockEntry{id: {Locator: "https://artifacts.example.test/service.yaml"}}} resolver, err := NewResolver(config) if err != nil { t.Fatal(err) @@ -412,6 +458,9 @@ func TestDevelopmentHTTPSRecordsLockAndExplicitCache(t *testing.T) { if len(lock.Extensions) != 1 || lock.Extensions[0].SourceSHA256 != SHA256Hex(data) || lock.Extensions[0].SourceBackend != "https" { t.Fatalf("missing transport lock: %+v", lock) } + if resolver.config.Locks[id].SourceSHA256 != SHA256Hex(data) || lock.Extensions[0].SourceLocator != config.Locks[id].Locator { + t.Fatal("development resolution failed to complete the explicit retrieval lock") + } if _, err := os.Stat(filepath.Join(cache, SHA256Hex(data)+".yaml")); err != nil { t.Fatal(err) } @@ -546,7 +595,7 @@ func TestLockedHTTPSMappedDefinitionValidation(t *testing.T) { func TestLockedHTTPSRejectsInvalidRetrievalURLs(t *testing.T) { const id = "https://runtimeconditions.io/extensions/example:1.0.0" - for _, locator := range []string{"http://artifacts.example.test/definition.yaml", "file:///definition.yaml", "https:///missing-host.yaml", "https://user:password@artifacts.example.test/definition.yaml", "https://artifacts.example.test/%invalid"} { + for _, locator := range []string{"http://artifacts.example.test/definition.yaml", "file:///definition.yaml", "https:///missing-host.yaml", "https://user:password@artifacts.example.test/definition.yaml", "https://artifacts.example.test/%invalid", "https://artifacts.example.test/definition.yaml#fragment"} { t.Run(locator, func(t *testing.T) { transport := &countingTransport{} resolver, err := NewResolver(ResolverConfig{ @@ -564,6 +613,20 @@ func TestLockedHTTPSRejectsInvalidRetrievalURLs(t *testing.T) { } } +func TestHTTPSRequiresExplicitRetrievalMapping(t *testing.T) { + id := "https://runtimeconditions.io/extensions/example:1.0.0" + for _, record := range []bool{false, true} { + transport := &countingTransport{} + resolver, err := NewResolver(ResolverConfig{Schemas: testSchemas(t), Network: true, RecordNetworkLocks: record, HTTPClient: &http.Client{Transport: transport}}) + if err != nil { + t.Fatal(err) + } + if _, err = resolver.Resolve(context.Background(), id); err == nil || diagnosticCode(t, err) != "RCB1209" || transport.requests.Load() != 0 { + t.Fatalf("missing mapping was not rejected before HTTP retrieval: %v", err) + } + } +} + type countingTransport struct { requests atomic.Int64 } @@ -669,10 +732,12 @@ func TestLockedHTTPSClosureOverTLS(t *testing.T) { _, _ = w.Write(data) })) defer server.Close() - rootID, dependencyID := server.URL+"/future/releases/root.yaml", server.URL+"/independent/base.yaml" + rootID, dependencyID := "https://example.test/root:1.0.0", "https://example.test/base:1.0.0" definitions["/future/releases/root.yaml"] = fixtureExtension(rootID, []string{dependencyID}, kindSpec("future.root")) definitions["/independent/base.yaml"] = fixtureExtension(dependencyID, nil, kindSpec("future.base")) - config := ResolverConfig{Schemas: testSchemas(t), Network: true, RecordNetworkLocks: true, HTTPClient: server.Client()} + config := ResolverConfig{Schemas: testSchemas(t), Network: true, RecordNetworkLocks: true, HTTPClient: server.Client(), Locks: map[string]LockEntry{ + rootID: {Locator: server.URL + "/future/releases/root.yaml"}, dependencyID: {Locator: server.URL + "/independent/base.yaml"}, + }} resolver, err := NewResolver(config) if err != nil { t.Fatal(err) diff --git a/tooling/extension-bindings/normalizer/validate.go b/tooling/extension-bindings/normalizer/validate.go index 2e467a3..c2834f6 100644 --- a/tooling/extension-bindings/normalizer/validate.go +++ b/tooling/extension-bindings/normalizer/validate.go @@ -110,11 +110,15 @@ func (s *Schemas) ValidateExtension(data map[string]any, definition ExtensionDef if definition.Kind != ExtensionKind { return diagnostic("structural", "RCB1104", definition.Metadata.ID, "/kind", "document is not a RuntimeConditionsExtensionDefinition") } - if _, err := parseExtensionIdentifier(definition.Metadata.ID); err != nil { + _, version, err := parseExtensionIdentifier(definition.Metadata.ID) + if err != nil { return diagnostic("structural", "RCB1105", definition.Metadata.ID, "/metadata/id", err.Error()) } + if definition.Metadata.Version != "" && definition.Metadata.Version != version { + return diagnostic("structural", "RCB1106", definition.Metadata.ID, "/metadata/version", "metadata.version must exactly match the version in metadata.id") + } for index, dependency := range definition.Spec.Dependencies { - if _, err := parseExtensionIdentifier(dependency); err != nil { + if _, _, err := parseExtensionIdentifier(dependency); err != nil { return diagnostic("extension-dependency", "RCB1110", definition.Metadata.ID, fmt.Sprintf("/spec/dependencies/%d", index), err.Error()) } } diff --git a/tooling/extension-bindings/normalizer/validate_test.go b/tooling/extension-bindings/normalizer/validate_test.go index 878c00a..7e7c829 100644 --- a/tooling/extension-bindings/normalizer/validate_test.go +++ b/tooling/extension-bindings/normalizer/validate_test.go @@ -1,6 +1,58 @@ package normalizer -import "testing" +import ( + "bytes" + "testing" +) + +func TestExtensionReferenceValidation(t *testing.T) { + const canonical = "https://runtimeconditions.io/extensions/example:1.0.0" + const artifact = "https://runtimeconditions.io/extensions/example/1.0.0/runtimeconditions.extension.yaml" + for _, test := range []struct { + name, id, version, dependency, code, pointer string + }{ + {"matching annotation", canonical, "1.0.0", "", "", ""}, + {"omitted annotation", canonical, "", "", "", ""}, + {"label version", "https://runtimeconditions.io/extensions/example:v1alpha1", "v1alpha1", "", "", ""}, + {"digest version", "https://runtimeconditions.io/extensions/example:sha256:abc", "sha256:abc", "", "", ""}, + {"unversioned id with annotation", "https://example.test/service", "1.0.0", "", "RCB1105", "/metadata/id"}, + {"artifact id", artifact, "1.0.0", "", "RCB1105", "/metadata/id"}, + {"artifact id with appended version", artifact + ":1.0.0", "1.0.0", "", "RCB1105", "/metadata/id"}, + {"empty version suffix", "https://example.test/service:", "1.0.0", "", "RCB1105", "/metadata/id"}, + {"mismatched annotation", canonical, "2.0.0", "", "RCB1106", "/metadata/version"}, + {"annotation spelling mismatch", "https://example.test/service:v1alpha1", "V1alpha1", "", "RCB1106", "/metadata/version"}, + {"canonical dependency", canonical, "1.0.0", "https://example.test/dependency:v1alpha1", "", ""}, + {"unversioned dependency", canonical, "1.0.0", "https://example.test/dependency", "RCB1110", "/spec/dependencies/0"}, + {"artifact dependency", canonical, "1.0.0", artifact, "RCB1110", "/spec/dependencies/0"}, + {"artifact dependency with appended version", canonical, "1.0.0", artifact + ":1.0.0", "RCB1110", "/spec/dependencies/0"}, + } { + t.Run(test.name, func(t *testing.T) { + var dependencies []string + if test.dependency != "" { + dependencies = []string{test.dependency} + } + data := bytes.Replace(fixtureExtension(test.id, dependencies, kindSpec("fixture.service")), []byte(" version: 1.0.0\n"), nil, 1) + if test.version != "" { + data = bytes.Replace(data, []byte("spec:\n"), []byte(" version: "+test.version+"\nspec:\n"), 1) + } + definition, mapping, err := DecodeExtension(data) + if err != nil { + t.Fatal(err) + } + err = testSchemas(t).ValidateExtension(mapping, definition) + if test.code == "" { + if err != nil { + t.Fatal(err) + } + if definition.Metadata.ID != test.id || definition.Metadata.Version != test.version { + t.Fatal("validation changed the identity or version annotation") + } + } else if err == nil || diagnosticCode(t, err) != test.code || err.(*DiagnosticError).Diagnostic.JSONPointer != test.pointer { + t.Fatalf("expected %s at %s, got %v", test.code, test.pointer, err) + } + }) + } +} func TestLanguageSpecificEmittersMayHandleHyphens(t *testing.T) { data := []byte("apiVersion: runtimeconditions.io/v1alpha1\nkind: RuntimeConditionsExtensionDefinition\nmetadata:\n id: https://runtimeconditions.io/test/hyphen-id:1.0.0\n version: 1.0.0\nspec:\n kinds:\n - name: service\n fieldValues:\n - field: mode\n targetKind: service\n values: [direct-mode]\n") diff --git a/tooling/extension-bindings/orchestrator/cli.go b/tooling/extension-bindings/orchestrator/cli.go index a3eb7ac..30ca690 100644 --- a/tooling/extension-bindings/orchestrator/cli.go +++ b/tooling/extension-bindings/orchestrator/cli.go @@ -26,6 +26,9 @@ type Options struct { // LinkedGoProfiler opts into library identity; GoProfiler must be empty. // The caller supplies verified build metadata. Standalone defaults are retained. LinkedGoProfiler *ProfilerIdentity + // UseGoLibraries calls the linked normalizer and Go emitter directly. + // Normalizer and GoEmitter must be empty; provenance hashes the host executable. + UseGoLibraries bool } var supportedOperations = map[string]bool{"resolve": true, "normalize": true, "generate": true, "verify": true, "package": true, "plan-release": true, "check": true, "update": true} diff --git a/tooling/extension-bindings/orchestrator/go_libraries.go b/tooling/extension-bindings/orchestrator/go_libraries.go new file mode 100644 index 0000000..b39d828 --- /dev/null +++ b/tooling/extension-bindings/orchestrator/go_libraries.go @@ -0,0 +1,117 @@ +package orchestrator + +import ( + "crypto/sha256" + "fmt" + "io" + "os" + + goemitter "github.com/runtimeconditions/extensions/tooling/extension-bindings/emitters/go" + "github.com/runtimeconditions/extensions/tooling/extension-bindings/normalizer" +) + +// goLibraryTool identifies the artifact executing a linked component, rather +// than a separately built tool or a module checksum. Its path is never executed. +func (p *Pipeline) goLibraryTool(key string) (string, Tool, error) { + if err := p.Context.Err(); err != nil { + return "", Tool{}, err + } + var identity, expected Tool + switch key { + case "normalizer": + identity = Tool{Name: normalizer.NormalizerName, Version: normalizer.NormalizerVersion} + expected = p.Project.Toolchain.Tools.Normalizer + case "go-emitter": + identity = Tool{Name: goemitter.EmitterName, Version: goemitter.EmitterVersion} + expected = p.Project.Toolchain.Tools.Emitters["go"] + default: + return "", Tool{}, fmt.Errorf("unknown Go library component %s", key) + } + path, cached := p.tools[key] + if cached { + identity.SHA256 = p.identities[key].SHA256 + } else { + var err error + path, err = os.Executable() + if err != nil { + return "", Tool{}, err + } + artifact, err := os.Open(path) + if err != nil { + return "", Tool{}, err + } + digest := sha256.New() + _, err = io.Copy(digest, artifact) + closeErr := artifact.Close() + if err != nil { + return "", Tool{}, err + } + if closeErr != nil { + return "", Tool{}, closeErr + } + identity.SHA256 = fmt.Sprintf("%x", digest.Sum(nil)) + } + if p.Project.Toolchain.Status == "released" && expected.SHA256 == "" { + return "", Tool{}, fmt.Errorf("%s: released lock lacks host executable identity", key) + } + if expected.SHA256 != "" && (expected.Name != identity.Name || expected.Version != identity.Version || expected.SHA256 != identity.SHA256) { + return "", Tool{}, fmt.Errorf("%s host executable identity differs from toolchain lock", key) + } + if err := p.Context.Err(); err != nil { + return "", Tool{}, err + } + p.tools[key], p.identities[key] = path, identity + return path, identity, nil +} + +func (p *Pipeline) normalizeGoLibrary(doc *Document) error { + _, identity, err := p.tool("normalizer") + if err != nil { + return err + } + _, core, err := p.core(doc.CoreVersion) + if err != nil { + return err + } + if err := p.Context.Err(); err != nil { + return err + } + model, err := normalizer.Normalize(doc.Closure, doc.Lock, p.Project.Schemas, normalizer.NormalizeConfig{ + CoreProfileSchema: core, + Normalizer: normalizer.ToolIdentity{Name: identity.Name, Version: identity.Version, SHA256: identity.SHA256}, + }) + if err != nil { + return err + } + if err := p.Context.Err(); err != nil { + return err + } + doc.Model = model + return nil +} + +func (p *Pipeline) emitGoLibrary(bt *BuiltTarget) error { + if _, _, err := p.tool("go-emitter"); err != nil { + return err + } + modelPath, targetPath, err := p.emitterInputs(bt) + if err != nil { + return err + } + // Use the emitter's own strict loaders, matching its standalone entry point. + model, err := goemitter.LoadModel(modelPath) + if err != nil { + return err + } + target, err := goemitter.LoadPackageTarget(targetPath) + if err != nil { + return err + } + if err := p.Context.Err(); err != nil { + return err + } + if err := goemitter.Emit(model, target, bt.Directory); err != nil { + return err + } + return p.Context.Err() +} diff --git a/tooling/extension-bindings/orchestrator/go_libraries_test.go b/tooling/extension-bindings/orchestrator/go_libraries_test.go new file mode 100644 index 0000000..e94eff0 --- /dev/null +++ b/tooling/extension-bindings/orchestrator/go_libraries_test.go @@ -0,0 +1,238 @@ +package orchestrator + +import ( + "bytes" + "context" + "errors" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/runtimeconditions/extensions/tooling/extension-bindings/normalizer" +) + +func goLibraryProject(t *testing.T) (*Project, Options) { + t.Helper() + p, o := profilerPipelineProject(t) + o.ToolingDir = t.TempDir() + if err := copyTree(filepath.Join(p.ToolingDirectory, "model"), filepath.Join(o.ToolingDir, "model")); err != nil { + t.Fatal(err) + } + p, err := DiscoverProject(p.Root, "", "", o.ToolingDir) + if err != nil { + t.Fatal(err) + } + _, linked := testProfilerIdentities(t) + o.UseGoLibraries, o.LinkedGoProfiler = true, &linked + return p, o +} + +func TestGoLibrariesGenerateAndVerifyWithoutToolExecutables(t *testing.T) { + standalonePath, standaloneIdentity := standaloneProfilerFixture(t) + for _, mode := range []string{"linked", "standalone"} { + t.Run(mode, func(t *testing.T) { + p, o := goLibraryProject(t) + if mode == "standalone" { + o.GoProfiler, o.LinkedGoProfiler = standalonePath, nil + p.Toolchain.Tools.Profilers = map[string]ProfilerIdentity{"go": standaloneIdentity} + } + goCommand := o.Go + o.Go = filepath.Join(p.Root, "no-build-command") + pipeline, err := NewPipeline(context.Background(), p, o) + if err != nil { + t.Fatal(err) + } + defer pipeline.Close() + plan, err := p.SelectTargets(nil) + if err != nil { + t.Fatal(err) + } + build, err := pipeline.Generate(plan) + if err != nil { + t.Fatal(err) + } + host, err := os.Executable() + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(host) + if err != nil { + t.Fatal(err) + } + digest := normalizer.SHA256Hex(data) + bt := build.Targets[0] + if bt.Document.Model.Metadata.Normalizer.SHA256 != digest || bt.PackageConfig["emitterSha256"] != digest { + t.Fatal("library provenance does not identify the executing host") + } + for _, directory := range []string{"tools", "normalizer-inputs"} { + if _, err := os.Stat(filepath.Join(pipeline.work, directory)); !os.IsNotExist(err) { + t.Fatalf("library mode created %s: %v", directory, err) + } + } + pipeline.Options.Go = goCommand // Native verification still requires Go. + loaded, err := pipeline.LoadBuild(plan, bt.Directory) + if err != nil { + t.Fatal(err) + } + if summary, err := pipeline.Verify(loaded); err != nil || summary.Status != "passed" { + t.Fatalf("verify library-generated bindings: %v %+v", err, summary) + } + }) + } +} + +func TestGoLibrariesMatchStandaloneGeneration(t *testing.T) { + p, o := profilerPipelineProject(t) + _, linked := testProfilerIdentities(t) + o.LinkedGoProfiler = &linked + var builds []*Build + for _, library := range []bool{false, true} { + o.UseGoLibraries = library + pipeline, err := NewPipeline(context.Background(), p, o) + if err != nil { + t.Fatal(err) + } + defer pipeline.Close() + plan, err := p.SelectTargets(nil) + if err != nil { + t.Fatal(err) + } + build, err := pipeline.Generate(plan) + if err != nil { + t.Fatal(err) + } + builds = append(builds, build) + } + standalone, library := builds[0].Targets[0], builds[1].Targets[0] + model := library.Document.Model + model.Metadata.Normalizer = standalone.Document.Model.Metadata.Normalizer + var err error + model.Metadata.SemanticSHA256, err = normalizer.ModelSemanticSHA256(model) + if err != nil { + t.Fatal(err) + } + left, err := normalizer.CanonicalModelYAML(model) + if err != nil { + t.Fatal(err) + } + right, err := standalone.Document.ModelYAML() + if err != nil || !bytes.Equal(left, right) { + t.Fatalf("normalization differs beyond executing-artifact provenance: %v", err) + } + leftTree, err := readTree(library.Directory) + if err != nil { + t.Fatal(err) + } + rightTree, err := readTree(standalone.Directory) + if err != nil { + t.Fatal(err) + } + for name, data := range leftTree { + if strings.HasSuffix(name, ".go") || name == "go.mod" { + data = bytes.ReplaceAll(data, []byte(library.Document.Model.Metadata.SemanticSHA256), []byte(model.Metadata.SemanticSHA256)) + if !bytes.Equal(data, rightTree[name]) { + t.Fatalf("native source differs: %s", name) + } + } + } +} + +func TestGoLibrariesRejectConflictsAndMismatchedLocksBeforeWork(t *testing.T) { + p, o := goLibraryProject(t) + pipeline, err := NewPipeline(context.Background(), p, o) + if err != nil { + t.Fatal(err) + } + defer pipeline.Close() + pins := pipeline.identities + o.WorkDir = filepath.Join(p.Root, "must-not-create") + for _, component := range []string{"normalizer", "go-emitter"} { + for _, field := range []string{"executable", "name", "version", "digest", "missing released pin"} { + t.Run(component+"/"+field, func(t *testing.T) { + project, options := *p, o + project.Toolchain = Toolchain{Status: "development"} + pin := pins[component] + switch field { + case "executable": + if component == "normalizer" { + options.Normalizer = "missing-normalizer" + } else { + options.GoEmitter = "missing-emitter" + } + case "name": + pin.Name = "different-component" + case "version": + pin.Version = "99.0.0" + case "digest": + pin.SHA256 = strings.Repeat("f", 64) + case "missing released pin": + project.Toolchain.Status, pin = "released", Tool{} + } + project.Toolchain.Tools.Normalizer = pins["normalizer"] + project.Toolchain.Tools.Emitters = map[string]Tool{"go": pins["go-emitter"]} + if component == "normalizer" { + project.Toolchain.Tools.Normalizer = pin + } else { + project.Toolchain.Tools.Emitters["go"] = pin + } + if got, err := NewPipeline(context.Background(), &project, options); err == nil { + got.Close() + t.Fatal("invalid library selection accepted") + } + if _, err := os.Stat(options.WorkDir); !os.IsNotExist(err) { + t.Fatal("rejected selection created work files") + } + }) + } + } + p.Toolchain.Status = "released" + p.Toolchain.Tools.Normalizer, p.Toolchain.Tools.Emitters = pins["normalizer"], map[string]Tool{"go": pins["go-emitter"]} + if accepted, err := NewPipeline(context.Background(), p, o); err != nil { + t.Fatalf("matching host pins rejected: %v", err) + } else { + accepted.Close() + } +} + +func TestGoLibrariesHonorCancellation(t *testing.T) { + p, o := goLibraryProject(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + pipeline, err := NewPipeline(ctx, p, o) + if err != nil { + t.Fatal(err) + } + defer pipeline.Close() + cancel() + doc := &Document{} + if err := pipeline.Normalize(doc); !errors.Is(err, context.Canceled) || doc.Model.APIVersion != "" { + t.Fatalf("canceled normalization changed the model: %v", err) + } + if err := pipeline.emit(&BuiltTarget{Target: Target{Language: "go"}}); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled emission: %v", err) + } + if pipeline.work != "" { + t.Fatal("canceled operation created work files") + } +} + +func TestGoLibrariesPythonOnlyCatalogDoesNotRequireGoEmitter(t *testing.T) { + p, o := testProject(t, "python") + o.UseGoLibraries = true + pipeline, err := NewPipeline(context.Background(), p, o) + if err != nil { + t.Fatal(err) + } + defer pipeline.Close() + p.Toolchain.Status = "released" + p.Toolchain.Tools.Normalizer = pipeline.identities["normalizer"] + selected, err := NewPipeline(context.Background(), p, o) + if err != nil { + t.Fatalf("Python-only catalog required a Go emitter pin: %v", err) + } + defer selected.Close() + if _, selectedGo := selected.tools["go-emitter"]; selectedGo { + t.Fatal("Python-only catalog selected a Go emitter") + } +} diff --git a/tooling/extension-bindings/orchestrator/pipeline.go b/tooling/extension-bindings/orchestrator/pipeline.go index 77a19ea..0c19198 100644 --- a/tooling/extension-bindings/orchestrator/pipeline.go +++ b/tooling/extension-bindings/orchestrator/pipeline.go @@ -62,6 +62,9 @@ type Build struct { } func NewPipeline(ctx context.Context, p *Project, o Options) (*Pipeline, error) { + if o.UseGoLibraries && (o.Normalizer != "" || o.GoEmitter != "") { + return nil, fmt.Errorf("Go library mode conflicts with standalone normalizer or Go emitter selection") + } if p.Toolchain.Status == "released" && len(o.Overrides) > 0 { return nil, fmt.Errorf("released generation forbids extension overrides") } @@ -84,6 +87,20 @@ func NewPipeline(ctx context.Context, p *Project, o Options) (*Pipeline, error) o.LinkedGoProfiler = &identity // Snapshot the caller's supplied identity. } pipeline := &Pipeline{Context: ctx, Project: p, Options: o, tools: map[string]string{}, identities: map[string]Tool{}, cores: map[string]map[string]any{}, HTTPClient: &http.Client{Timeout: 60 * time.Second, CheckRedirect: func(_ *http.Request, _ []*http.Request) error { return fmt.Errorf("redirects are forbidden") }}} + if o.UseGoLibraries { + keys := []string{"normalizer"} + for _, set := range p.Catalog.Packages { + if _, usesGo := set.Languages["go"]; usesGo { + keys = append(keys, "go-emitter") + break + } + } + for _, key := range keys { + if _, _, err := pipeline.goLibraryTool(key); err != nil { + return nil, err + } + } + } for _, path := range []string{o.WorkDir, o.Cache} { if path != "" { if err := rejectSymlinkPath(path); err != nil { @@ -190,6 +207,9 @@ func (p *Pipeline) python(args ...string) ([]byte, error) { return p.run(p.Project.Root, p.pythonEnv(), p.Options.Python, append(flags, args...)...) } func (p *Pipeline) tool(key string) (string, Tool, error) { + if p.Options.UseGoLibraries && (key == "normalizer" || key == "go-emitter") { + return p.goLibraryTool(key) + } if path, ok := p.tools[key]; ok { return path, p.identities[key], nil } @@ -578,6 +598,9 @@ func (p *Pipeline) Normalize(doc *Document) error { if doc.Model.APIVersion != "" { return nil } + if p.Options.UseGoLibraries { + return p.normalizeGoLibrary(doc) + } _, core, err := p.core(doc.CoreVersion) if err != nil { return err @@ -785,6 +808,9 @@ func (p *Pipeline) emitterInputs(bt *BuiltTarget) (string, string, error) { return modelPath, targetPath, nil } func (p *Pipeline) emit(bt *BuiltTarget) error { + if p.Options.UseGoLibraries && bt.Target.Language == "go" { + return p.emitGoLibrary(bt) + } modelPath, targetPath, err := p.emitterInputs(bt) if err != nil { return err From 317f265fe463883c4144b7b5a7c0500460e03f9d Mon Sep 17 00:00:00 2001 From: Colin Lacy Date: Tue, 10 Nov 2026 13:14:10 +0000 Subject: [PATCH 7/7] reverts normalizer changes --- EXTENSION_RELEASES.md | 146 +---------- tooling/common/build_extensions_site.py | 104 ++------ tooling/common/test_build_extensions_site.py | 112 +-------- tooling/extension-bindings/README.md | 19 +- .../extension-bindings/normalizer/resolve.go | 57 ++--- .../normalizer/resolve_test.go | 229 ++---------------- .../extension-bindings/normalizer/validate.go | 8 +- .../normalizer/validate_test.go | 54 +---- 8 files changed, 85 insertions(+), 644 deletions(-) diff --git a/EXTENSION_RELEASES.md b/EXTENSION_RELEASES.md index 7d446b8..c9bd1eb 100644 --- a/EXTENSION_RELEASES.md +++ b/EXTENSION_RELEASES.md @@ -1,52 +1,18 @@ -# First-party extension release contract +# Extension release and compatibility contract ## Purpose -This contract governs extensions published by Runtime Conditions and the -first-party tools that produce and consume them. It defines the reference forms -supported by that tooling; the core profile specification remains unchanged. - An extension release is an immutable semantic vocabulary artifact. Its Runtime Conditions `apiVersion` identifies the document schema understood by validators; it does not identify the extension's semantic release. -The canonical identity rules below are the target for an incremental tooling -migration. Inputs that still use artifact URLs as identifiers must be migrated. -First-party tooling MUST reject unsupported reference forms rather than retain -legacy acceptance, aliases, automatic conversion, or fallback behavior. - ## Identity and versioning -- First-party tooling MUST accept and emit only complete `:` extension references. Unversioned identifiers and artifact-URL extension references MUST be rejected. -- A first-party extension release MUST use a complete, explicitly versioned identifier of the form `https://runtimeconditions.io/extensions/:` or `https://runtimeconditions.io/extensions//:`. -- The complete identifier MUST appear unchanged in the definition's `metadata.id`, profile extension declarations and selectors, dependency references, normalized model identity and ownership fields, and binding or SDK manifest identity fields. -- A generator MUST copy that identifier verbatim. It MUST NOT reconstruct it from a retrieval URL, a native package version, or a separate version annotation. -- `metadata.version` is optional descriptive metadata. When present, it MUST exactly equal the version in the canonical identifier. Tools MUST NOT append it to `metadata.id`. - A published extension identifier MUST resolve permanently to the same semantic definition. - A semantic change MUST produce a new extension identifier. Existing identifiers are never repointed or overwritten. -- Provider extensions SHOULD use a Semantic Versioning 2.0.0 release. Existing labels such as `v1alpha1` remain valid first-party release versions. +- Provider extensions SHOULD encode a Semantic Versioning 2.0.0 release in the identifier and repeat it as `metadata.version` so humans and automation can distinguish document schema version from extension semantic version. - An additive vocabulary or validation change increments the minor version, a meaning-changing removal or incompatible reclassification increments the major version, and a correction that does not change accepted vocabulary or profile meaning increments the patch version. -- During pre-adoption development, `0.x.y` releases may introduce incompatible conventions through new identifiers and package versions. Published identifiers, tags, and assets remain immutable. - -Canonical reference examples: - -```yaml -extensions: - - https://runtimeconditions.io/extensions/common-integrations:v1alpha1 - - https://runtimeconditions.io/extensions/env-configuration:v1alpha1 - - https://runtimeconditions.io/extensions/aws/s3:0.2.0 -``` - -A definition for the first reference declares: - -```yaml -metadata: - id: https://runtimeconditions.io/extensions/common-integrations:v1alpha1 - version: v1alpha1 -``` +- During pre-adoption development, `0.x.y` releases may replace earlier experimental conventions without compatibility promises. Before public adoption, the same immutability and compatibility rules apply to every published identifier. -The `version` annotation may be omitted when authoring a release; `metadata.id` -still supplies its complete identity. Optionality does not permit changing an -already published definition. Native binding package versions and the Runtime -Conditions document `apiVersion` are versioned independently. +The AWS S3 candidate therefore uses `https://runtimeconditions.io/aws/aws-s3:0.1.0` while retaining `apiVersion: runtimeconditions.io/v1alpha1` as the Runtime Conditions document schema version. ## Authoritative provenance @@ -77,100 +43,10 @@ Humans review semantic bridge changes, added or removed operations, resource cla ## HTTPS catalog lookup -The canonical identifier and the artifact retrieval URL are separate values. -A first-party catalog or configured resolver maps the complete identifier to -the location of its definition. The `rc` namespace used for providerless entries -inside this repository is not inserted into their canonical identifiers. - -Resolvers MUST preserve the requested identifier for equality, dependency -deduplication, cache identity, cycle detection, and returned catalog keys. A -retrieval mapping MUST NOT replace that identifier with the artifact URL. The -retrieved first-party definition's `metadata.id` MUST match the requested -identifier exactly. Missing mappings, unavailable artifacts, identity mismatches, -and inconsistent version annotations MUST fail resolution. - -The first-party publisher accepts only the canonical identifier forms in this -contract. A providerless identifier omits the catalog's internal `rc` namespace. -The artifact layout is -`https://runtimeconditions.io/extensions//releases//runtimeconditions.extension.yaml` -or -`https://runtimeconditions.io/extensions///releases//runtimeconditions.extension.yaml`. -Each version occupies one URL path segment and is percent-encoded there when -necessary. Its exact spelling in the canonical identifier remains unchanged. - -For example, the identifier -`https://runtimeconditions.io/extensions/common-integrations:v1alpha1` maps to -`https://runtimeconditions.io/extensions/common-integrations/releases/v1alpha1/runtimeconditions.extension.yaml`. -The publisher copies the definition bytes unchanged. It does not create artifact -URL identities, legacy routes, or aliases. Definition IDs and dependency -references must be canonical; an optional `metadata.version` must match the -version in the definition ID. - -### Generated lookup index - -The publisher writes `https://runtimeconditions.io/extensions/index.yaml`: - -```yaml -apiVersion: runtimeconditions.io/extension-index/v1alpha1 -kind: RuntimeConditionsExtensionIndex -extensions: - - id: https://runtimeconditions.io/extensions/common-integrations:v1alpha1 - url: https://runtimeconditions.io/extensions/common-integrations/releases/v1alpha1/runtimeconditions.extension.yaml - sourceSha256: -``` - -Entries are sorted by their exact `id` strings. Each canonical ID appears once; -`url` is its retrieval location and `sourceSha256` is the lowercase hexadecimal -SHA-256 of its published bytes. This generated first-party lookup contract does -not extend the core profile format. Consumers must explicitly integrate the -lookup entries into their retrieval mappings or dependency locks; writing the -index does not change consumer discovery behavior by itself. - -The publisher MUST reject unsupported IDs or dependency reference forms, -inconsistent version annotations, duplicate IDs, and conflicting output paths -before writing artifacts. Canonical definitions and the lookup index are the -only extension publication outputs. Existing Go package discovery pages remain -a separate publication output. - -Support for other retrieval backends remains an implementation capability. -First-party tools must keep the complete versioned reference as the identity -regardless of how the artifact is retrieved. - -## Incremental migration gates - -Each implementation increment must identify its files and validation before -editing. Consumer support must precede changes to published definition identities. - -1. **Record the tooling contract.** Establish the complete canonical identifier, - optional version annotation, retrieval mapping, and strict rejection rules in - this document. This step changes documentation only. -2. **Prepare consumers with synthetic fixtures.** Audit the shared resolver, - binding normalizer and emitters, site publishing, and language profilers. - Verify that canonical identifiers survive resolution, ownership checks, and - profile generation unchanged, including when a version annotation is present - or absent. Verify that artifact-URL extension references are rejected. Keep - cross-repository compatibility checks outside the shared binding-tooling suite. -3. **Migrate the common/env dependency pair.** Replace artifact-URL identities - with canonical references and use the canonical publication layout. Update - the definitions, dependency edge, catalog entries, generated metadata, and native package versions in bounded - increments. Regenerate bindings through the tooling; do not hand-edit generated - identities or digests. Verify installed consumers before publication. -4. **Update the CLI integration.** After compatible profiler and binding releases - are available, update the required dependency pins and fixtures. Require the - installed CLI to emit the canonical references exactly and reject inputs - that still expose artifact-URL extension references. -5. **Expand one dependency group at a time.** Apply the same checks to remaining - extensions, SDK mappings, language bindings, profilers, demos, and adapters. - Each group must resolve, generate, and validate before the next group moves. - -For each migrated group, record the canonical identifiers, retrieval -locations, affected dependency edges, artifact digests, package versions, and -consumer release requirements. A new definition identity changes its semantic -digest even when its vocabulary is unchanged; dependent models and bindings must -be regenerated. Publication continues through the existing release verification -and dry-run process. - -Incremental implementation does not authorize compatibility branches. The -current catalog cannot pass strict site assembly until its definitions and -dependency references have been migrated. Complete that migration and the -required consumer updates before deploying the resulting site. +A definition uses `metadata.uri` and `metadata.version`; its identifier is +`:`. The canonical URI has `//` or `/`. +The versioned artifact is retrieved from +`https:///extensions////runtimeconditions.extension.yaml`. +Providerless identities use `rc` for lookup without changing the declared URI. +A missing definition is not found; no alternate path or provider is tried. +Only HTTPS retrieval is currently supported. `file:` and `oci:` are deferred. diff --git a/tooling/common/build_extensions_site.py b/tooling/common/build_extensions_site.py index 001e36a..d6d9e11 100644 --- a/tooling/common/build_extensions_site.py +++ b/tooling/common/build_extensions_site.py @@ -1,43 +1,29 @@ -"""Publish canonical extension definitions, their lookup index, and Go discovery. - -Only complete : extension references are supported. Definition -bytes are published unchanged beneath release paths; extensions/index.yaml maps -their exact identities to retrieval URLs and source checksums. +"""Build a static site tree so every catalog extension is reachable at its +own `metadata.id` URL. + +Each extension's id is the URL it's meant to be resolved at, e.g. +https://runtimeconditions.io/extensions/aws-s3/0.1.0/runtimeconditions.extension.yaml. +This copies each extension.yaml to the output directory at the path portion +of that URL, so publishing the output directory as a static site (GitHub +Pages or otherwise) makes every id resolvable by itself - no separate +mapping to keep in sync. Generated Go modules also receive discovery pages +at their public module coordinates from the package catalog. """ from __future__ import annotations import argparse -import re +import shutil import sys -from hashlib import sha256 from pathlib import Path from html import escape -from urllib.parse import quote - -import yaml +from urllib.parse import urlparse from go_bindings import GoBinding, load_bindings, validate_source from serialization import read_document REPO_ROOT = Path(__file__).resolve().parents[2] EXPECTED_HOST = "runtimeconditions.io" -CANONICAL_REFERENCE = re.compile( - rf"https://{re.escape(EXPECTED_HOST)}/extensions/" - r"([A-Za-z0-9][A-Za-z0-9._-]*(?:/[A-Za-z0-9][A-Za-z0-9._-]*)?):([^/?#\s]+)" -) - - -def canonical_parts(reference: str) -> tuple[str, str]: - match = CANONICAL_REFERENCE.fullmatch(reference) if isinstance(reference, str) else None - if match is None: - raise ValueError(f"expected canonical extension reference :, got {reference!r}") - location, version = match.groups() - if location.startswith("rc/"): - raise ValueError("canonical providerless references must omit the internal rc namespace") - if version in {".", ".."} or "\\" in version or any(ord(char) < 32 or ord(char) == 127 for char in version): - raise ValueError(f"invalid canonical extension version in {reference!r}") - return location, version def build_go_pages(output_dir: Path, root: Path) -> None: @@ -110,69 +96,31 @@ def build(output_dir: Path, root: Path = REPO_ROOT) -> int: return 1 errors = [] - entries = {} - outputs = {} for extension_path in extension_paths: + doc = read_document(extension_path) + extension_id = doc.get("metadata", {}).get("id", "") + parsed = urlparse(extension_id) relative = extension_path.relative_to(root) - try: - doc = read_document(extension_path) - metadata = doc.get("metadata") - if doc.get("kind") != "RuntimeConditionsExtensionDefinition" or not isinstance(metadata, dict): - raise ValueError("expected extension definition with metadata object") - extension_id = metadata.get("id") - location, version = canonical_parts(extension_id) - if "version" in metadata and metadata["version"] != version: - raise ValueError("metadata.version must exactly match the canonical reference version") - spec = doc.get("spec", {}) - if not isinstance(spec, dict) or not isinstance(spec.get("dependencies", []), list): - raise ValueError("expected spec object with a dependencies array when present") - for dependency in spec.get("dependencies", []): - canonical_parts(dependency) - destination = Path("extensions", *location.split("/"), "releases", version, "runtimeconditions.extension.yaml") - if extension_id in entries: - raise ValueError(f"duplicate extension id {extension_id!r}") - if destination in outputs: - raise ValueError(f"conflicting output path {destination}") - source = extension_path.read_bytes() - outputs[destination] = source - entries[extension_id] = { - "id": extension_id, - "url": f"https://{EXPECTED_HOST}/" + quote(destination.as_posix(), safe="/"), - "sourceSha256": sha256(source).hexdigest(), - } - except (ValueError, OSError) as error: - errors.append(f"{relative}: {error}") - - index = { - "apiVersion": "runtimeconditions.io/extension-index/v1alpha1", - "kind": "RuntimeConditionsExtensionIndex", - "extensions": [entries[identifier] for identifier in sorted(entries)], - } - outputs[Path("extensions/index.yaml")] = yaml.safe_dump( - index, sort_keys=False, allow_unicode=True, width=4096 - ).encode("utf-8") - for relative, source in outputs.items(): - destination = output_dir / relative - if (not destination.resolve().is_relative_to(output_dir.resolve()) - or any(parent.exists() and not parent.is_dir() for parent in destination.parents) - or destination.exists() and (not destination.is_file() or destination.read_bytes() != source)): - errors.append(f"conflicting output path {relative}") + + if parsed.scheme != "https" or parsed.netloc != EXPECTED_HOST: + errors.append(f"{relative}: metadata.id {extension_id!r} is not an https://{EXPECTED_HOST}/... URL") + continue + + destination = output_dir / parsed.path.lstrip("/") + destination.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(extension_path, destination) + print(f"{relative} -> {parsed.path}") if errors: for error in errors: print(error, file=sys.stderr) - print(f"\nExtension publication failed with {len(errors)} error(s).", file=sys.stderr) + print(f"\n{len(errors)} extension(s) have an id that can't be published this way.", file=sys.stderr) return 1 try: build_go_pages(output_dir, root) - for relative, source in outputs.items(): - destination = output_dir / relative - destination.parent.mkdir(parents=True, exist_ok=True) - destination.write_bytes(source) - print(f"extension publication -> /{relative.as_posix()}") except (ValueError, KeyError, OSError) as error: - print(f"Site assembly failed: {error}", file=sys.stderr) + print(f"Go binding discovery failed: {error}", file=sys.stderr) return 1 return 0 diff --git a/tooling/common/test_build_extensions_site.py b/tooling/common/test_build_extensions_site.py index 241fd3c..c590006 100644 --- a/tooling/common/test_build_extensions_site.py +++ b/tooling/common/test_build_extensions_site.py @@ -2,10 +2,7 @@ from __future__ import annotations -from contextlib import redirect_stderr -from hashlib import sha256 from html.parser import HTMLParser -from io import StringIO from pathlib import Path import tempfile import unittest @@ -46,20 +43,10 @@ def setUp(self) -> None: self.write_catalog() self.definition = self.root / "catalog/vendor/service/releases/0.1.0/runtimeconditions.extension.yaml" self.definition.parent.mkdir(parents=True) - self.document = {"kind": "RuntimeConditionsExtensionDefinition", "metadata": { - "id": "https://runtimeconditions.io/extensions/vendor/service:0.1.0", "version": "0.1.0"}} - self.write_definition() + self.definition.write_text(yaml.safe_dump({"kind": "RuntimeConditionsExtensionDefinition", "metadata": { + "id": "https://runtimeconditions.io/extensions/vendor/service/0.1.0/runtimeconditions.extension.yaml"}})) self.write_source() - def write_definition(self) -> None: - self.definition.write_text(yaml.safe_dump(self.document), encoding="utf-8") - - def assert_rejected(self, message: str) -> None: - errors = StringIO() - with redirect_stderr(errors): - self.assertEqual(build(self.output, self.root), 1) - self.assertIn(message, errors.getvalue()) - def write_catalog(self) -> None: path = self.root / "tooling/extension-bindings/packages.yaml" path.parent.mkdir(parents=True, exist_ok=True) @@ -82,101 +69,8 @@ def test_site_contains_discovery_install_and_unaliased_import(self) -> None: self.assertIn("go get runtimeconditions.io/x/vendor/service@v0.1.0", page) self.assertFalse((self.output / "x/vendor/pending").exists()) self.assertIn("runtimeconditions.io/x/vendor/service", (self.output / "x/index.html").read_text()) - published = self.output / "extensions/vendor/service/releases/0.1.0/runtimeconditions.extension.yaml" + published = self.output / "extensions/vendor/service/0.1.0/runtimeconditions.extension.yaml" self.assertEqual(published.read_bytes(), self.definition.read_bytes()) - index = yaml.safe_load((self.output / "extensions/index.yaml").read_text()) - self.assertEqual(index, { - "apiVersion": "runtimeconditions.io/extension-index/v1alpha1", - "kind": "RuntimeConditionsExtensionIndex", - "extensions": [{ - "id": self.document["metadata"]["id"], - "url": "https://runtimeconditions.io/extensions/vendor/service/releases/0.1.0/runtimeconditions.extension.yaml", - "sourceSha256": sha256(published.read_bytes()).hexdigest(), - }], - }) - self.assertFalse((self.output / "extensions/vendor/service:0.1.0").exists()) - self.assertFalse((self.output / "extensions/vendor/service/0.1.0").exists()) - - def test_providerless_and_digest_versions_publish_exact_ids(self) -> None: - for version in ("v1alpha1", "sha256:abc123"): - with self.subTest(version=version): - reference = f"https://runtimeconditions.io/extensions/common-integrations:{version}" - self.document["metadata"] = {"id": reference} - self.write_definition() - output = self.output / version - self.assertEqual(build(output, self.root), 0) - artifact = output / f"extensions/common-integrations/releases/{version}/runtimeconditions.extension.yaml" - self.assertEqual(artifact.read_bytes(), self.definition.read_bytes()) - entry = yaml.safe_load((output / "extensions/index.yaml").read_text())["extensions"][0] - self.assertEqual(entry["id"], reference) - encoded = version.replace(":", "%3A") - self.assertEqual(entry["url"], f"https://runtimeconditions.io/extensions/common-integrations/releases/{encoded}/runtimeconditions.extension.yaml") - - def test_legacy_and_invalid_references_are_rejected_before_writing(self) -> None: - for reference in ( - "https://runtimeconditions.io/extensions/vendor/service/0.1.0/runtimeconditions.extension.yaml", - "https://runtimeconditions.io/extensions/vendor/service", - "https://runtimeconditions.io/extensions/vendor/service:", - "https://runtimeconditions.io/extensions/rc/service:0.1.0", - "https://runtimeconditions.io/extensions/vendor/service:..", - "https://runtimeconditions.io/extensions/vendor/../escape:1", - "https://runtimeconditions.io/extensions/vendor/service:1?query=value", - "https://runtimeconditions.io/extensions/vendor/service:1#fragment", - "https://runtimeconditions.io/extensions/vendor/service:1\x00", - "http://runtimeconditions.io/extensions/vendor/service:1", - "https://other.example/extensions/vendor/service:1", - None, - ): - with self.subTest(reference=reference): - self.document["metadata"]["id"] = reference - self.write_definition() - self.assert_rejected("canonical") - self.assertFalse(self.output.exists()) - - def test_version_annotation_must_match_canonical_id(self) -> None: - for version in ("0.2.0", "", None, 1): - with self.subTest(version=version): - self.document["metadata"]["version"] = version - self.write_definition() - self.assert_rejected("metadata.version") - self.assertFalse(self.output.exists()) - - def test_legacy_dependency_reference_is_rejected(self) -> None: - self.document["spec"] = {"dependencies": [ - "https://runtimeconditions.io/extensions/common-integrations/v1alpha1/runtimeconditions.extension.yaml"]} - self.write_definition() - self.assert_rejected("canonical") - self.assertFalse(self.output.exists()) - - def test_duplicate_id_is_rejected_before_writing(self) -> None: - duplicate = self.root / "catalog/vendor/duplicate/releases/0.1.0/runtimeconditions.extension.yaml" - duplicate.parent.mkdir(parents=True) - duplicate.write_bytes(self.definition.read_bytes()) - self.assert_rejected("duplicate extension id") - self.assertFalse(self.output.exists()) - - def test_conflicting_artifact_path_is_rejected_before_writing(self) -> None: - artifact = self.output / "extensions/vendor/service/releases/0.1.0/runtimeconditions.extension.yaml" - artifact.parent.mkdir(parents=True) - artifact.write_bytes(b"unrelated content\n") - self.assert_rejected("conflicting output path") - self.assertEqual(artifact.read_bytes(), b"unrelated content\n") - self.assertFalse((self.output / "extensions/index.yaml").exists()) - self.assertFalse((self.output / "x").exists()) - - def test_lookup_entries_are_sorted_and_reproducible(self) -> None: - additional = self.root / "catalog/rc/common-integrations/common-v1alpha1.yaml" - additional.parent.mkdir(parents=True) - reference = "https://runtimeconditions.io/extensions/common-integrations:v1alpha1" - additional.write_text(yaml.safe_dump({"kind": "RuntimeConditionsExtensionDefinition", "metadata": {"id": reference}})) - self.assertEqual(build(self.output, self.root), 0) - index = (self.output / "extensions/index.yaml").read_bytes() - entries = yaml.safe_load(index)["extensions"] - self.assertEqual([entry["id"] for entry in entries], [reference, self.document["metadata"]["id"]]) - self.assertEqual(entries[0]["sourceSha256"], sha256(additional.read_bytes()).hexdigest()) - other_output = self.output.parent / "second-site" - self.assertEqual(build(other_output, self.root), 0) - self.assertEqual((other_output / "extensions/index.yaml").read_bytes(), index) def test_mismatched_generated_module_fails_site_build(self) -> None: (self.root / "bindings/service/go/go.mod").write_text("module github.com/old/module\n") diff --git a/tooling/extension-bindings/README.md b/tooling/extension-bindings/README.md index b6afeae..363de28 100644 --- a/tooling/extension-bindings/README.md +++ b/tooling/extension-bindings/README.md @@ -9,13 +9,6 @@ GitHub Actions generates and commits Go and Python bindings to `main`, then a separate manual workflow publishes verified Go module tags and release assets. Registry publication of Python bindings remains later work. -The first-party identity contract and migration gates are in -[`EXTENSION_RELEASES.md`](../../EXTENSION_RELEASES.md). The target contract uses -the complete versioned `metadata.id` unchanged throughout resolution, models, -manifests, and generated profiles. Its migration is incremental; existing -definitions and released bindings still use artifact URL identities. The core -profile specification is unchanged by this tooling convention. - ## Automatic Go and Python binding generation `.github/workflows/binding-generate.yml` runs on relevant `main` changes to the @@ -385,12 +378,12 @@ go run ./cmd/rc-binding-model \ --dependency-lock-output ``` -Output paths must not already exist. HTTPS retrieval records the artifact URL -and exact source digest in the dependency lock. Canonical identity support and -its retrieval mappings must pass the migration gates in -[`EXTENSION_RELEASES.md`](../../EXTENSION_RELEASES.md) before catalog identities -change. An artifact URL used for transport must not replace the requested -identifier. A previously written lock is supplied with `--dependency-lock`. +Output paths must not already exist. HTTPS retrieval requires a dependency lock +with an exact source digest and the URL derived from the requested identifier: +`https:///extensions////runtimeconditions.extension.yaml`. +Providerless identities use provider `rc` for retrieval while retaining their +declared identity. HTTPS redirects are rejected; `file:` and `oci:` retrieval +are unsupported. A previously written lock is supplied with `--dependency-lock`. The extension cache is content-addressed. Each extension entry must be named ``, `.yaml`, or diff --git a/tooling/extension-bindings/normalizer/resolve.go b/tooling/extension-bindings/normalizer/resolve.go index d8028ff..24d4acf 100644 --- a/tooling/extension-bindings/normalizer/resolve.go +++ b/tooling/extension-bindings/normalizer/resolve.go @@ -339,7 +339,7 @@ func backendRank(backend string) int { } func (r *Resolver) Resolve(ctx context.Context, rootID string) (ResolvedClosure, error) { - if _, _, err := parseExtensionIdentifier(rootID); err != nil { + if _, err := parseExtensionIdentifier(rootID); err != nil { return ResolvedClosure{}, diagnostic("unknown-extension", "RCB1203", rootID, "", err.Error()) } state := visitState{activeIndex: map[string]int{}, complete: map[string]bool{}} @@ -451,7 +451,7 @@ func (r *Resolver) visit(ctx context.Context, id string, state *visitState) erro } func (r *Resolver) load(ctx context.Context, id string) (ResolvedDocument, error) { - parsed, _, err := parseExtensionIdentifier(id) + parsed, err := parseExtensionIdentifier(id) if err != nil { return ResolvedDocument{}, diagnostic("unknown-extension", "RCB1207", id, "", err.Error()) } @@ -467,54 +467,31 @@ func (r *Resolver) load(ctx context.Context, id string) (ResolvedDocument, error return r.finishLoad(id, data, candidate.backend, candidate.locator) } if strings.EqualFold(parsed.Scheme, "https") { - return r.loadHTTPS(ctx, id, r.config.Locks[id].Locator) + if parsed.Hostname() == "" || parsed.User != nil { + return ResolvedDocument{}, diagnostic("unknown-extension", "RCB1207", id, "", "HTTPS retrieval requires a host and forbids credentials") + } + return r.loadHTTPS(ctx, id, id) } return ResolvedDocument{}, diagnostic("unknown-extension", "RCB1207", id, "", fmt.Sprintf("automatic retrieval for scheme %q is unavailable; supply the exact identifier through a catalog, package, cache, or override", parsed.Scheme)) } -// parseExtensionIdentifier requires :. The separator is the -// first literal colon in the final path component (or opaque URI body), excluding -// scheme and authority colons. Version labels may themselves contain colons. -// The complete reference retains its exact spelling throughout resolution. -func parseExtensionIdentifier(id string) (*url.URL, string, error) { +// parseExtensionIdentifier validates identity without selecting a retrieval +// backend or imposing a provider/service/version layout. The exact spelling is +// retained throughout resolution, model provenance, and dependency locks. +func parseExtensionIdentifier(id string) (*url.URL, error) { parsed, err := url.Parse(id) - if err != nil || !parsed.IsAbs() || strings.ContainsAny(id, "?#\\") || strings.ContainsFunc(id, func(r rune) bool { - return unicode.IsSpace(r) || unicode.IsControl(r) + if err != nil || !parsed.IsAbs() || strings.ContainsFunc(id, func(r rune) bool { + return unicode.IsSpace(r) || r < 0x20 || r == 0x7f }) { - return nil, "", fmt.Errorf("extension reference must be an absolute : URI without whitespace, query, or fragment") - } - component := parsed.Opaque - if component == "" { - path := parsed.EscapedPath() - component = path[strings.LastIndex(path, "/")+1:] - } - location, version, versioned := strings.Cut(component, ":") - if !versioned || location == "" || version == "" || version == "." || version == ".." || strings.Contains(version, "/") { - return nil, "", fmt.Errorf("extension reference must include a non-empty location and explicit version as :") - } - name, err := url.PathUnescape(location) - if err != nil || strings.HasSuffix(strings.ToLower(name), ".yaml") || strings.HasSuffix(strings.ToLower(name), ".yml") || strings.HasSuffix(strings.ToLower(name), ".json") { - return nil, "", fmt.Errorf("extension reference location must identify an extension, not a definition artifact URL") - } - if parsed.User != nil || (strings.EqualFold(parsed.Scheme, "https") && (parsed.Hostname() == "" || parsed.Opaque != "")) { - return nil, "", fmt.Errorf("extension reference forbids credentials and HTTPS references require a host") + return nil, fmt.Errorf("extension identifier must be a non-empty absolute URI with a scheme") } - return parsed, version, nil + return parsed, nil } func (r *Resolver) loadHTTPS(ctx context.Context, id, locator string) (ResolvedDocument, error) { if !r.config.Network { return ResolvedDocument{}, diagnostic("unknown-extension", "RCB1208", id, "", "network resolution is disabled") } - if locator == "" { - return ResolvedDocument{}, diagnostic("extension-dependency", "RCB1209", id, "", "HTTPS resolution requires an explicit retrieval locator") - } - parsed, err := url.Parse(locator) - if err != nil || !strings.EqualFold(parsed.Scheme, "https") || parsed.Hostname() == "" || parsed.User != nil || strings.ContainsAny(locator, "#\\") || strings.ContainsFunc(locator, func(r rune) bool { - return unicode.IsSpace(r) || unicode.IsControl(r) - }) { - return ResolvedDocument{}, diagnostic("unknown-extension", "RCB1207", id, "", "retrieval locator must be an HTTPS URL with a host and without credentials or a fragment") - } lock, locked := r.config.Locks[id] if (!locked || lock.SourceSHA256 == "" || lock.Locator == "") && !r.config.RecordNetworkLocks { return ResolvedDocument{}, diagnostic("extension-dependency", "RCB1209", id, "", "HTTPS resolution requires source digest and locator locks") @@ -541,7 +518,7 @@ func (r *Resolver) loadHTTPS(ctx context.Context, id, locator string) (ResolvedD if err != nil { return ResolvedDocument{}, err } - if locked && lock.SourceSHA256 != "" && SHA256Hex(data) != lock.SourceSHA256 { + if locked && SHA256Hex(data) != lock.SourceSHA256 { return ResolvedDocument{}, diagnostic("extension-dependency", "RCB1212", id, "", "HTTPS content does not match the dependency lock") } document, err := r.finishLoad(id, data, "https", response.Request.URL.String()) @@ -551,11 +528,11 @@ func (r *Resolver) loadHTTPS(ctx context.Context, id, locator string) (ResolvedD if lock.SemanticSHA256 != "" && document.SemanticSHA256 != lock.SemanticSHA256 { return ResolvedDocument{}, diagnostic("extension-dependency", "RCB1213", id, "", "HTTPS semantic digest does not match the dependency lock") } - if !locked || lock.SourceSHA256 == "" { + if !locked { if r.config.Locks == nil { r.config.Locks = map[string]LockEntry{} } - r.config.Locks[id] = LockEntry{SourceSHA256: document.SourceSHA256, SemanticSHA256: document.SemanticSHA256, SourceBackend: "https", Locator: locator} + r.config.Locks[id] = LockEntry{SourceSHA256: document.SourceSHA256, SemanticSHA256: document.SemanticSHA256, Locator: locator} } if r.config.CacheDir != "" { if err := os.MkdirAll(r.config.CacheDir, 0755); err != nil { diff --git a/tooling/extension-bindings/normalizer/resolve_test.go b/tooling/extension-bindings/normalizer/resolve_test.go index e30c32f..52490d0 100644 --- a/tooling/extension-bindings/normalizer/resolve_test.go +++ b/tooling/extension-bindings/normalizer/resolve_test.go @@ -25,7 +25,7 @@ func fixtureExtension(id string, dependencies []string, specBody string) []byte dependencyYAML.WriteByte('\n') } } - return []byte(fmt.Sprintf("apiVersion: runtimeconditions.io/v1alpha1\nkind: RuntimeConditionsExtensionDefinition\nmetadata:\n id: %q\n version: 1.0.0\nspec:\n%s%s", id, dependencyYAML.String(), specBody)) + return []byte(fmt.Sprintf("apiVersion: runtimeconditions.io/v1alpha1\nkind: RuntimeConditionsExtensionDefinition\nmetadata:\n id: %s\n version: 1.0.0\nspec:\n%s%s", id, dependencyYAML.String(), specBody)) } func kindSpec(name string) string { @@ -72,35 +72,25 @@ func closureIDs(closure ResolvedClosure) []string { func TestExtensionIdentifierSchemesAndExactSpelling(t *testing.T) { for _, id := range []string{ - "https://runtimeconditions.io/extensions/common-integrations:v1alpha1", - "https://example.test/arbitrary/nested/service:sha256:abc", - "https://EXAMPLE.test:8443/CaseSensitive/%73ervice:V1+build", - "file:///opt/extensions/service:1.0.0", "oci://example.test/extensions:1.0.0", - "urn:service:release-2026", "custom+resolver:service:1.0.0", + "https://example.test/arbitrary/nested/path/release.yaml?revision=abc", + "https://EXAMPLE.test:8443/CaseSensitive/%73ervice/V1+build", + "file:///opt/extensions/release.yaml", "oci://example.test/extensions@sha256:abc", + "urn:example:extension:release-2026", "custom+resolver:immutable-release", } { - parsed, version, err := parseExtensionIdentifier(id) - if err != nil || parsed.String() != id || version == "" { + parsed, err := parseExtensionIdentifier(id) + if err != nil || parsed.String() != id { t.Errorf("identifier %q changed or was rejected: %v", id, err) } } - for _, id := range []string{ - "", "relative/path:1", "/absolute/path:1", "//example.test/path:1", "1bad:release:1", - "urn:bad version:1", "https://example.test/%invalid:1", "https://example.test:8443", - "https://example.test/service:", "https://example.test/:1", "https://example.test/service:.", - "https://example.test/service:..", "https:///service:1", "https://user@example.test/service:1", - "https://example.test/service:1?revision=abc", "https://example.test/service:1#fragment", - "https://example.test/service:bad\\version", "https://example.test/service:bad\u0085version", - "https://example.test/definition.yaml:1.0.0", "file:///opt/definition.yml:1.0.0", - "https://example.test/definition.json:1.0.0", "https://example.test/definition%2Eyaml:1.0.0", - } { - if _, _, err := parseExtensionIdentifier(id); err == nil { + for _, id := range []string{"", "relative/path", "/absolute/path", "//example.test/path", "1bad:release", "urn:bad version", "https://example.test/%invalid"} { + if _, err := parseExtensionIdentifier(id); err == nil { t.Errorf("invalid extension identifier accepted: %q", id) } } } -func TestVersionedIdentifiersWithoutAnnotationResolveFromEveryLocalBackend(t *testing.T) { - for _, id := range []string{"https://example.test/service:1.0.0", "urn:service:1.0.0", "file:///opt/extensions/service:1.0.0", "oci://registry.example/extensions:1.0.0", "custom+resolver:service:1.0.0"} { +func TestVersionlessIdentifiersResolveFromEveryLocalBackend(t *testing.T) { + for _, id := range []string{"urn:example:release-a", "file:///opt/extensions/release.yaml", "oci://registry.example/extensions@sha256:abc", "custom+resolver:release-a"} { t.Run(id, func(t *testing.T) { data := bytes.ReplaceAll(fixtureExtension(id, nil, kindSpec("future.service")), []byte(" version: 1.0.0\n"), nil) directory := t.TempDir() @@ -141,44 +131,8 @@ func TestVersionedIdentifiersWithoutAnnotationResolveFromEveryLocalBackend(t *te } } -func TestUnversionedReferencesAreRejectedAcrossBackends(t *testing.T) { - for _, id := range []string{ - "https://runtimeconditions.io/extensions/common-integrations/v1alpha1/runtimeconditions.extension.yaml", - "https://runtimeconditions.io/extensions/common-integrations/v1alpha1/runtimeconditions.extension.yaml:v1alpha1", - "https://example.test/service", "https://example.test:8443", "urn:release-a", - "file:///opt/extensions/release.yaml", "oci://registry.example/extensions", "custom+resolver:release-a", - } { - t.Run(id, func(t *testing.T) { - data := fixtureExtension(id, nil, kindSpec("fixture.service")) - directory, cache := t.TempDir(), t.TempDir() - path := writeFixture(t, directory, "definition.yaml", data) - writeFixture(t, cache, SHA256Hex(data)+".yaml", data) - transport := &countingTransport{} - for name, config := range map[string]ResolverConfig{ - "catalog": {CatalogRoots: []string{directory}}, "package": {PackageRoots: []string{directory}}, - "cache": {CacheDir: cache}, "override": {Overrides: map[string]string{id: path}}, - "https": {Network: true, HTTPClient: &http.Client{Transport: transport}, Locks: map[string]LockEntry{id: {SourceSHA256: SHA256Hex(data), Locator: "https://artifacts.example.test/definition.yaml"}}}, - } { - t.Run(name, func(t *testing.T) { - config.Schemas = testSchemas(t) - resolver, err := NewResolver(config) - if err != nil { - t.Fatal(err) - } - if _, err = resolver.Resolve(context.Background(), id); err == nil || diagnosticCode(t, err) != "RCB1203" { - t.Fatalf("expected reference rejection, got %v", err) - } - }) - } - if transport.requests.Load() != 0 { - t.Fatal("invalid reference reached HTTP transport") - } - }) - } -} - func TestUnavailableRetrievalDoesNotInvalidateIdentifier(t *testing.T) { - for _, id := range []string{"urn:service:1.0.0", "oci://example.test/service:1.0.0", "file:///opt/extensions/service:1.0.0"} { + for _, id := range []string{"urn:example:release", "oci://example.test/release", "file:///opt/extensions/release.yaml"} { resolver := fixtureResolver(t) _, err := resolver.Resolve(context.Background(), id) if err == nil || diagnosticCode(t, err) != "RCB1207" || !strings.Contains(err.Error(), "automatic retrieval") { @@ -188,14 +142,14 @@ func TestUnavailableRetrievalDoesNotInvalidateIdentifier(t *testing.T) { } func TestRequestedIdentifierMustExactlyMatchDefinition(t *testing.T) { - id := "https://example.test/ReleaseA:1.0.0" + id := "urn:example:ReleaseA" path := writeFixture(t, t.TempDir(), "release.yaml", fixtureExtension(id, nil, kindSpec("future.service"))) - _, err := NewResolver(ResolverConfig{Schemas: testSchemas(t), Overrides: map[string]string{"https://example.test/releasea:1.0.0": path}}) + _, err := NewResolver(ResolverConfig{Schemas: testSchemas(t), Overrides: map[string]string{"urn:example:releasea": path}}) if err == nil || diagnosticCode(t, err) != "RCB1201" { t.Fatalf("case-sensitive identity mismatch accepted: %v", err) } resolver := fixtureResolver(t) - _, err = resolver.finishLoad("https://example.test/releasea:1.0.0", fixtureExtension(id, nil, kindSpec("future.service")), "https", "https://example.test/release.yaml") + _, err = resolver.finishLoad("urn:example:releasea", fixtureExtension(id, nil, kindSpec("future.service")), "https", "https://example.test/release.yaml") if err == nil || diagnosticCode(t, err) != "RCB1220" { t.Fatalf("retrieved identity mismatch accepted: %v", err) } @@ -394,7 +348,7 @@ func TestResolverBackendsProduceOneSemanticModel(t *testing.T) { writeFixture(t, cache, sourceDigest+".yaml", data) schemas := testSchemas(t) - httpsLocator := "https://artifacts.example.test/definitions/backend.yaml" + httpsLocator := id configs := map[string]ResolverConfig{ "override": {Schemas: schemas, Overrides: map[string]string{id: path}}, @@ -445,7 +399,7 @@ func TestDevelopmentHTTPSRecordsLockAndExplicitCache(t *testing.T) { id := "https://new.example.test/future/service:1.0.0" data := fixtureExtension(id, nil, kindSpec("future.kind")) cache := filepath.Join(t.TempDir(), "explicit-cache") - config := ResolverConfig{Schemas: testSchemas(t), Network: true, RecordNetworkLocks: true, CacheDir: cache, HTTPClient: &http.Client{Transport: staticHTTPTransport{data: data}}, Locks: map[string]LockEntry{id: {Locator: "https://artifacts.example.test/service.yaml"}}} + config := ResolverConfig{Schemas: testSchemas(t), Network: true, RecordNetworkLocks: true, CacheDir: cache, HTTPClient: &http.Client{Transport: staticHTTPTransport{data: data}}} resolver, err := NewResolver(config) if err != nil { t.Fatal(err) @@ -458,9 +412,6 @@ func TestDevelopmentHTTPSRecordsLockAndExplicitCache(t *testing.T) { if len(lock.Extensions) != 1 || lock.Extensions[0].SourceSHA256 != SHA256Hex(data) || lock.Extensions[0].SourceBackend != "https" { t.Fatalf("missing transport lock: %+v", lock) } - if resolver.config.Locks[id].SourceSHA256 != SHA256Hex(data) || lock.Extensions[0].SourceLocator != config.Locks[id].Locator { - t.Fatal("development resolution failed to complete the explicit retrieval lock") - } if _, err := os.Stat(filepath.Join(cache, SHA256Hex(data)+".yaml")); err != nil { t.Fatal(err) } @@ -487,146 +438,6 @@ func TestDevelopmentHTTPSRecordsLockAndExplicitCache(t *testing.T) { } } -type mappedHTTPTransport struct { - documents map[string][]byte - requests []string -} - -func (transport *mappedHTTPTransport) RoundTrip(request *http.Request) (*http.Response, error) { - locator := request.URL.String() - transport.requests = append(transport.requests, locator) - data, exists := transport.documents[locator] - if !exists { - return nil, fmt.Errorf("unexpected retrieval URL %q", locator) - } - return (staticHTTPTransport{data: data}).RoundTrip(request) -} - -func TestLockedHTTPSCanonicalIDsSurviveNormalization(t *testing.T) { - const rootID = "https://runtimeconditions.io/extensions/env-configuration:v1alpha1" - const dependencyID = "https://runtimeconditions.io/extensions/common-integrations:v1alpha1" - const rootURL = "https://artifacts.example.test/canonical/env/definition.yaml" - const dependencyURL = "https://artifacts.example.test/canonical/common/definition.yaml" - root := bytes.Replace(fixtureExtension(rootID, []string{dependencyID}, kindSpec("fixture.env")), []byte("version: 1.0.0"), []byte("version: v1alpha1"), 1) - dependency := bytes.Replace(fixtureExtension(dependencyID, nil, kindSpec("fixture.common")), []byte(" version: 1.0.0\n"), nil, 1) - transport := &mappedHTTPTransport{documents: map[string][]byte{rootURL: root, dependencyURL: dependency}} - config := ResolverConfig{ - Schemas: testSchemas(t), Network: true, HTTPClient: &http.Client{Transport: transport}, - Locks: map[string]LockEntry{ - rootID: {SourceSHA256: SHA256Hex(root), SourceBackend: "https", Locator: rootURL}, - dependencyID: {SourceSHA256: SHA256Hex(dependency), SourceBackend: "https", Locator: dependencyURL}, - }, - } - resolver, err := NewResolver(config) - if err != nil { - t.Fatal(err) - } - closure, err := resolver.Resolve(context.Background(), rootID) - if err != nil { - t.Fatal(err) - } - if got := strings.Join(transport.requests, "\n"); got != rootURL+"\n"+dependencyURL { - t.Fatalf("retrieved URLs = %q", got) - } - if closure.Root != rootID || closure.ByID[rootID].Definition.Metadata.ID != rootID || closure.ByID[dependencyID].Definition.Metadata.ID != dependencyID { - t.Fatal("retrieval URLs replaced canonical closure identifiers") - } - lock := BuildDependencyLock(closure) - model, err := Normalize(closure, lock, config.Schemas, testNormalizeConfig()) - if err != nil { - t.Fatal(err) - } - if model.RootExtension.ID != rootID || model.RootExtension.Version != "v1alpha1" || len(model.Extensions) != 2 || len(lock.Extensions) != 2 { - t.Fatal("canonical root or extension inventory changed during normalization") - } - for i, id := range []string{dependencyID, rootID} { - if model.Extensions[i].ID != id || lock.Extensions[i].ID != id || lock.Extensions[i].SourceLocator != config.Locks[id].Locator { - t.Fatalf("model or lock lost identity/location separation for %s", id) - } - } - if model.Extensions[0].Version != "" || len(model.DependencyEdges) != 1 || model.DependencyEdges[0] != (DependencyEdge{From: rootID, To: dependencyID}) { - t.Fatal("dependency identity, omitted version, or dependency edge changed") - } - owners := map[string]string{"fixture.env": rootID, "fixture.common": dependencyID} - if len(model.Vocabulary.Owners) != len(owners) { - t.Fatal("normalized vocabulary owner inventory changed") - } - for _, owner := range model.Vocabulary.Owners { - if owner.Owner != owners[owner.Kind] { - t.Fatalf("vocabulary owner changed to %q", owner.Owner) - } - } -} - -func TestLockedHTTPSMappedDefinitionValidation(t *testing.T) { - const id = "https://runtimeconditions.io/extensions/example:1.0.0" - const locator = "https://artifacts.example.test/example/definition.yaml" - for _, test := range []struct { - name, definitionID, sourceDigest, semanticDigest, code string - }{ - {"identity mismatch", "https://runtimeconditions.io/extensions/other:1.0.0", "", "", "RCB1220"}, - {"source digest mismatch", id, strings.Repeat("0", 64), "", "RCB1212"}, - {"semantic digest mismatch", id, "", strings.Repeat("f", 64), "RCB1213"}, - } { - t.Run(test.name, func(t *testing.T) { - data := fixtureExtension(test.definitionID, nil, kindSpec("fixture.example")) - digest := test.sourceDigest - if digest == "" { - digest = SHA256Hex(data) - } - transport := &mappedHTTPTransport{documents: map[string][]byte{locator: data}} - resolver, err := NewResolver(ResolverConfig{ - Schemas: testSchemas(t), Network: true, HTTPClient: &http.Client{Transport: transport}, - Locks: map[string]LockEntry{id: {SourceSHA256: digest, SemanticSHA256: test.semanticDigest, Locator: locator}}, - }) - if err != nil { - t.Fatal(err) - } - _, err = resolver.Resolve(context.Background(), id) - if err == nil || diagnosticCode(t, err) != test.code { - t.Fatalf("expected %s, got %v", test.code, err) - } - if len(transport.requests) != 1 || transport.requests[0] != locator { - t.Fatalf("retrieved URLs = %v", transport.requests) - } - }) - } -} - -func TestLockedHTTPSRejectsInvalidRetrievalURLs(t *testing.T) { - const id = "https://runtimeconditions.io/extensions/example:1.0.0" - for _, locator := range []string{"http://artifacts.example.test/definition.yaml", "file:///definition.yaml", "https:///missing-host.yaml", "https://user:password@artifacts.example.test/definition.yaml", "https://artifacts.example.test/%invalid", "https://artifacts.example.test/definition.yaml#fragment"} { - t.Run(locator, func(t *testing.T) { - transport := &countingTransport{} - resolver, err := NewResolver(ResolverConfig{ - Schemas: testSchemas(t), Network: true, HTTPClient: &http.Client{Transport: transport}, - Locks: map[string]LockEntry{id: {SourceSHA256: strings.Repeat("a", 64), Locator: locator}}, - }) - if err != nil { - t.Fatal(err) - } - _, err = resolver.Resolve(context.Background(), id) - if err == nil || diagnosticCode(t, err) != "RCB1207" || transport.requests.Load() != 0 { - t.Fatalf("invalid retrieval URL reached transport or lacked RCB1207: %v", err) - } - }) - } -} - -func TestHTTPSRequiresExplicitRetrievalMapping(t *testing.T) { - id := "https://runtimeconditions.io/extensions/example:1.0.0" - for _, record := range []bool{false, true} { - transport := &countingTransport{} - resolver, err := NewResolver(ResolverConfig{Schemas: testSchemas(t), Network: true, RecordNetworkLocks: record, HTTPClient: &http.Client{Transport: transport}}) - if err != nil { - t.Fatal(err) - } - if _, err = resolver.Resolve(context.Background(), id); err == nil || diagnosticCode(t, err) != "RCB1209" || transport.requests.Load() != 0 { - t.Fatalf("missing mapping was not rejected before HTTP retrieval: %v", err) - } - } -} - type countingTransport struct { requests atomic.Int64 } @@ -732,12 +543,10 @@ func TestLockedHTTPSClosureOverTLS(t *testing.T) { _, _ = w.Write(data) })) defer server.Close() - rootID, dependencyID := "https://example.test/root:1.0.0", "https://example.test/base:1.0.0" + rootID, dependencyID := server.URL+"/future/releases/root.yaml", server.URL+"/independent/base.yaml" definitions["/future/releases/root.yaml"] = fixtureExtension(rootID, []string{dependencyID}, kindSpec("future.root")) definitions["/independent/base.yaml"] = fixtureExtension(dependencyID, nil, kindSpec("future.base")) - config := ResolverConfig{Schemas: testSchemas(t), Network: true, RecordNetworkLocks: true, HTTPClient: server.Client(), Locks: map[string]LockEntry{ - rootID: {Locator: server.URL + "/future/releases/root.yaml"}, dependencyID: {Locator: server.URL + "/independent/base.yaml"}, - }} + config := ResolverConfig{Schemas: testSchemas(t), Network: true, RecordNetworkLocks: true, HTTPClient: server.Client()} resolver, err := NewResolver(config) if err != nil { t.Fatal(err) diff --git a/tooling/extension-bindings/normalizer/validate.go b/tooling/extension-bindings/normalizer/validate.go index c2834f6..2e467a3 100644 --- a/tooling/extension-bindings/normalizer/validate.go +++ b/tooling/extension-bindings/normalizer/validate.go @@ -110,15 +110,11 @@ func (s *Schemas) ValidateExtension(data map[string]any, definition ExtensionDef if definition.Kind != ExtensionKind { return diagnostic("structural", "RCB1104", definition.Metadata.ID, "/kind", "document is not a RuntimeConditionsExtensionDefinition") } - _, version, err := parseExtensionIdentifier(definition.Metadata.ID) - if err != nil { + if _, err := parseExtensionIdentifier(definition.Metadata.ID); err != nil { return diagnostic("structural", "RCB1105", definition.Metadata.ID, "/metadata/id", err.Error()) } - if definition.Metadata.Version != "" && definition.Metadata.Version != version { - return diagnostic("structural", "RCB1106", definition.Metadata.ID, "/metadata/version", "metadata.version must exactly match the version in metadata.id") - } for index, dependency := range definition.Spec.Dependencies { - if _, _, err := parseExtensionIdentifier(dependency); err != nil { + if _, err := parseExtensionIdentifier(dependency); err != nil { return diagnostic("extension-dependency", "RCB1110", definition.Metadata.ID, fmt.Sprintf("/spec/dependencies/%d", index), err.Error()) } } diff --git a/tooling/extension-bindings/normalizer/validate_test.go b/tooling/extension-bindings/normalizer/validate_test.go index 7e7c829..878c00a 100644 --- a/tooling/extension-bindings/normalizer/validate_test.go +++ b/tooling/extension-bindings/normalizer/validate_test.go @@ -1,58 +1,6 @@ package normalizer -import ( - "bytes" - "testing" -) - -func TestExtensionReferenceValidation(t *testing.T) { - const canonical = "https://runtimeconditions.io/extensions/example:1.0.0" - const artifact = "https://runtimeconditions.io/extensions/example/1.0.0/runtimeconditions.extension.yaml" - for _, test := range []struct { - name, id, version, dependency, code, pointer string - }{ - {"matching annotation", canonical, "1.0.0", "", "", ""}, - {"omitted annotation", canonical, "", "", "", ""}, - {"label version", "https://runtimeconditions.io/extensions/example:v1alpha1", "v1alpha1", "", "", ""}, - {"digest version", "https://runtimeconditions.io/extensions/example:sha256:abc", "sha256:abc", "", "", ""}, - {"unversioned id with annotation", "https://example.test/service", "1.0.0", "", "RCB1105", "/metadata/id"}, - {"artifact id", artifact, "1.0.0", "", "RCB1105", "/metadata/id"}, - {"artifact id with appended version", artifact + ":1.0.0", "1.0.0", "", "RCB1105", "/metadata/id"}, - {"empty version suffix", "https://example.test/service:", "1.0.0", "", "RCB1105", "/metadata/id"}, - {"mismatched annotation", canonical, "2.0.0", "", "RCB1106", "/metadata/version"}, - {"annotation spelling mismatch", "https://example.test/service:v1alpha1", "V1alpha1", "", "RCB1106", "/metadata/version"}, - {"canonical dependency", canonical, "1.0.0", "https://example.test/dependency:v1alpha1", "", ""}, - {"unversioned dependency", canonical, "1.0.0", "https://example.test/dependency", "RCB1110", "/spec/dependencies/0"}, - {"artifact dependency", canonical, "1.0.0", artifact, "RCB1110", "/spec/dependencies/0"}, - {"artifact dependency with appended version", canonical, "1.0.0", artifact + ":1.0.0", "RCB1110", "/spec/dependencies/0"}, - } { - t.Run(test.name, func(t *testing.T) { - var dependencies []string - if test.dependency != "" { - dependencies = []string{test.dependency} - } - data := bytes.Replace(fixtureExtension(test.id, dependencies, kindSpec("fixture.service")), []byte(" version: 1.0.0\n"), nil, 1) - if test.version != "" { - data = bytes.Replace(data, []byte("spec:\n"), []byte(" version: "+test.version+"\nspec:\n"), 1) - } - definition, mapping, err := DecodeExtension(data) - if err != nil { - t.Fatal(err) - } - err = testSchemas(t).ValidateExtension(mapping, definition) - if test.code == "" { - if err != nil { - t.Fatal(err) - } - if definition.Metadata.ID != test.id || definition.Metadata.Version != test.version { - t.Fatal("validation changed the identity or version annotation") - } - } else if err == nil || diagnosticCode(t, err) != test.code || err.(*DiagnosticError).Diagnostic.JSONPointer != test.pointer { - t.Fatalf("expected %s at %s, got %v", test.code, test.pointer, err) - } - }) - } -} +import "testing" func TestLanguageSpecificEmittersMayHandleHyphens(t *testing.T) { data := []byte("apiVersion: runtimeconditions.io/v1alpha1\nkind: RuntimeConditionsExtensionDefinition\nmetadata:\n id: https://runtimeconditions.io/test/hyphen-id:1.0.0\n version: 1.0.0\nspec:\n kinds:\n - name: service\n fieldValues:\n - field: mode\n targetKind: service\n values: [direct-mode]\n")