diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6c4fa84..3a5692c 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -82,7 +82,7 @@ jobs: - name: Test run: go test ${{ matrix.test_args }} ./... - - name: Test discovery against DSC 3.3.0 + - name: Test discovery and resource operations against DSC 3.3.0 shell: pwsh run: | $arch = if ((go env GOHOSTARCH) -eq 'arm64') { 'aarch64' } else { 'x86_64' } diff --git a/README.md b/README.md index 673d27a..383816d 100644 --- a/README.md +++ b/README.md @@ -19,9 +19,15 @@ On Windows, use `go build -o dscpkg.exe ./cmd/dscpkg`. Pass repository origins in precedence order with repeatable `--repository` flags. A bare host defaults to HTTPS; specify `http://` explicitly for local testing. -For each resource, the first repository catalog containing that exact resource -type and version is selected. A later repository is not used to recover from an -integrity or signature error in the selected repository. +For each resource, the first repository that can satisfy the request is selected. +With `--version`, the resource version is exact. Without it, dscpkg selects the +newest compatible stable resource version **within that repository**, not the +newest version across all repositories. Numeric version components and prerelease +precedence are compared numerically, not lexically. Latest selection excludes +prereleases and resource versions without a package archive for the target +OS/architecture; it also considers older package versions when necessary. +A later repository or older resource version is never used to bypass malformed +metadata, integrity failures, or signing errors. ```sh dscpkg install \ @@ -31,16 +37,137 @@ dscpkg install \ --version 2026-06-30-preview ``` -`--packages-dir` selects the package cache and registry directory. By default it +`--packages-dir` selects the package cache and registry directory. All commands, +including discovery and DSC resource operations, also honor +`DSCPKG_PACKAGES_DIR`; the flag takes precedence. Without either override it uses `/dscpkg/packages`. `--platform` overrides the detected `os_arch` archive key, for example `linux_amd64`. +## Declare installed resources with DscPkg/Resource + +DSC **3.3.0** can discover the native `DscPkg/Resource` resource directly beside +the dscpkg executable. It does not need to install itself through dscpkg. +Use [`examples/00-resources.yaml`](examples/00-resources.yaml) as a dependency +configuration: + +```yaml +$schema: https://aka.ms/dsc/schemas/v3/bundled/config/document.json +resources: + - name: LinuxGroups + type: DscPkg/Resource + properties: + resource: DscExtras.Linux/groups + state: installed + repositories: + - https://bpoe.github.io/dsc-extras/ + - name: LinuxServices + type: DscPkg/Resource + properties: + resource: DscExtras.Linux/services + state: installed + repositories: + - https://bpoe.github.io/dsc-extras/ +``` + +Run `sudo dsc config set --file 00-resources.yaml` on compatible Linux. +The first instance installs the latest compatible shared `dscextras/linux` +package; the second recognizes its services manifest without another download, +registration, or executable copy. Subsequent configurations can use both resources +through `DscPkg/Discover`, with no per-package `DSC_RESOURCE_PATH` changes. +Use the same identity for installation and subsequent DSC runs. + +The instance properties are: + +- `resource` (required): DSC type, such as `DscExtras.Linux/groups`. +- `version` (optional): exact resource version. Add `version: 0.0.1` to pin it. +- `state` (optional): defaults to `installed`. `missing` is also an observed state; + Set rejects removal requests, without deleting files or registrations. +- `repositories` (optional): ordered origins used only when installation is + necessary. No implicit repository is configured; supply these when applying + a configuration to an empty store. +- `package`, `packageVersion`, `path`: read-only observed package identity, + package version, and absolute installation directory. + +`version` and `packageVersion` are independent: the latter describes the package +providing the resource, not the desired resource version. CLI +`--package-version` remains an independent exact selector. + +Get reports the exact installed version, or `state: missing`. Without a version, +it reports the newest locally installed matching version. Test uses that same +local inspection and reports `_inDesiredState` and differing properties using +DSC's `stateAndDiff` contract. **Neither Get nor Test contacts repositories, +creates a store, or writes state.** A compatible installation of any version +satisfies an unversioned desired state, including one supplied by a shared package +without an individual registry entry. + +Set first checks local compliance, then calls the same `Client.Install` operation +as `dscpkg install` only when needed. **Latest-on-install is not automatic +updating**: an unversioned Set never upgrades a compliant installed resource, +even if repositories publish newer versions. Repeated Set works offline and +does not download again. Pin versions for reproducible installation. Older +package versions remain installed; no implicit cleanup runs. + +The underlying commands accept properties as one JSON object on stdin: + +```sh +echo '{"resource":"DscExtras.Linux/groups"}' | dscpkg resource get +echo '{"resource":"DscExtras.Linux/groups"}' | dscpkg resource test +echo '{"resource":"DscExtras.Linux/groups","repositories":["https://bpoe.github.io/dsc-extras/"]}' | dscpkg resource set +dscpkg resource export +``` + +Test returns two JSON Lines: the observed state including `_inDesiredState`, +then an array of differing property names. Export needs no input and emits one +`DscPkg/Resource` instance per installed resource type/exact-version pair, in +deterministic, deduplicated JSON Lines. It includes unregistered resources from +shared packages, never contacts repositories, and does not modify the store. +Only Export adds `_name`, using the existing resource type and exact version +directly, for example `"DscExtras.Linux/groups@0.1.1"`. DSC uses this as the +configuration instance name and removes `_name` from the generated properties. +Get, Test, and Set do not emit `_name`. +Use `dsc resource export --resource DscPkg/Resource` to obtain a DSC configuration. +Export filtering is disabled. + +Get/Test/Export parse JSON and YAML native manifests and the `resources` entries +of `.dsc.manifests.*` lists; a list file is not itself a resource instance. +Adapter-backed entries (`.dsc.adaptedresource.*` or `requireAdapter`) are +deliberately excluded with a diagnostic: an offline inventory cannot establish +whether their external adapter is available. Discovery still advertises those +files unchanged for DSC to handle. Invalid native manifest identities or corrupt +platform provenance fail explicitly; legacy packages without provenance are +skipped with reinstall guidance, as in discovery. + +Export cannot recover repository origins reliably and therefore omits them. +When reapplying exported configuration to a machine without those resources, +add `repositories` to the exported instances. Observed package/path fields are +not desired-state constraints. + +### dscd and persistent DSC server discovery + +The isolated DSC 3.3.0 integration test runs `00-resources.yaml` and then a +configuration using a newly installed resource **in the same `dsc server` +process**. Default `preDeployment` discovery retains stale resource lookup state +and the second configuration fails with `Resource not found`. Add the following +to the **consuming configuration** to refresh discovery: + +```yaml +directives: + resourceDiscovery: duringDeployment +``` + +The same-server test succeeds with this directive. Separate DSC CLI processes +already see newly installed packages without it. This change does not modify +dscd or its configuration ordering. It implements declarative local dependencies, +not Steve's proposed installation extension, remote discovery, or DSC's future +automatic resource-acquisition protocol. + ## Discover installed resources with DSC With DSC **3.3.0 or later**, dscpkg's `DscPkg/Discover` extension exposes locally installed packages without adding every package directory to `DSC_RESOURCE_PATH`. The native installers and release archives place `dscpkg.dsc.extension.json` -alongside the executable. DSC finds that manifest through PATH and runs +and `dscpkg.dsc.resource.json` alongside the executable. DSC finds the extension +manifest through PATH and runs `dscpkg discover`. Verify the extension and resources in a new terminal: ```sh @@ -61,7 +188,7 @@ The public example package can be installed on Linux with: ```sh dscpkg install --repository https://bpoe.github.io/dsc-extras/ \ - --resource DscExtras.Linux/groups --version 0.0.1 + --resource DscExtras.Linux/groups dsc resource list 'DscExtras.Linux/*' dsc resource get --resource DscExtras.Linux/groups --input '{"name":"root"}' dsc resource get --resource DscExtras.Linux/services \ @@ -111,7 +238,7 @@ move the affected version aside and reinstall to repair it. ### Installation scope and package-store selection -Machine-wide MSI/DEB/RPM installation registers the **CLI and extension**, not a +Machine-wide MSI/DEB/RPM installation registers the **CLI, extension, and resource**, not a machine-wide resource store. By default discovery uses the invoking identity's `/dscpkg/packages`: `%LOCALAPPDATA%\dscpkg\packages` on Windows, `${XDG_CACHE_HOME:-$HOME/.cache}/dscpkg/packages` on Linux, and @@ -119,12 +246,14 @@ Windows, `${XDG_CACHE_HOME:-$HOME/.cache}/dscpkg/packages` on Linux, and does not expose them to another account or service identity. For example, dscd running as LocalSystem or root does not use an interactive user's store. -For a custom or shared store, install packages with `--packages-dir` and configure -one copy of the extension manifest's `discover.args` as -`["discover", "--packages-dir", "/absolute/path/to/packages"]` (use a correctly -JSON-escaped Windows path on Windows). Place that copy in a DSC search directory -and avoid competing copies of `DscPkg/Discover`. Use an administrator-owned custom -location rather than modifying installer-owned files, which upgrades replace. +For a custom or shared store, set `DSCPKG_PACKAGES_DIR` in the environment of +both the CLI and the DSC/dscd process. This selects one consistent store for +installation, resource state, and extension discovery without editing manifests. +For example, an administrator can set it to `/var/lib/dscpkg/packages` for a +system service. This is an explicit scope override, not a new default or layout. +An interactive user and `sudo`/root may have different environments, so configure +the variable for the service identity as well. Do not modify installer-owned +manifests, which upgrades replace. Every invoking identity needs access to the store and executable; DSC executes package code with that identity's permissions. Share stores only with trusted writers. @@ -134,20 +263,26 @@ writers. DSC 3.3.0 searches PATH for manifests when `DSC_RESOURCE_PATH` is **unset**. If that variable is explicitly set, DSC searches those directories instead of PATH for the extension manifest. Keep any existing entries, but also include -the directory containing `dscpkg.dsc.extension.json` once (for example, +the directory containing both dscpkg manifests once (for example, `%ProgramFiles%\dscpkg` or `/usr/bin`). The `dscpkg` executable must remain on PATH. You do **not** need to add individual package directories. `DSC_RESTRICTED_PATH` and DSC resource-path policy/settings can similarly restrict discovery; the extension and its executable must be allowed by that configuration. -This is strictly **local discovery** of already installed packages. It is not -remote resource discovery or automatic installation. Resources must first be -installed explicitly with `dscpkg install`; no future remote-install protocol is -implemented. +This extension is strictly **local discovery** of already installed packages. +Install dependencies explicitly with `dscpkg install` or declare them with +`DscPkg/Resource`; no future remote-install protocol is implemented. ## Commands -Install a resource and optionally select an exact package version: +Install the latest compatible stable resource version: + +```sh +dscpkg install --repository https://resources.example.com \ + --resource Microsoft.GuestConfiguration/users +``` + +Or pin an exact resource version and optionally an independent exact package version: ```sh dscpkg install --repository https://resources.example.com \ @@ -327,12 +462,14 @@ CI also downloads DSC 3.3.0 into runner-local temporary directories and runs: ./tests/test-discovery.ps1 -DscPath /path/to/dsc ``` -This validates the extension against the official bundled schema exported by +This validates both manifests against the official bundled schemas exported by that engine and runs `go test -tags=integration -count=1 -v ./tests`. Tests build safe shared-executable resources, publish to a loopback HTTP server, install -through the CLI into an isolated per-user store, and exercise extension listing, -resource listing/invocation, platform exclusion, fresh-process discovery, and -explicit `DSC_RESOURCE_PATH` bootstrap. They never install DSC system-wide or +through the CLI and native DSC resource into isolated per-user stores, and +exercise all four resource operations, instance-schema validation of Export, +one-download shared-package installation, offline idempotence, platform exclusion, +fresh-process discovery, same-server `preDeployment`/`duringDeployment` behavior, +and explicit `DSC_RESOURCE_PATH` bootstrap. They never install DSC system-wide or change the developer's package store. To use an existing DSC 3.3.0 directly, set `DSC_TEST_BIN` and run the tagged Go tests. @@ -349,11 +486,12 @@ Releases include Linux AMD64/ARM64 and Windows AMD64 archives, a macOS archive f native architecture of the macOS CI runner, Windows MSI, Linux AMD64/ARM64 DEB/RPM, and `SHA256SUMS`. macOS filenames use Go's `darwin` platform name and the actual `amd64` or `arm64` build architecture. Extract an archive and put its `bin` -directory on PATH, keeping **both** the executable and -`dscpkg.dsc.extension.json` together. This also applies to manual ZIP installation: -copy both files from `bin` into a directory on PATH, open a new terminal, and -verify with `dsc extension list DscPkg/Discover`. For a source build, copy the -repository's `dscpkg.dsc.extension.json` alongside the built executable. If +directory on PATH, keeping **all three files** together: `dscpkg` (`dscpkg.exe` +on Windows), `dscpkg.dsc.extension.json`, and `dscpkg.dsc.resource.json`. +This also applies to manual ZIP installation: copy all three files from `bin` +into a directory on PATH, open a new terminal, and verify with +`dsc extension list DscPkg/Discover` and `dsc resource list DscPkg/Resource`. +For a source build, copy both repository manifests alongside the executable. If `DSC_RESOURCE_PATH` is set, follow the bootstrap instructions above. Alternatively, use the appropriate native installer: @@ -374,13 +512,13 @@ msiexec /i dscpkg-v1.2.3-windows-amd64.msi /qn /norestart The MSI installs to `%ProgramFiles%\dscpkg` and adds that directory to the system PATH; open a new terminal after installation. DEB/RPM install `/usr/bin/dscpkg`. -Both install the discovery extension manifest alongside the executable. +Both install the discovery extension and resource manifests alongside the executable. These are machine-wide CLI installations, not services. No DSC runtime is bundled or required to install dscpkg. Use Installed apps on Windows, `sudo apt remove dscpkg`, or `sudo dnf remove -dscpkg` to uninstall. Upgrades replace the executable and extension manifest; -uninstall removes both and the installer's managed PATH entry, but never deletes users' +dscpkg` to uninstall. Upgrades replace the executable and both manifests; +uninstall removes all three and the installer's managed PATH entry, but never deletes users' downloaded resource packages, registries, or caches. Linux follows the native package manager's version ordering; MSI blocks downgrades. diff --git a/cmd/dscpkg/main.go b/cmd/dscpkg/main.go index e9b9cb7..02df74f 100644 --- a/cmd/dscpkg/main.go +++ b/cmd/dscpkg/main.go @@ -5,6 +5,7 @@ import ( "errors" "flag" "fmt" + "io" "os" "path/filepath" "runtime" @@ -33,6 +34,9 @@ func run(args []string, stdout, stderr *os.File) error { return usageError() } command := args[0] + if command == "resource" { + return runResource(args[1:], os.Stdin, stdout, stderr) + } flags := flag.NewFlagSet(command, flag.ContinueOnError) flags.SetOutput(stderr) var packagesDir *string @@ -47,7 +51,7 @@ func run(args []string, stdout, stderr *os.File) error { flags.Var(&repositories, "repository", "repository origin, in precedence order (repeatable)") flags.StringVar(&platform, "platform", currentPlatform(), "target os_arch platform") flags.StringVar(&resource, "resource", "", "resource type (Namespace/name)") - flags.StringVar(&version, "version", "", "resource version") + flags.StringVar(&version, "version", "", "exact resource version (default: latest compatible stable)") flags.StringVar(&selectedPackageVersion, "package-version", "", "exact package version (optional)") case "update", "cleanup": packagesDir = flags.String("packages-dir", defaultPackagesDir(), "package cache directory") @@ -94,8 +98,8 @@ func run(args []string, stdout, stderr *os.File) error { client.Platform = platform switch command { case "install": - if resource == "" || version == "" { - return errors.New("install requires --resource and --version") + if resource == "" { + return errors.New("install requires --resource") } result, err := client.Install(resource, version, selectedPackageVersion) if err != nil { @@ -158,16 +162,94 @@ func printJSON(out *os.File, value any) error { } func usageError() error { - return errors.New("usage: dscpkg [flags]") + return errors.New("usage: dscpkg [flags]") } func defaultPackagesDir() string { + if dir := os.Getenv("DSCPKG_PACKAGES_DIR"); dir != "" { + return dir + } if dir, err := os.UserCacheDir(); err == nil { return filepath.Join(dir, "dscpkg", "packages") } + return filepath.Join(".", "packages") } +func runResource(args []string, stdin io.Reader, stdout, stderr io.Writer) error { + if len(args) == 0 { + return errors.New("usage: dscpkg resource [--packages-dir DIR]") + } + operation := args[0] + switch operation { + case "get", "test", "set", "export": + default: + return fmt.Errorf("unknown resource operation %q", operation) + } + flags := flag.NewFlagSet("resource "+operation, flag.ContinueOnError) + flags.SetOutput(stderr) + packagesDir := flags.String("packages-dir", defaultPackagesDir(), "package cache directory") + if err := flags.Parse(args[1:]); errors.Is(err, flag.ErrHelp) { + return nil + } else if err != nil { + return err + } + if flags.NArg() != 0 { + return errors.New("resource operations do not accept positional arguments") + } + client := dscpkg.NewClient(*packagesDir, nil) + encoder := json.NewEncoder(stdout) + if operation == "export" { + resources, err := client.InstalledResources(stderr) + if err != nil { + return err + } + for _, resource := range resources { + instance := struct { + Name string `json:"_name"` + dscpkg.ResourceState + }{ + Name: resource.Resource + "@" + resource.Version, ResourceState: resource, + } + if err := encoder.Encode(instance); err != nil { + return err + } + } + return nil + } + var desired dscpkg.ResourceState + decoder := json.NewDecoder(io.LimitReader(stdin, 16<<20+1)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(&desired); err != nil { + return fmt.Errorf("read resource properties: %w", err) + } + var extra any + if err := decoder.Decode(&extra); err != io.EOF { + return errors.New("expected exactly one resource properties object") + } + var actual dscpkg.ResourceState + var differences []string + var err error + switch operation { + case "get": + actual, err = client.GetResource(desired, stderr) + case "test": + actual, differences, err = client.TestResource(desired, stderr) + case "set": + actual, err = client.SetResource(desired, stderr) + } + if err != nil { + return err + } + if err := encoder.Encode(actual); err != nil { + return err + } + if operation == "test" { + return encoder.Encode(differences) + } + return nil +} + func currentPlatform() string { osName := runtime.GOOS if osName == "darwin" { diff --git a/cmd/dscpkg/main_test.go b/cmd/dscpkg/main_test.go index 9a44a9c..0a81b4d 100644 --- a/cmd/dscpkg/main_test.go +++ b/cmd/dscpkg/main_test.go @@ -183,3 +183,107 @@ func TestPublishHelpListsPublishingFlags(t *testing.T) { } } } + +func TestResourceCommands(t *testing.T) { + t.Setenv("DSCPKG_PACKAGES_DIR", filepath.Join(t.TempDir(), "absent")) + for _, test := range []struct { + operation, input, output, errorText string + }{ + {"get", `{"resource":"Example/One"}`, `{"resource":"Example/One","state":"missing"}` + "\n", ""}, + {"test", `{"resource":"Example/One"}`, `{"resource":"Example/One","state":"missing","_inDesiredState":false}` + "\n" + `["state"]` + "\n", ""}, + {"test", `{"resource":"Example/One","state":"missing"}`, `{"resource":"Example/One","state":"missing","_inDesiredState":true}` + "\n[]\n", ""}, + {"export", "", "", ""}, + {"export", "not read", "", ""}, + {"get", "{}", "", "invalid resource"}, + {"get", "null", "", "invalid resource"}, + {"get", `{"resource":"Example/One","unknown":true}`, "", "unknown field"}, + {"test", `{"resource":"Example/One","state":"absent"}`, "", "invalid resource state"}, + {"get", `{"resource":"Example/One"} {}`, "", "exactly one"}, + {"set", `{"resource":"Example/One"}`, "", "repository"}, + {"set", `{"resource":"Example/One","state":"missing"}`, "", "removal is not supported"}, + } { + t.Run(test.operation+test.input, func(t *testing.T) { + var out, diagnostics bytes.Buffer + err := runResource([]string{test.operation}, strings.NewReader(test.input), &out, &diagnostics) + if test.errorText != "" { + if err == nil || !strings.Contains(err.Error(), test.errorText) { + t.Fatalf("err=%v want=%q", err, test.errorText) + } + } else if err != nil { + t.Fatal(err) + } + if out.String() != test.output || diagnostics.Len() != 0 { + t.Fatalf("stdout=%q stderr=%q", out.String(), diagnostics.String()) + } + }) + } +} + +func TestResourceExportNames(t *testing.T) { + store := t.TempDir() + t.Setenv("DSCPKG_PACKAGES_DIR", store) + for _, version := range []string{"0.1.1", "0.2.0"} { + directory := filepath.Join(store, "dscextras", "linux", version) + if err := os.MkdirAll(directory, 0o755); err != nil { + t.Fatal(err) + } + metadata := `{"schemaVersion":1,"platform":"` + currentPlatform() + `"}` + if err := os.WriteFile(filepath.Join(filepath.Dir(directory), "."+version+".dscpkg.json"), []byte(metadata), 0o644); err != nil { + t.Fatal(err) + } + manifests := `{"resources":[{"type":"DscExtras.Linux/groups","version":"` + version + + `"},{"type":"DscExtras.Linux/services","version":"` + version + `"}]}` + if err := os.WriteFile(filepath.Join(directory, "linux.dsc.manifests.json"), []byte(manifests), 0o644); err != nil { + t.Fatal(err) + } + } + var stdout, stderr bytes.Buffer + if err := runResource([]string{"export"}, strings.NewReader(""), &stdout, &stderr); err != nil { + t.Fatal(err) + } + lines := bytes.Split(bytes.TrimSpace(stdout.Bytes()), []byte("\n")) + if len(lines) != 4 { + t.Fatalf("expected one instance per resource/version pair: %s", &stdout) + } + names := make(map[string]bool) + for _, line := range lines { + var instance struct { + Name string `json:"_name"` + Resource string `json:"resource"` + Version string `json:"version"` + } + if err := json.Unmarshal(line, &instance); err != nil { + t.Fatal(err) + } + want := instance.Resource + "@" + instance.Version + if instance.Name != want || names[instance.Name] { + t.Fatalf("expected unique unencoded resource@version name %q: %s", want, line) + } + names[instance.Name] = true + } + for _, operation := range []string{"get", "test", "set"} { + stdout.Reset() + if err := runResource([]string{operation}, strings.NewReader(`{"resource":"DscExtras.Linux/groups"}`), &stdout, &stderr); err != nil { + t.Fatal(err) + } + if bytes.Contains(stdout.Bytes(), []byte(`"_name"`)) { + t.Fatalf("%s emitted Export-only _name: %s", operation, &stdout) + } + } + if stderr.Len() != 0 { + t.Fatalf("unexpected diagnostics: %s", &stderr) + } +} + +func TestPackageStoreOverride(t *testing.T) { + directory := filepath.Join(t.TempDir(), "shared store") + t.Setenv("DSCPKG_PACKAGES_DIR", directory) + if got := defaultPackagesDir(); got != directory { + t.Fatalf("default store=%q want=%q", got, directory) + } + var stdout, stderr bytes.Buffer + if err := runResource([]string{"get", "--packages-dir", filepath.Join(t.TempDir(), "explicit")}, + strings.NewReader(`{"resource":"Example/One"}`), &stdout, &stderr); err != nil { + t.Fatal(err) + } +} diff --git a/dscpkg.dsc.resource.json b/dscpkg.dsc.resource.json new file mode 100644 index 0000000..47669ed --- /dev/null +++ b/dscpkg.dsc.resource.json @@ -0,0 +1,70 @@ +{ + "$schema": "https://aka.ms/dsc/schemas/v3/bundled/resource/manifest.json", + "type": "DscPkg/Resource", + "version": "0.1.0", + "description": "Ensure a DSC resource is installed in the local dscpkg package store.", + "get": { + "executable": "dscpkg", + "args": ["resource", "get"], + "input": "stdin" + }, + "test": { + "executable": "dscpkg", + "args": ["resource", "test"], + "input": "stdin", + "return": "stateAndDiff" + }, + "set": { + "executable": "dscpkg", + "args": ["resource", "set"], + "input": "stdin", + "implementsPretest": true, + "return": "state" + }, + "export": { + "executable": "dscpkg", + "args": ["resource", "export"], + "input": "stdin", + "supportsFiltering": false + }, + "schema": { + "embedded": { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["resource"], + "additionalProperties": false, + "properties": { + "resource": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*/[A-Za-z0-9][A-Za-z0-9_.-]*$", + "description": "DSC resource identity (Namespace/name)." + }, + "version": { + "type": "string", + "pattern": "^[A-Za-z0-9][A-Za-z0-9_.-]*$", + "description": "Exact resource version; omit to accept any installed version." + }, + "state": { + "type": "string", + "enum": ["installed", "missing"], + "default": "installed", + "description": "Only installed is supported for Set; missing is an observed state." + }, + "repositories": { + "type": "array", + "items": {"type": "string", "minLength": 1}, + "description": "Repository origins in precedence order, needed only when installation is necessary." + }, + "package": {"type": "string", "readOnly": true}, + "packageVersion": {"type": "string", "readOnly": true}, + "path": {"type": "string", "readOnly": true}, + "_name": { + "type": "string", + "readOnly": true, + "description": "Export-only resource@version name, consumed by DSC as the configuration instance name." + }, + "_inDesiredState": {"type": "boolean", "readOnly": true} + } + } + } +} diff --git a/examples/00-resources.yaml b/examples/00-resources.yaml new file mode 100644 index 0000000..713f036 --- /dev/null +++ b/examples/00-resources.yaml @@ -0,0 +1,18 @@ +$schema: https://aka.ms/dsc/schemas/v3/bundled/config/document.json + +resources: + - name: LinuxGroups + type: DscPkg/Resource + properties: + resource: DscExtras.Linux/groups + state: installed + repositories: + - https://bpoe.github.io/dsc-extras/ + + - name: LinuxServices + type: DscPkg/Resource + properties: + resource: DscExtras.Linux/services + state: installed + repositories: + - https://bpoe.github.io/dsc-extras/ diff --git a/go.mod b/go.mod index ba393ef..323eeed 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/Bpoe/dscpkg go 1.23 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a62c313 --- /dev/null +++ b/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/dscpkg/archive.go b/internal/dscpkg/archive.go index 7cbdf6d..5ef28a6 100644 --- a/internal/dscpkg/archive.go +++ b/internal/dscpkg/archive.go @@ -37,40 +37,62 @@ func (c *Client) installPackage(repo *repository, resolved packageResolution) (b } else if err != nil && !errors.Is(err, os.ErrNotExist) { return false, err } - infoURL, err := packageDocumentURL(repo.discovery.Packages, resolved.Package, resolved.PackageVersion) + archive, archiveURL := resolved.archive, resolved.archiveURL + if archive == nil { + selected, uri, err := c.packageArchive(repo, resolved.Package, resolved.PackageVersion) + if errors.Is(err, errNoCompatiblePackage) { + return false, fmt.Errorf("package %s %s has no archive for platform %s", resolved.Package, resolved.PackageVersion, c.Platform) + } + if err != nil { + return false, err + } + archive, archiveURL = &selected, uri + } + return c.downloadPackage(resolved, *archive, archiveURL) +} + +func (c *Client) packageArchive(repo *repository, packageName, version string) (Archive, string, error) { + infoURL, err := packageDocumentURL(repo.discovery.Packages, packageName, version) if err != nil { - return false, err + return Archive{}, "", err } info, err := c.getDocument(infoURL) if err != nil { - return false, err + return Archive{}, "", err } - if err := c.verifyDocument(repo, namespace(resolved.Package), infoURL, info); err != nil { - return false, fmt.Errorf("package descriptor %s signature check: %w", infoURL, err) + if err := c.verifyDocument(repo, namespace(packageName), infoURL, info); err != nil { + return Archive{}, "", fmt.Errorf("package descriptor %s signature check: %w", infoURL, err) } var descriptor PackageDescriptor if err := json.Unmarshal(info.body, &descriptor); err != nil { - return false, fmt.Errorf("parse package descriptor %s: %w", infoURL, err) + return Archive{}, "", fmt.Errorf("parse package descriptor %s: %w", infoURL, err) + } + if descriptor.Archives == nil { + return Archive{}, "", fmt.Errorf("package descriptor %s has no archives object", infoURL) } archive, exists := descriptor.Archives[c.Platform] if !exists { - return false, fmt.Errorf("package %s %s has no archive for platform %s", resolved.Package, resolved.PackageVersion, c.Platform) + return Archive{}, "", errNoCompatiblePackage } if strings.TrimSpace(archive.URL) == "" { - return false, errors.New("package archive descriptor has no URL") + return Archive{}, "", errors.New("package archive descriptor has no URL") } if len(archive.Hashes) == 0 { - return false, errors.New("package archive descriptor has no hashes") + return Archive{}, "", errors.New("package archive descriptor has no hashes") } for _, entry := range archive.Hashes { if _, _, err := splitArchiveHash(entry); err != nil { - return false, err + return Archive{}, "", err } } archiveURL, err := resolveURL(infoURL, archive.URL) if err != nil { - return false, err + return Archive{}, "", err } + return archive, archiveURL, nil +} + +func (c *Client) downloadPackage(resolved packageResolution, archive Archive, archiveURL string) (bool, error) { parent := filepath.Dir(resolved.Path) if err := os.MkdirAll(parent, 0o755); err != nil { return false, err diff --git a/internal/dscpkg/client.go b/internal/dscpkg/client.go index c9af771..bbecb5a 100644 --- a/internal/dscpkg/client.go +++ b/internal/dscpkg/client.go @@ -265,7 +265,7 @@ func (c *Client) resolve(resource, version, requestedPackageVersion string) (pac return packageResolution{}, nil, err } version = strings.ToLower(strings.TrimSpace(version)) - if !validSegment(version) { + if version != "" && !validSegment(version) { return packageResolution{}, nil, fmt.Errorf("invalid resource version %q", version) } if len(c.Repositories) == 0 { @@ -283,77 +283,123 @@ func (c *Client) resolve(resource, version, requestedPackageVersion string) (pac if err != nil { return packageResolution{}, nil, err } - entry, found := catalog.Resources[resource][version] - if !found { - continue - } - if strings.TrimSpace(entry.URL) == "" { - return packageResolution{}, nil, fmt.Errorf("catalog entry for %s@%s has no descriptor URL", resource, version) - } - if _, _, err := parseDigest(entry.Digest); err != nil { - return packageResolution{}, nil, fmt.Errorf("catalog entry for %s@%s has invalid digest: %w", resource, version, err) + versions := []string{version} + if version == "" { + versions = nil + for candidate := range catalog.Resources[resource] { + if isStableVersion(candidate) { + versions = append(versions, candidate) + } + } + sort.Slice(versions, func(i, j int) bool { return orderVersions(versions[i], versions[j]) > 0 }) } - descriptorURL, err := resolveURL(repo.catalogURL, entry.URL) - if err != nil { - return packageResolution{}, nil, err + for _, candidate := range versions { + entry, found := catalog.Resources[resource][candidate] + if !found { + continue + } + resolved, err := c.resolveEntry(repo, resource, candidate, requestedPackageVersion, entry, version == "") + if errors.Is(err, errNoCompatiblePackage) && version == "" { + continue + } + if err != nil { + return packageResolution{}, nil, err + } + return resolved, repo, nil } - response, err := c.getDocument(descriptorURL) + } + return packageResolution{}, nil, fmt.Errorf("resource %s@%s was not found with a compatible package in any configured repository", resource, version) +} + +var errNoCompatiblePackage = errors.New("no compatible package version") + +func (c *Client) resolveEntry(repo *repository, resource, version, requestedPackageVersion string, entry CatalogEntry, requirePlatform bool) (packageResolution, error) { + if strings.TrimSpace(entry.URL) == "" { + return packageResolution{}, fmt.Errorf("catalog entry for %s@%s has no descriptor URL", resource, version) + } + if _, _, err := parseDigest(entry.Digest); err != nil { + return packageResolution{}, fmt.Errorf("catalog entry for %s@%s has invalid digest: %w", resource, version, err) + } + descriptorURL, err := resolveURL(repo.catalogURL, entry.URL) + if err != nil { + return packageResolution{}, err + } + response, err := c.getDocument(descriptorURL) + if err != nil { + return packageResolution{}, err + } + if err := verifyDigest(response.body, entry.Digest); err != nil { + return packageResolution{}, fmt.Errorf("resource descriptor %s integrity check: %w", descriptorURL, err) + } + if err := c.verifyDocument(repo, namespace(resource), descriptorURL, response); err != nil { + return packageResolution{}, fmt.Errorf("resource descriptor %s signature check: %w", descriptorURL, err) + } + var descriptor ResourceDescriptor + if err := json.Unmarshal(response.body, &descriptor); err != nil { + return packageResolution{}, fmt.Errorf("parse resource descriptor %s: %w", descriptorURL, err) + } + if len(descriptor.Packages) != 1 { + return packageResolution{}, fmt.Errorf("resource descriptor for %s@%s must identify exactly one package; got %d", resource, version, len(descriptor.Packages)) + } + var packageName string + var packageInfo struct { + Versions map[string]json.RawMessage `json:"versions"` + } + for name, info := range descriptor.Packages { + packageName, err = normalizePackage(name) if err != nil { - return packageResolution{}, nil, err - } - if err := verifyDigest(response.body, entry.Digest); err != nil { - return packageResolution{}, nil, fmt.Errorf("resource descriptor %s integrity check: %w", descriptorURL, err) + return packageResolution{}, err } - if err := c.verifyDocument(repo, namespace(resource), descriptorURL, response); err != nil { - return packageResolution{}, nil, fmt.Errorf("resource descriptor %s signature check: %w", descriptorURL, err) - } - var descriptor ResourceDescriptor - if err := json.Unmarshal(response.body, &descriptor); err != nil { - return packageResolution{}, nil, fmt.Errorf("parse resource descriptor %s: %w", descriptorURL, err) - } - if len(descriptor.Packages) != 1 { - return packageResolution{}, nil, fmt.Errorf("resource descriptor for %s@%s must identify exactly one package; got %d", resource, version, len(descriptor.Packages)) - } - var packageName string - var packageInfo struct { - Versions map[string]json.RawMessage `json:"versions"` + packageInfo = info + } + versions := make([]string, 0, len(packageInfo.Versions)) + for candidate := range packageInfo.Versions { + if validSegment(candidate) { + versions = append(versions, strings.ToLower(candidate)) } - for name, info := range descriptor.Packages { - packageName, err = normalizePackage(name) - if err != nil { - return packageResolution{}, nil, err + } + if len(versions) == 0 { + return packageResolution{}, fmt.Errorf("package %s has no valid published versions", packageName) + } + sort.Slice(versions, func(i, j int) bool { return orderVersions(versions[i], versions[j]) < 0 }) + selected := versions[len(versions)-1] + if requestedPackageVersion != "" { + selected = strings.ToLower(requestedPackageVersion) + found := false + for _, candidate := range versions { + if candidate == selected { + found = true } - packageInfo = info } - versions := make([]string, 0, len(packageInfo.Versions)) - for candidate := range packageInfo.Versions { - if validSegment(candidate) { - versions = append(versions, strings.ToLower(candidate)) + if !found { + if requirePlatform { + return packageResolution{}, errNoCompatiblePackage } + return packageResolution{}, fmt.Errorf("package version %q is not available for %s", requestedPackageVersion, packageName) } - if len(versions) == 0 { - return packageResolution{}, nil, fmt.Errorf("package %s has no valid published versions", packageName) - } - sort.Slice(versions, func(i, j int) bool { return compareVersions(versions[i], versions[j]) < 0 }) - selected := versions[len(versions)-1] - if requestedPackageVersion != "" { - selected = strings.ToLower(requestedPackageVersion) - found := false - for _, candidate := range versions { - if candidate == selected { - found = true - } + versions = []string{selected} + } + resolved := packageResolution{ + Resource: resource, Version: version, Package: packageName, PackageVersion: selected, + Digest: entry.Digest, Path: filepath.Join(c.PackagesDir, filepath.FromSlash(packageName), selected), + } + if requirePlatform { + for i := len(versions) - 1; i >= 0; i-- { + archive, archiveURL, err := c.packageArchive(repo, packageName, versions[i]) + if errors.Is(err, errNoCompatiblePackage) { + continue } - if !found { - return packageResolution{}, nil, fmt.Errorf("package version %q is not available for %s", requestedPackageVersion, packageName) + if err != nil { + return packageResolution{}, err } + resolved.PackageVersion = versions[i] + resolved.Path = filepath.Join(c.PackagesDir, filepath.FromSlash(packageName), versions[i]) + resolved.archive, resolved.archiveURL = &archive, archiveURL + return resolved, nil } - return packageResolution{ - Resource: resource, Version: version, Package: packageName, PackageVersion: selected, - Digest: entry.Digest, Path: filepath.Join(c.PackagesDir, filepath.FromSlash(packageName), selected), - }, repo, nil + return packageResolution{}, errNoCompatiblePackage } - return packageResolution{}, nil, fmt.Errorf("resource %s@%s was not found in any configured repository", resource, version) + return resolved, nil } func (c *Client) discover(origin string) (*repository, error) { @@ -627,31 +673,96 @@ func parseDigest(value string) (string, []byte, error) { } func compareVersions(a, b string) int { - prefixA, _, _ := strings.Cut(a, "-") - prefixB, _, _ := strings.Cut(b, "-") + prefixA, preA, _ := strings.Cut(a, "-") + prefixB, preB, _ := strings.Cut(b, "-") partsA, partsB := strings.Split(prefixA, "."), strings.Split(prefixB, ".") count := len(partsA) if len(partsB) > count { count = len(partsB) } for i := 0; i < count; i++ { - av, bv := uint64(0), uint64(0) + av, bv := "0", "0" if i < len(partsA) { - fmt.Sscan(partsA[i], &av) + av = partsA[i] } if i < len(partsB) { - fmt.Sscan(partsB[i], &bv) + bv = partsB[i] + } + if compared := compareVersionPart(av, bv); compared != 0 { + return compared } - if av < bv { + } + if preA == "" && preB != "" { + return 1 + } + if preA != "" && preB == "" { + return -1 + } + aParts, bParts := strings.Split(preA, "."), strings.Split(preB, ".") + for i := 0; i < len(aParts) && i < len(bParts); i++ { + if compared := compareVersionPart(aParts[i], bParts[i]); compared != 0 { + return compared + } + } + if len(aParts) < len(bParts) { + return -1 + } + if len(aParts) > len(bParts) { + return 1 + } + return 0 +} + +func orderVersions(a, b string) int { + if compared := compareVersions(a, b); compared != 0 { + return compared + } + return strings.Compare(a, b) +} + +func numericVersionPart(value string) bool { + if value == "" { + return false + } + for _, r := range value { + if r < '0' || r > '9' { + return false + } + } + return true +} + +func compareVersionPart(a, b string) int { + an, bn := numericVersionPart(a), numericVersionPart(b) + if an && bn { + a, b = strings.TrimLeft(a, "0"), strings.TrimLeft(b, "0") + if len(a) < len(b) { return -1 } - if av > bv { + if len(a) > len(b) { return 1 } + } else if an != bn { + if an { + return -1 + } + return 1 } return strings.Compare(a, b) } +func isStableVersion(value string) bool { + if !validSegment(value) { + return false + } + for _, part := range strings.Split(value, ".") { + if !numericVersionPart(part) { + return false + } + } + return true +} + func packageDocumentURL(endpoint, packageName, version string) (string, error) { name, err := normalizePackage(packageName) if err != nil { diff --git a/internal/dscpkg/discover.go b/internal/dscpkg/discover.go index 3e97395..dfd2101 100644 --- a/internal/dscpkg/discover.go +++ b/internal/dscpkg/discover.go @@ -16,6 +16,13 @@ type discoveredManifest struct { ManifestPath string `json:"manifestPath"` } +type InstalledManifest struct { + ManifestPath string + Package string + PackageVersion string + Path string +} + // Discover emits DSC 3.3 discovery JSON Lines using only the local package store. // It does not acquire the store's write lock, create files, or consult repositories. func Discover(packagesDir string, stdout, stderr io.Writer) error { @@ -23,25 +30,41 @@ func Discover(packagesDir string, stdout, stderr io.Writer) error { } func discover(packagesDir, platform string, stdout, stderr io.Writer) error { - root, err := filepath.Abs(packagesDir) + manifests, err := EnumerateInstalledManifests(packagesDir, platform, stderr) if err != nil { return err } + encoder := json.NewEncoder(stdout) + for _, manifest := range manifests { + if err := encoder.Encode(discoveredManifest{ManifestPath: manifest.ManifestPath}); err != nil { + return fmt.Errorf("write discovered manifest: %w", err) + } + } + return nil +} + +// EnumerateInstalledManifests is the read-only platform gate for discovery and inventory. +func EnumerateInstalledManifests(packagesDir, platform string, stderr io.Writer) ([]InstalledManifest, error) { + root, err := filepath.Abs(packagesDir) + if err != nil { + return nil, err + } directories, err := packageDirectories(root) if err != nil { - return fmt.Errorf("enumerate installed packages: %w", err) + return nil, fmt.Errorf("enumerate installed packages: %w", err) } - var manifests []string + var manifests []InstalledManifest + seen := make(map[string]bool) for _, directory := range directories { installed, err := readInstalledPackage(directory) if errors.Is(err, os.ErrNotExist) { if _, err := fmt.Fprintf(stderr, "dscpkg: skipping %q: platform provenance is missing; move this version directory aside and reinstall its resources with dscpkg to verify the platform\n", directory); err != nil { - return err + return nil, err } continue } if err != nil { - return fmt.Errorf("verify package %q (move it aside and reinstall to repair): %w", directory, err) + return nil, fmt.Errorf("verify package %q (move it aside and reinstall to repair): %w", directory, err) } if installed.Platform != platform { continue @@ -51,21 +74,23 @@ func discover(packagesDir, platform string, stdout, stderr io.Writer) error { return err } if entry.Type().IsRegular() && isResourceManifest(entry.Name()) { - manifests = append(manifests, path) + key := canonicalPath(path) + if !seen[key] { + seen[key] = true + pkg := filepath.Dir(directory) + manifests = append(manifests, InstalledManifest{ + ManifestPath: path, Path: directory, PackageVersion: filepath.Base(directory), + Package: filepath.Base(filepath.Dir(pkg)) + "/" + filepath.Base(pkg), + }) + } } return nil }); err != nil { - return fmt.Errorf("enumerate manifests in %q: %w", directory, err) + return nil, fmt.Errorf("enumerate manifests in %q: %w", directory, err) } } - sort.Strings(manifests) - encoder := json.NewEncoder(stdout) - for _, path := range uniquePaths(manifests) { - if err := encoder.Encode(discoveredManifest{ManifestPath: path}); err != nil { - return fmt.Errorf("write discovered manifest: %w", err) - } - } - return nil + sort.Slice(manifests, func(i, j int) bool { return manifests[i].ManifestPath < manifests[j].ManifestPath }) + return manifests, nil } func isResourceManifest(name string) bool { diff --git a/internal/dscpkg/model.go b/internal/dscpkg/model.go index 4b757f5..efe4b8e 100644 --- a/internal/dscpkg/model.go +++ b/internal/dscpkg/model.go @@ -79,6 +79,8 @@ type packageResolution struct { PackageVersion string `json:"packageVersion"` Digest string `json:"digest"` Path string `json:"path"` + archive *Archive + archiveURL string } func normalizeResource(resource string) (string, error) { diff --git a/internal/dscpkg/resource.go b/internal/dscpkg/resource.go new file mode 100644 index 0000000..535e097 --- /dev/null +++ b/internal/dscpkg/resource.go @@ -0,0 +1,205 @@ +package dscpkg + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "sort" + "strings" + + "gopkg.in/yaml.v3" +) + +type ResourceState struct { + Resource string `json:"resource"` + Version string `json:"version,omitempty"` + State string `json:"state"` + Repositories []string `json:"repositories,omitempty"` + Package string `json:"package,omitempty"` + PackageVersion string `json:"packageVersion,omitempty"` + Path string `json:"path,omitempty"` + InDesiredState *bool `json:"_inDesiredState,omitempty"` +} + +type manifestIdentity struct { + Type string `json:"type" yaml:"type"` + Version string `json:"version" yaml:"version"` + RequireAdapter string `json:"requireAdapter" yaml:"requireAdapter"` + Resources []manifestIdentity `json:"resources" yaml:"resources"` +} + +func validateResourceState(desired ResourceState) error { + if _, err := normalizeResource(desired.Resource); err != nil { + return err + } + if desired.Version != "" && !validSegment(strings.ToLower(desired.Version)) { + return fmt.Errorf("invalid resource version %q", desired.Version) + } + if desired.State != "" && desired.State != "installed" && desired.State != "missing" { + return fmt.Errorf("invalid resource state %q: expected installed or missing", desired.State) + } + return nil +} + +// InstalledResources reads the same compatible manifests advertised by discovery. +// Adapter-backed entries need adapter discovery, so are deliberately not asserted here. +func (c *Client) InstalledResources(stderr io.Writer) ([]ResourceState, error) { + manifests, err := EnumerateInstalledManifests(c.PackagesDir, c.Platform, stderr) + if err != nil { + return nil, err + } + result := make([]ResourceState, 0) + for _, manifest := range manifests { + file, err := os.Open(manifest.ManifestPath) + if err != nil { + return nil, err + } + data, readErr := io.ReadAll(io.LimitReader(file, maxDocumentSize+1)) + closeErr := file.Close() + if readErr != nil { + return nil, readErr + } + if closeErr != nil { + return nil, closeErr + } + if len(data) > maxDocumentSize { + return nil, fmt.Errorf("manifest %q exceeds size limit", manifest.ManifestPath) + } + var identity manifestIdentity + if strings.HasSuffix(strings.ToLower(manifest.ManifestPath), ".json") { + err = json.Unmarshal(data, &identity) + } else { + err = yaml.Unmarshal(data, &identity) + } + if err != nil { + return nil, fmt.Errorf("parse manifest %q: %w", manifest.ManifestPath, err) + } + identities := []manifestIdentity{identity} + name := strings.ToLower(manifest.ManifestPath) + if strings.Contains(name, ".dsc.manifests.") { + if identity.Resources == nil { + return nil, fmt.Errorf("manifest list %q has no resources array", manifest.ManifestPath) + } + identities = identity.Resources + } + for _, identity := range identities { + if identity.RequireAdapter != "" || strings.Contains(name, ".dsc.adaptedresource.") { + if _, err := fmt.Fprintf(stderr, "dscpkg: skipping adapter-backed resource in %q: offline inventory cannot verify adapter availability\n", manifest.ManifestPath); err != nil { + return nil, err + } + continue + } + if _, err := normalizeResource(identity.Type); err != nil { + return nil, fmt.Errorf("manifest %q: %w", manifest.ManifestPath, err) + } + if !validSegment(strings.ToLower(identity.Version)) { + return nil, fmt.Errorf("manifest %q has invalid resource version %q", manifest.ManifestPath, identity.Version) + } + result = append(result, ResourceState{ + Resource: identity.Type, Version: identity.Version, State: "installed", + Package: manifest.Package, PackageVersion: manifest.PackageVersion, Path: manifest.Path, + }) + } + } + sort.Slice(result, func(i, j int) bool { + a, b := result[i], result[j] + if strings.ToLower(a.Resource) != strings.ToLower(b.Resource) { + return strings.ToLower(a.Resource) < strings.ToLower(b.Resource) + } + if compared := compareVersions(a.Version, b.Version); compared != 0 { + return compared > 0 + } + if a.Version != b.Version { + return a.Version < b.Version + } + if compared := compareVersions(a.PackageVersion, b.PackageVersion); compared != 0 { + return compared > 0 + } + return a.Path < b.Path + }) + seen := make(map[string]bool) + unique := make([]ResourceState, 0, len(result)) + for _, state := range result { + key := strings.ToLower(state.Resource + "@" + state.Version) + if !seen[key] { + seen[key] = true + unique = append(unique, state) + } + } + return unique, nil +} + +func (c *Client) GetResource(desired ResourceState, stderr io.Writer) (ResourceState, error) { + if err := validateResourceState(desired); err != nil { + return ResourceState{}, err + } + resource, err := normalizeResource(desired.Resource) + if err != nil { + return ResourceState{}, err + } + installed, err := c.InstalledResources(stderr) + if err != nil { + return ResourceState{}, err + } + for _, state := range installed { + if strings.EqualFold(state.Resource, resource) && + (desired.Version == "" || strings.EqualFold(state.Version, desired.Version)) { + state.Repositories = desired.Repositories + return state, nil + } + } + return ResourceState{Resource: desired.Resource, State: "missing", Repositories: desired.Repositories}, nil +} + +func (c *Client) TestResource(desired ResourceState, stderr io.Writer) (ResourceState, []string, error) { + actual, err := c.GetResource(desired, stderr) + if err != nil { + return ResourceState{}, nil, err + } + state := desired.State + if state == "" { + state = "installed" + } + differences := make([]string, 0) + if actual.State != state { + differences = append(differences, "state") + } + if desired.Version != "" && !strings.EqualFold(desired.Version, actual.Version) { + differences = append(differences, "version") + } + compliant := len(differences) == 0 + actual.InDesiredState = &compliant + return actual, differences, nil +} + +func (c *Client) SetResource(desired ResourceState, stderr io.Writer) (ResourceState, error) { + if desired.State == "missing" { + return ResourceState{}, errors.New("state missing is observation-only; resource removal is not supported") + } + actual, differences, err := c.TestResource(desired, stderr) + if err != nil { + return ResourceState{}, err + } + actual.InDesiredState = nil + if len(differences) == 0 { + return actual, nil + } + installer := *c + if len(desired.Repositories) > 0 { + installer.Repositories = desired.Repositories + } + if _, err := installer.Install(desired.Resource, desired.Version, ""); err != nil { + return ResourceState{}, err + } + actual, differences, err = c.TestResource(desired, stderr) + if err != nil { + return ResourceState{}, err + } + if len(differences) != 0 { + return ResourceState{}, fmt.Errorf("installed package does not expose the requested discoverable resource %s@%s; differing properties: %s", desired.Resource, desired.Version, strings.Join(differences, ", ")) + } + actual.InDesiredState = nil + return actual, nil +} diff --git a/internal/dscpkg/resource_test.go b/internal/dscpkg/resource_test.go new file mode 100644 index 0000000..bf22632 --- /dev/null +++ b/internal/dscpkg/resource_test.go @@ -0,0 +1,345 @@ +package dscpkg + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "reflect" + "strings" + "sync/atomic" + "testing" +) + +func TestCompareVersions(t *testing.T) { + for _, pair := range [][2]string{ + {"1.9.0", "1.10.0"}, {"1.0.0-rc.9", "1.0.0-rc.10"}, + {"1.0.0-rc.10", "1.0.0"}, {"1.0.0-alpha", "1.0.0-alpha.1"}, + {"1.0.0-1", "1.0.0-alpha"}, {"9", "10"}, + {"99999999999999999999", "100000000000000000000"}, + } { + if compareVersions(pair[0], pair[1]) >= 0 || compareVersions(pair[1], pair[0]) <= 0 { + t.Fatalf("incorrect precedence: %q < %q", pair[0], pair[1]) + } + } + for _, version := range []string{"1", "1.2", "1.10.0"} { + if !isStableVersion(version) || compareVersions(version, version) != 0 { + t.Fatalf("invalid stable version handling: %q", version) + } + } + for _, version := range []string{"1.0.0-preview", "latest", "1..0", "../1"} { + if isStableVersion(version) { + t.Fatalf("non-stable version accepted: %q", version) + } + } +} + +func publishResourceFixture(t *testing.T, repository, pkg, packageVersion, resourceVersion, platform string, resources ...string) { + t.Helper() + identities := make([]manifestIdentity, 0, len(resources)) + refs := make([]string, 0, len(resources)) + for _, resource := range resources { + identities = append(identities, manifestIdentity{Type: resource, Version: resourceVersion}) + refs = append(refs, resource+"@"+resourceVersion) + } + data, err := json.Marshal(manifestIdentity{Resources: identities}) + if err != nil { + t.Fatal(err) + } + archive := filepath.Join(t.TempDir(), "package.zip") + if err := os.WriteFile(archive, makeZip(t, "shared.dsc.manifests.json", data), 0o644); err != nil { + t.Fatal(err) + } + if err := Publish(PublishOptions{ + Repository: repository, Package: pkg, PackageVersion: packageVersion, + Platform: platform, Archive: archive, Resources: refs, + }); err != nil { + t.Fatal(err) + } +} + +func TestLatestCompatibleResolution(t *testing.T) { + t.Setenv("DSC_RESOURCE_PATH", "") + primary, secondary := t.TempDir(), t.TempDir() + for _, version := range []string{"1.9.0", "1.10.0", "2.0.0-preview", "3.0.0"} { + platform := "linux_amd64" + if version == "3.0.0" { + platform = "windows_amd64" + } + publishResourceFixture(t, primary, testPackage, version, version, platform, testResourceOne) + } + // The newest package can be incompatible independently of the resource version. + publishResourceFixture(t, primary, testPackage, "8.0.0", "1.10.0", "windows_amd64", testResourceOne) + publishResourceFixture(t, secondary, testPackage, "9.0.0", "9.0.0", "linux_amd64", testResourceOne) + publishResourceFixture(t, secondary, testPackage, "9.0.0", "9.0.0", "darwin_arm64", testResourceOne) + first := httptest.NewServer(http.FileServer(http.Dir(primary))) + defer first.Close() + second := httptest.NewServer(http.FileServer(http.Dir(secondary))) + defer second.Close() + for _, test := range []struct{ version, pin, resourceVersion, packageVersion string }{ + {"", "", "1.10.0", "1.10.0"}, + {"", "1.9.0", "1.9.0", "1.9.0"}, + {"1.9.0", "", "1.9.0", "1.9.0"}, + {"2.0.0-preview", "", "2.0.0-preview", "2.0.0-preview"}, + } { + t.Run(test.version+"/"+test.pin, func(t *testing.T) { + client := NewClient(t.TempDir(), []string{first.URL, second.URL}) + client.Platform = "linux_amd64" + result, err := client.Install(testResourceOne, test.version, test.pin) + if err != nil { + t.Fatal(err) + } + if result.Version != test.resourceVersion || result.PackageVersion != test.packageVersion { + t.Fatalf("wrong resolved versions: %+v", result) + } + }) + } + client := NewClient(t.TempDir(), []string{first.URL, second.URL}) + client.Platform = "freebsd_amd64" + if _, err := client.Install(testResourceOne, "", ""); err == nil { + t.Fatal("unsupported platforms must not install") + } + client.Platform = "linux_amd64" + if _, err := client.Install(testResourceTwo, "", ""); err == nil { + t.Fatal("missing resource must fail") + } + // Once the first repository has only incompatible candidates, use the next one. + client.Platform = "darwin_arm64" + result, err := client.Install(testResourceOne, "", "") + if err != nil || result.Version != "9.0.0" { + t.Fatalf("repository precedence: %+v %v", result, err) + } +} + +func TestLatestResolutionErrorsDoNotFallBack(t *testing.T) { + for _, failure := range []string{"catalog", "digest", "signature", "archive signature"} { + t.Run(failure, func(t *testing.T) { + repo := newTestRepository(t) + repo.discovery() + repo.set("/v1/policies.json", []byte(`{"policies":[]}`), nil) + descriptor := []byte(`{"packages":{"example.test/shared":{"versions":{"4.0.0":{}}}}}`) + entry := CatalogEntry{URL: "/descriptor.json", Digest: "sha256:" + digest(descriptor)} + repo.set("/descriptor.json", descriptor, nil) + switch failure { + case "digest": + entry.Digest = "sha256:" + strings.Repeat("0", 64) + case "signature", "archive signature": + key, policy := testSigningPolicy(t) + policies, _ := json.Marshal(policyDocument{Policies: []Policy{policy}}) + repo.set("/v1/policies.json", policies, nil) + if failure == "archive signature" { + signed := signedAsset(t, key, descriptor) + repo.set("/descriptor.json", signed.body, signed.headers) + repo.set("/v1/packages/example.test/shared/4.0.0.json", []byte(`{"archives":{}}`), nil) + } + } + catalog, _ := json.Marshal(Catalog{Resources: map[string]map[string]CatalogEntry{ + testResourceOne: {"1.0.0": entry}, + }}) + if failure == "catalog" { + catalog = []byte("{") + } + repo.set("/v1/catalog.json", catalog, nil) + var fallbackCalls atomic.Int64 + fallback := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fallbackCalls.Add(1) + http.NotFound(w, r) + })) + defer fallback.Close() + client := NewClient(t.TempDir(), []string{repo.server.URL, fallback.URL}) + if _, err := client.Install(testResourceOne, "", ""); err == nil || fallbackCalls.Load() != 0 { + t.Fatalf("failed metadata must not fall back: err=%v fallback requests=%d", err, fallbackCalls.Load()) + } + }) + } +} + +func TestResourceStateAndSharedSet(t *testing.T) { + t.Setenv("DSC_RESOURCE_PATH", "") + repository := t.TempDir() + publishResourceFixture(t, repository, testPackage, "4.0.0", "1.0.0", currentPlatform(), testResourceOne, testResourceTwo) + var requests atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + http.FileServer(http.Dir(repository)).ServeHTTP(w, r) + })) + defer server.Close() + client := NewClient(filepath.Join(t.TempDir(), "missing"), nil) + desired := ResourceState{Resource: testResourceOne, Repositories: []string{server.URL}} + actual, differences, err := client.TestResource(desired, io.Discard) + if err != nil || actual.State != "missing" || !reflect.DeepEqual(differences, []string{"state"}) { + t.Fatalf("missing: %+v %v %v", actual, differences, err) + } + if _, err := os.Stat(client.PackagesDir); !os.IsNotExist(err) { + t.Fatal("Test created the package store") + } + actual, err = client.SetResource(desired, io.Discard) + if err != nil || actual.Version != "1.0.0" || actual.PackageVersion != "4.0.0" { + t.Fatalf("Set: %+v %v", actual, err) + } + initialRequests := requests.Load() + before := discoverySnapshot(t, client.PackagesDir) + for _, resource := range []string{testResourceOne, testResourceTwo} { + for _, version := range []string{"", "1.0.0"} { + desired.Resource, desired.Version = resource, version + state, diff, err := client.TestResource(desired, io.Discard) + if err != nil || len(diff) != 0 || !*state.InDesiredState { + t.Fatalf("shared resource compliance: %+v %v %v", state, diff, err) + } + if _, err := client.SetResource(desired, io.Discard); err != nil { + t.Fatal(err) + } + } + } + if requests.Load() != initialRequests { + t.Fatal("compliant operations accessed the network") + } + registry, err := ReadRegistry(client.PackagesDir) + if err != nil || len(registry.Resources) != 1 { + t.Fatalf("shared Set must not register a second resource: %+v %v", registry, err) + } + server.Close() + if _, err := client.GetResource(desired, io.Discard); err != nil { + t.Fatalf("offline Get: %v", err) + } + if _, err := client.SetResource(desired, io.Discard); err != nil { + t.Fatalf("offline compliant Set: %v", err) + } + if after := discoverySnapshot(t, client.PackagesDir); !reflect.DeepEqual(before, after) { + t.Fatal("Get/Test/compliant Set mutated the store") + } + desired.Version = "2.0.0" + actual, diff, err := client.TestResource(desired, io.Discard) + if err != nil || actual.State != "missing" || !reflect.DeepEqual(diff, []string{"state", "version"}) { + t.Fatalf("wrong version: %+v %v %v", actual, diff, err) + } + desired.State = "missing" + if _, err := client.SetResource(desired, io.Discard); err == nil { + t.Fatal("Set must reject removal") + } +} + +func TestResourceExplicitSetAndNoAutomaticUpgrade(t *testing.T) { + t.Setenv("DSC_RESOURCE_PATH", "") + repository := t.TempDir() + for _, version := range []string{"1.0.0", "2.0.0"} { + publishResourceFixture(t, repository, testPackage, version, version, currentPlatform(), testResourceOne) + } + server := httptest.NewServer(http.FileServer(http.Dir(repository))) + defer server.Close() + client := NewClient(t.TempDir(), []string{server.URL}) + desired := ResourceState{Resource: testResourceOne, Version: "1.0.0"} + if _, err := client.SetResource(desired, io.Discard); err != nil { + t.Fatal(err) + } + desired.Version = "" + actual, err := client.SetResource(desired, io.Discard) + if err != nil || actual.Version != "1.0.0" { + t.Fatalf("unversioned Set upgraded installed resource: %+v %v", actual, err) + } + desired.Version = "2.0.0" + if _, err := client.SetResource(desired, io.Discard); err != nil { + t.Fatal(err) + } + actual, err = client.GetResource(ResourceState{Resource: testResourceOne}, io.Discard) + if err != nil || actual.Version != "2.0.0" { + t.Fatalf("newest installed version: %+v %v", actual, err) + } + states, err := client.InstalledResources(io.Discard) + if err != nil || len(states) != 2 { + t.Fatalf("Set deleted older versions: %+v %v", states, err) + } +} + +func TestResourceInventoryFormsAndPlatform(t *testing.T) { + root := t.TempDir() + for _, pkg := range []struct{ name, platform string }{ + {"example/a/1", currentPlatform()}, {"example/z/2", currentPlatform()}, + {"example/foreign/1", "other_arch"}, {"example/legacy/1", ""}, + } { + directory := discoveryPackage(t, root, pkg.name, pkg.platform) + discoveryFile(t, filepath.Join(directory, "one.dsc.resource.json"), `{"type":"Example/One","version":"1.0.0"}`) + if pkg.name != "example/a/1" { + continue + } + discoveryFile(t, filepath.Join(directory, "two.dsc.resource.yaml"), "type: Example/Two\nversion: 1.0.0\nschema: !embedded {}\n") + discoveryFile(t, filepath.Join(directory, "three.dsc.resource.yml"), "type: Example/Three\nversion: 1.0.0\n") + discoveryFile(t, filepath.Join(directory, "list.dsc.manifests.json"), `{"resources":[{"type":"Example/Four","version":"2.0.0"},{"type":"Example/One","version":"1.0.0"},{"type":"Example/Adapted","version":"1.0.0","requireAdapter":"Example/Adapter"}]}`) + discoveryFile(t, filepath.Join(directory, "list.dsc.manifests.yaml"), "resources:\n - type: Example/Five\n version: 1.0.0\n") + discoveryFile(t, filepath.Join(directory, "adapted.dsc.adaptedresource.json"), `{"type":"Example/Adapted","version":"1.0.0","requireAdapter":"Example/Adapter"}`) + } + client := NewClient(root, nil) + before := discoverySnapshot(t, root) + var diagnostics bytes.Buffer + states, err := client.InstalledResources(&diagnostics) + if err != nil || len(states) != 5 { + t.Fatalf("inventory: %+v %v", states, err) + } + if !strings.Contains(diagnostics.String(), "adapter") || !strings.Contains(diagnostics.String(), "reinstall") { + t.Fatalf("missing limitations diagnostics: %s", &diagnostics) + } + for i := range 3 { + repeated, err := client.InstalledResources(io.Discard) + if err != nil || !reflect.DeepEqual(repeated, states) { + t.Fatalf("nondeterministic inventory %d: %+v %v", i, repeated, err) + } + } + if after := discoverySnapshot(t, root); !reflect.DeepEqual(before, after) { + t.Fatal("inventory modified filesystem state") + } + if states[2].Resource != "Example/One" || states[2].Package != "example/z" { + t.Fatalf("duplicate selection should prefer newer package: %+v", states) + } + discoveryFile(t, filepath.Join(root, "example", "a", ".1.dscpkg.json"), "{") + if _, err := client.InstalledResources(io.Discard); err == nil { + t.Fatal("corrupt provenance was hidden") + } +} + +func TestResourceMalformedManifestsAndFinalState(t *testing.T) { + for _, fixture := range []struct{ name, content string }{ + {"bad.dsc.resource.json", "{"}, {"bad.dsc.resource.yaml", ":"}, + {"bad.dsc.resource.json", "{}"}, {"bad.dsc.manifests.json", `{"type":"Example/NotAnInstance","version":"1.0.0"}`}, + } { + t.Run(fixture.name+fixture.content, func(t *testing.T) { + root := t.TempDir() + dir := discoveryPackage(t, root, "example/shared/1", currentPlatform()) + discoveryFile(t, filepath.Join(dir, fixture.name), fixture.content) + if _, err := NewClient(root, nil).InstalledResources(io.Discard); err == nil { + t.Fatal("malformed manifest accepted") + } + }) + } + t.Setenv("DSC_RESOURCE_PATH", "") + repository := t.TempDir() + // The catalog advertises One, but the archive only contains Two. + publishResourceFixture(t, repository, testPackage, "1.0.0", "1.0.0", currentPlatform(), testResourceTwo) + catalogPath := filepath.Join(repository, "v1", "catalog.json") + data, err := os.ReadFile(catalogPath) + if err != nil { + t.Fatal(err) + } + var catalog Catalog + if err := json.Unmarshal(data, &catalog); err != nil { + t.Fatal(err) + } + catalog.Resources[testResourceOne] = catalog.Resources[testResourceTwo] + data, err = json.Marshal(catalog) + if err != nil { + t.Fatal(err) + } + discoveryFile(t, catalogPath, string(data)) + server := httptest.NewServer(http.FileServer(http.Dir(repository))) + defer server.Close() + client := NewClient(t.TempDir(), []string{server.URL}) + if _, err := client.SetResource(ResourceState{Resource: testResourceOne}, io.Discard); err == nil || !strings.Contains(err.Error(), "does not expose") { + t.Fatalf("Set must verify discoverability after installation: %v", err) + } + _, err = client.SetResource(ResourceState{Resource: "invalid"}, io.Discard) + if err == nil { + t.Fatal("invalid resource identity accepted") + } +} diff --git a/packaging/build.ps1 b/packaging/build.ps1 index 6c80f32..1f4b322 100644 --- a/packaging/build.ps1 +++ b/packaging/build.ps1 @@ -24,7 +24,9 @@ $executable = if ($IsWindows) { 'dscpkg.exe' } else { 'dscpkg' } try { New-Item -ItemType Directory -Path (Join-Path $package 'bin') -Force | Out-Null Copy-Item -LiteralPath $binary -Destination (Join-Path $package "bin/$executable") - Copy-Item -LiteralPath (Join-Path $repo 'dscpkg.dsc.extension.json') -Destination (Join-Path $package 'bin') + foreach ($manifest in @('dscpkg.dsc.extension.json', 'dscpkg.dsc.resource.json')) { + Copy-Item -LiteralPath (Join-Path $repo $manifest) -Destination (Join-Path $package 'bin') + } foreach ($file in @('README.md', 'LICENSE')) { Copy-Item -LiteralPath (Join-Path $repo $file) -Destination $package } diff --git a/packaging/linux/inspect.sh b/packaging/linux/inspect.sh index 4f9ec90..e7e4263 100644 --- a/packaging/linux/inspect.sh +++ b/packaging/linux/inspect.sh @@ -28,12 +28,14 @@ rpm2archive -n < "$rpm" > "$stage/rpm.tar" tar -xf "$stage/rpm.tar" -C "$stage/rpm" for format in deb rpm; do extracted="$stage/$format" - diff <(printf '%s\n' usr/bin/dscpkg usr/bin/dscpkg.dsc.extension.json usr/share/doc/dscpkg/README.md usr/share/licenses/dscpkg/LICENSE | sort) \ + diff <(printf '%s\n' usr/bin/dscpkg usr/bin/dscpkg.dsc.extension.json usr/bin/dscpkg.dsc.resource.json usr/share/doc/dscpkg/README.md usr/share/licenses/dscpkg/LICENSE | sort) \ <(find "$extracted" -type f -printf '%P\n' | sort) [[ $(stat -c %a "$extracted/usr/bin/dscpkg") == 755 ]] cmp "$binary" "$extracted/usr/bin/dscpkg" [[ $(stat -c %a "$extracted/usr/bin/dscpkg.dsc.extension.json") == 644 ]] cmp "$root/dscpkg.dsc.extension.json" "$extracted/usr/bin/dscpkg.dsc.extension.json" + [[ $(stat -c %a "$extracted/usr/bin/dscpkg.dsc.resource.json") == 644 ]] + cmp "$root/dscpkg.dsc.resource.json" "$extracted/usr/bin/dscpkg.dsc.resource.json" cmp "$root/README.md" "$extracted/usr/share/doc/dscpkg/README.md" cmp "$root/LICENSE" "$extracted/usr/share/licenses/dscpkg/LICENSE" "$extracted/usr/bin/dscpkg" publish -h diff --git a/packaging/linux/nfpm.yaml b/packaging/linux/nfpm.yaml index 95ffed8..2ad3e81 100644 --- a/packaging/linux/nfpm.yaml +++ b/packaging/linux/nfpm.yaml @@ -20,6 +20,12 @@ contents: mode: 0644 owner: root group: root + - src: dscpkg.dsc.resource.json + dst: /usr/bin/dscpkg.dsc.resource.json + file_info: + mode: 0644 + owner: root + group: root - src: README.md dst: /usr/share/doc/dscpkg/README.md file_info: diff --git a/packaging/linux/test-install.sh b/packaging/linux/test-install.sh index aa148d6..11667f1 100644 --- a/packaging/linux/test-install.sh +++ b/packaging/linux/test-install.sh @@ -30,6 +30,7 @@ case "$format" in esac [[ ! -e /usr/bin/dscpkg ]] [[ ! -e /usr/bin/dscpkg.dsc.extension.json ]] +[[ ! -e /usr/bin/dscpkg.dsc.resource.json ]] cache="$HOME/.cache/dscpkg/packages" [[ ! -e "$cache" ]] mkdir -p "$cache/.dscpkg" "$cache/test-resource" @@ -57,8 +58,11 @@ verify_install() { [[ $(command -v dscpkg) -ef /usr/bin/dscpkg ]] cmp "$1" /usr/bin/dscpkg cmp "$root/dscpkg.dsc.extension.json" /usr/bin/dscpkg.dsc.extension.json + cmp "$root/dscpkg.dsc.resource.json" /usr/bin/dscpkg.dsc.resource.json env -u DSC_RESOURCE_PATH -u DSC_RESTRICTED_PATH "$DSC_TEST_BIN" --ignore-settings-file \ extension list DscPkg/Discover -o json | grep -F '"type":"DscPkg/Discover"' + env -u DSC_RESOURCE_PATH -u DSC_RESTRICTED_PATH "$DSC_TEST_BIN" --ignore-settings-file \ + resource list DscPkg/Resource -o json | grep -F '"type":"DscPkg/Resource"' dscpkg publish -h if [[ $format == deb ]]; then # dpkg-query, not Bash, expands this field. @@ -76,11 +80,13 @@ candidate=$(package_path "$assets" "$version") install_package "$candidate" verify_install "$binary" "$version" rm /usr/bin/dscpkg.dsc.extension.json +rm /usr/bin/dscpkg.dsc.resource.json install_package "$candidate" verify_install "$binary" "$version" remove_package [[ ! -e /usr/bin/dscpkg ]] [[ ! -e /usr/bin/dscpkg.dsc.extension.json ]] +[[ ! -e /usr/bin/dscpkg.dsc.resource.json ]] preserved if cmp -s "$fixtures/v1.0.0/dscpkg" "$fixtures/v1.0.1/dscpkg"; then @@ -90,10 +96,12 @@ fi install_package "$(package_path "$fixtures/v1.0.0/assets" 1.0.0)" verify_install "$fixtures/v1.0.0/dscpkg" 1.0.0 printf '{"stale":"upgrade fixture"}\n' > /usr/bin/dscpkg.dsc.extension.json +printf '{"stale":"upgrade fixture"}\n' > /usr/bin/dscpkg.dsc.resource.json install_package "$(package_path "$fixtures/v1.0.1/assets" 1.0.1)" verify_install "$fixtures/v1.0.1/dscpkg" 1.0.1 remove_package [[ ! -e /usr/bin/dscpkg ]] [[ ! -e /usr/bin/dscpkg.dsc.extension.json ]] +[[ ! -e /usr/bin/dscpkg.dsc.resource.json ]] preserved echo "$format install, reinstall, upgrade, uninstall and data retention passed." diff --git a/packaging/verify.ps1 b/packaging/verify.ps1 index df16bac..aa0f028 100644 --- a/packaging/verify.ps1 +++ b/packaging/verify.ps1 @@ -24,12 +24,14 @@ try { $actual = @(Get-ChildItem -LiteralPath $stage -Recurse -File | ForEach-Object { [IO.Path]::GetRelativePath($stage, $_.FullName).Replace('\', '/') }) - $expected = @("$name/bin/$executable", "$name/bin/dscpkg.dsc.extension.json", "$name/LICENSE", "$name/README.md") + $expected = @("$name/bin/$executable", "$name/bin/dscpkg.dsc.extension.json", "$name/bin/dscpkg.dsc.resource.json", "$name/LICENSE", "$name/README.md") Assert-Equal (($actual | Sort-Object) -join "`n") (($expected | Sort-Object) -join "`n") 'Archive contents' $unpacked = Join-Path $stage "$name/bin/$executable" Assert-Equal (Get-SHA256 $unpacked) (Get-SHA256 $binary) 'Archive executable hash' - Assert-Equal (Get-SHA256 (Join-Path $stage "$name/bin/dscpkg.dsc.extension.json")) ` - (Get-SHA256 (Join-Path (Split-Path $PSScriptRoot -Parent) 'dscpkg.dsc.extension.json')) 'Archive discovery extension' + foreach ($manifest in @('dscpkg.dsc.extension.json', 'dscpkg.dsc.resource.json')) { + Assert-Equal (Get-SHA256 (Join-Path $stage "$name/bin/$manifest")) ` + (Get-SHA256 (Join-Path (Split-Path $PSScriptRoot -Parent) $manifest)) "Archive $manifest" + } foreach ($file in @('README.md', 'LICENSE')) { Assert-Equal (Get-SHA256 (Join-Path $stage "$name/$file")) ` (Get-SHA256 (Join-Path (Split-Path $PSScriptRoot -Parent) $file)) "Archive $file" diff --git a/packaging/windows/msi/Package.wxs b/packaging/windows/msi/Package.wxs index 1d19667..f7dc365 100644 --- a/packaging/windows/msi/Package.wxs +++ b/packaging/windows/msi/Package.wxs @@ -25,11 +25,15 @@ + + + + diff --git a/packaging/windows/msi/inspect.ps1 b/packaging/windows/msi/inspect.ps1 index 8f519b3..5783bb9 100644 --- a/packaging/windows/msi/inspect.ps1 +++ b/packaging/windows/msi/inspect.ps1 @@ -29,7 +29,7 @@ try { } $files = @(Read-MsiTable $msi 'SELECT `FileName` FROM `File`' @('Name')) $names = @($files | ForEach-Object { ($_.Name -split '\|')[-1] }) -Assert-Equal (($names | Sort-Object) -join ',') 'dscpkg.dsc.extension.json,dscpkg.exe,LICENSE,README.md' 'MSI file set' +Assert-Equal (($names | Sort-Object) -join ',') 'dscpkg.dsc.extension.json,dscpkg.dsc.resource.json,dscpkg.exe,LICENSE,README.md' 'MSI file set' $directories = @(Read-MsiTable $msi 'SELECT `Directory_Parent`, `DefaultDir` FROM `Directory` WHERE `Directory` = ''INSTALLFOLDER''' @('Parent', 'Name')) Assert-Equal $directories[0].Parent 'ProgramFiles64Folder' 'MSI destination parent' Assert-Equal (($directories[0].Name -split '\|')[-1]) 'dscpkg' 'MSI destination directory' @@ -61,8 +61,10 @@ try { Assert-Equal (Get-SHA256 (Join-Path $binaries[0].DirectoryName $file)) ` (Get-SHA256 (Join-Path $repo $file)) "MSI $file" } - Assert-Equal (Get-SHA256 (Join-Path $binaries[0].DirectoryName 'dscpkg.dsc.extension.json')) ` - (Get-SHA256 (Join-Path $repo 'dscpkg.dsc.extension.json')) 'MSI discovery extension' + foreach ($manifest in @('dscpkg.dsc.extension.json', 'dscpkg.dsc.resource.json')) { + Assert-Equal (Get-SHA256 (Join-Path $binaries[0].DirectoryName $manifest)) ` + (Get-SHA256 (Join-Path $repo $manifest)) "MSI $manifest" + } Invoke-Checked $binaries[0].FullName @('publish', '-h') $verified = $true } finally { diff --git a/packaging/windows/test-install.ps1 b/packaging/windows/test-install.ps1 index df1c1d9..3c15694 100644 --- a/packaging/windows/test-install.ps1 +++ b/packaging/windows/test-install.ps1 @@ -18,6 +18,7 @@ $null = & "$PSScriptRoot/../version.ps1" $Version $installDirectory = Join-Path $env:ProgramFiles 'dscpkg' $installedBinary = Join-Path $installDirectory 'dscpkg.exe' $installedExtension = Join-Path $installDirectory 'dscpkg.dsc.extension.json' +$installedResource = Join-Path $installDirectory 'dscpkg.dsc.resource.json' if (Test-Path -LiteralPath $installDirectory) { throw 'dscpkg is already installed; refusing to modify it.' } $logs = [IO.Path]::GetFullPath($LogDirectory) New-Item -ItemType Directory -Path $logs -Force | Out-Null @@ -41,6 +42,8 @@ function Assert-Installed { Assert-Equal (Get-SHA256 $installedBinary) (Get-SHA256 $ExpectedBinary) 'Installed executable' Assert-Equal (Get-SHA256 $installedExtension) ` (Get-SHA256 (Join-Path $PSScriptRoot '../../dscpkg.dsc.extension.json')) 'Installed discovery extension' + Assert-Equal (Get-SHA256 $installedResource) ` + (Get-SHA256 (Join-Path $PSScriptRoot '../../dscpkg.dsc.resource.json')) 'Installed resource manifest' $machinePath = [Environment]::GetEnvironmentVariable('Path', 'Machine') $entries = @($machinePath -split ';' | ForEach-Object { $_.TrimEnd('\') }) Assert-Equal @($entries | Where-Object { $_ -ieq $installDirectory }).Count 1 'Product PATH entry count' @@ -54,6 +57,11 @@ function Assert-Installed { $extension = $extensions | ConvertFrom-Json Assert-Equal $extension.type 'DscPkg/Discover' 'Installed DSC extension' Assert-Equal $extension.path $installedExtension 'Installed DSC extension path' + $resources = & $env:DSC_TEST_BIN --ignore-settings-file resource list DscPkg/Resource -o json + if ($LASTEXITCODE -ne 0) { throw 'DSC resource discovery failed.' } + $resource = $resources | ConvertFrom-Json + Assert-Equal $resource.type 'DscPkg/Resource' 'Installed DSC resource' + Assert-Equal $resource.path $installedResource 'Installed DSC resource path' } finally { $env:DSC_RESOURCE_PATH, $env:DSC_RESTRICTED_PATH = $resourcePath, $restrictedPath } @@ -67,6 +75,7 @@ function Remove-TestProduct { $script:installed = $null if (Test-Path -LiteralPath $installedBinary) { throw 'Uninstall left the executable behind.' } if (Test-Path -LiteralPath $installedExtension) { throw 'Uninstall left the discovery extension behind.' } + if (Test-Path -LiteralPath $installedResource) { throw 'Uninstall left the resource manifest behind.' } $entries = @([Environment]::GetEnvironmentVariable('Path', 'Machine') -split ';' | ForEach-Object { $_.TrimEnd('\') }) if ($entries -icontains $installDirectory) { throw 'Uninstall left its PATH entry behind.' } @@ -82,6 +91,7 @@ try { $installed = $candidate Assert-Installed $BinaryPath Remove-Item -LiteralPath $installedExtension + Remove-Item -LiteralPath $installedResource Invoke-Msi '/fa' $candidate (Join-Path $logs 'repair-release.log') Assert-Installed $BinaryPath Remove-TestProduct 'uninstall-release' @@ -95,6 +105,7 @@ try { $installed = $old Assert-Installed $oldBinary Set-Content -LiteralPath $installedExtension -Value '{"stale":"upgrade fixture"}' + Set-Content -LiteralPath $installedResource -Value '{"stale":"upgrade fixture"}' Invoke-Msi '/i' $new (Join-Path $logs 'upgrade.log') $installed = $new Assert-Installed $newBinary diff --git a/tests/discovery_test.go b/tests/discovery_test.go index e18c641..648eafd 100644 --- a/tests/discovery_test.go +++ b/tests/discovery_test.go @@ -15,6 +15,7 @@ import ( "path/filepath" "runtime" "strings" + "sync/atomic" "testing" ) @@ -71,19 +72,22 @@ func TestDiscoveryDSC33(t *testing.T) { t.Fatalf("build %s: %v\n%s", build.source, err, output) } } - manifest, err := os.ReadFile(filepath.Join(root, "dscpkg.dsc.extension.json")) - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(filepath.Join(bin, "dscpkg.dsc.extension.json"), manifest, 0o644); err != nil { - t.Fatal(err) + for _, name := range []string{"dscpkg.dsc.extension.json", "dscpkg.dsc.resource.json"} { + manifest, err := os.ReadFile(filepath.Join(root, name)) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(bin, name), manifest, 0o644); err != nil { + t.Fatal(err) + } } // Both tools use an isolated default per-user store; no machine configuration is changed. t.Setenv("LOCALAPPDATA", filepath.Join(temp, "cache")) t.Setenv("XDG_CACHE_HOME", filepath.Join(temp, "cache")) t.Setenv("HOME", temp) t.Setenv("USERPROFILE", temp) - t.Setenv("PATH", bin+string(os.PathListSeparator)+os.Getenv("PATH")) + t.Setenv("PATH", bin) + unsetIntegrationEnv(t, "DSCPKG_PACKAGES_DIR") unsetIntegrationEnv(t, "DSC_RESOURCE_PATH") unsetIntegrationEnv(t, "DSC_RESTRICTED_PATH") @@ -118,7 +122,13 @@ func TestDiscoveryDSC33(t *testing.T) { } repository := filepath.Join(temp, "repository") - server := httptest.NewServer(http.FileServer(http.Dir(repository))) + var downloads atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, ".zip") { + downloads.Add(1) + } + http.FileServer(http.Dir(repository)).ServeHTTP(w, r) + })) defer server.Close() platform := runtime.GOOS + "_" + runtime.GOARCH install := func(pkg, targetPlatform string, resources ...string) string { @@ -130,12 +140,16 @@ func TestDiscoveryDSC33(t *testing.T) { args = append(args, "--resource", resource+"@1.0.0") } run(cli, args...) + if pkg == "example/shared" { + integrationResourceScenario(t, dsc, cli, temp, server.URL, &downloads, run) + } output := run(cli, "install", "--repository", server.URL, "--platform", targetPlatform, - "--resource", resources[0], "--version", "1.0.0") + "--resource", resources[0]) var result struct { - Path string `json:"path"` + Path string `json:"path"` + Version string `json:"version"` } - if err := json.Unmarshal(output, &result); err != nil || result.Path == "" { + if err := json.Unmarshal(output, &result); err != nil || result.Path == "" || result.Version != "1.0.0" { t.Fatalf("invalid install result: %s, %v", output, err) } return result.Path diff --git a/tests/resource_test.go b/tests/resource_test.go new file mode 100644 index 0000000..c3c2450 --- /dev/null +++ b/tests/resource_test.go @@ -0,0 +1,255 @@ +//go:build integration + +package tests + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + "sync/atomic" + "testing" + "time" +) + +func integrationResourceScenario(t *testing.T, dsc, cli, temp, repository string, downloads *atomic.Int64, run func(string, ...string) []byte) { + t.Helper() + list := integrationObjects(t, run(dsc, "resource", "list", "DscPkg/Resource", "-o", "json")) + if len(list) != 1 || list["DscPkg/Resource"] == nil { + t.Fatalf("resource manifest was not bootstrapped through PATH: %v", list) + } + input := fmt.Sprintf(`{"resource":"DscPkg.Test/One","state":"installed","repositories":[%q]}`, repository) + assertState := func(operation, input, field, expected string) []byte { + t.Helper() + output := run(dsc, "resource", operation, "--resource", "DscPkg/Resource", "--input", input, "-o", "json") + var result map[string]json.RawMessage + if err := json.Unmarshal(output, &result); err != nil { + t.Fatal(err) + } + if !bytes.Contains(result[field], []byte(expected)) { + t.Fatalf("%s missing %s in %s: %s", operation, expected, field, output) + } + if bytes.Contains(result[field], []byte(`"_name"`)) { + t.Fatalf("%s returned Export-only _name: %s", operation, output) + } + return output + } + assertState("get", input, "actualState", `"state":"missing"`) + output := assertState("test", input, "actualState", `"state":"missing"`) + var compliance struct { + InDesiredState bool `json:"inDesiredState"` + DiffProperties []string `json:"differingProperties"` + } + if err := json.Unmarshal(output, &compliance); err != nil || compliance.InDesiredState || + len(compliance.DiffProperties) != 1 || compliance.DiffProperties[0] != "state" { + t.Fatalf("missing resource should not be compliant: %s %v", output, err) + } + if output := run(cli, "resource", "export"); len(output) != 0 { + t.Fatalf("empty inventory: %s", output) + } + config := fmt.Sprintf(`$schema: https://aka.ms/dsc/schemas/v3/bundled/config/document.json +resources: + - name: One + type: DscPkg/Resource + properties: + resource: DscPkg.Test/One + state: installed + repositories: [%q] + - name: Two + type: DscPkg/Resource + properties: + resource: DscPkg.Test/Two + state: installed + repositories: [%q] +`, repository, repository) + configPath := filepath.Join(temp, "00-resources.yaml") + if err := os.WriteFile(configPath, []byte(config), 0o644); err != nil { + t.Fatal(err) + } + before := downloads.Load() + assertConfigurationSuccess(t, run(dsc, "config", "set", "--file", configPath, "-o", "json"), 2) + if downloads.Load()-before != 1 { + t.Fatalf("shared resources require exactly one archive download; got %d", downloads.Load()-before) + } + installed := downloads.Load() + output = assertState("test", input, "actualState", `"state":"installed"`) + if err := json.Unmarshal(output, &compliance); err != nil || !compliance.InDesiredState { + t.Fatalf("installed resource should be compliant: %s %v", output, err) + } + // Invalid repository origins prove that compliant operations do not resolve remotely. + offline := `{"resource":"DscPkg.Test/Two","version":"1.0.0","repositories":["invalid://offline"]}` + assertState("get", offline, "actualState", `"package":"example/shared"`) + assertState("test", offline, "actualState", `"state":"installed"`) + assertState("set", offline, "afterState", `"state":"installed"`) + run(dsc, "config", "set", "--file", configPath, "-o", "json") + if downloads.Load() != installed { + t.Fatal("repeat Set downloaded a package") + } + export := run(dsc, "resource", "export", "--resource", "DscPkg/Resource", "-o", "json") + var exported struct { + Resources []struct { + Name string `json:"name"` + Type string `json:"type"` + Properties struct { + Resource string `json:"resource"` + Version string `json:"version"` + State string `json:"state"` + Name json.RawMessage `json:"_name"` + } `json:"properties"` + } `json:"resources"` + } + if err := json.Unmarshal(export, &exported); err != nil || len(exported.Resources) != 2 { + t.Fatalf("DSC Export must return two schema-validated instances: %s %v", export, err) + } + for _, resource := range exported.Resources { + if resource.Type != "DscPkg/Resource" || resource.Properties.Version != "1.0.0" || resource.Properties.State != "installed" || + resource.Name != resource.Properties.Resource+"@"+resource.Properties.Version || len(resource.Properties.Name) != 0 { + t.Fatalf("invalid exported resource: %+v", resource) + } + } + first := run(cli, "resource", "export") + if !bytes.Equal(first, run(cli, "resource", "export")) || bytes.Count(first, []byte("\n")) != 2 { + t.Fatalf("Export must produce deterministic JSON Lines: %s", first) + } + for _, line := range bytes.Split(bytes.TrimSpace(first), []byte("\n")) { + var instance struct { + Name string `json:"_name"` + Resource string `json:"resource"` + Version string `json:"version"` + } + if err := json.Unmarshal(line, &instance); err != nil || instance.Name != instance.Resource+"@"+instance.Version { + t.Fatalf("Export instance name must be resource@version: %s %v", line, err) + } + } + // Force DSC to validate every exported instance as input, including observed fields. + assertConfigurationSuccess(t, run(dsc, "config", "test", "--input", string(export), "-o", "json"), 2) + for _, discoveryMode := range []string{"preDeployment", "duringDeployment"} { + t.Run("server "+discoveryMode, func(t *testing.T) { + t.Setenv("DSCPKG_PACKAGES_DIR", filepath.Join(temp, "server-"+discoveryMode)) + integrationServerConfigurations(t, dsc, config, discoveryMode) + }) + } +} + +func assertConfigurationSuccess(t *testing.T, data []byte, count int) { + t.Helper() + var result struct { + HadErrors bool `json:"hadErrors"` + Results []json.RawMessage `json:"results"` + } + if err := json.Unmarshal(data, &result); err != nil || result.HadErrors || len(result.Results) != count { + t.Fatalf("configuration failed: %s %v", data, err) + } +} + +func integrationServerConfigurations(t *testing.T, dsc, config, mode string) { + t.Helper() + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + defer cancel() + command := exec.CommandContext(ctx, dsc, "--ignore-settings-file", "--trace-level", "error", "--progress-format", "none", "server") + var stderr bytes.Buffer + command.Stderr = &stderr + stdin, err := command.StdinPipe() + if err != nil { + t.Fatal(err) + } + stdout, err := command.StdoutPipe() + if err != nil { + t.Fatal(err) + } + if err := command.Start(); err != nil { + t.Fatal(err) + } + defer func() { + stdin.Close() + if err := command.Wait(); err != nil { + t.Errorf("DSC server exit: %v\n%s", err, &stderr) + } + }() + encoder, decoder := json.NewEncoder(stdin), json.NewDecoder(stdout) + id := 0 + request := func(method string, params any) map[string]json.RawMessage { + t.Helper() + id++ + if err := encoder.Encode(map[string]any{"jsonrpc": "2.0", "id": id, "method": method, "params": params}); err != nil { + t.Fatal(err) + } + for { + var response map[string]json.RawMessage + if err := decoder.Decode(&response); err != nil { + t.Fatalf("read server response: %v", err) + } + if len(response["id"]) == 0 { + continue + } + if string(response["id"]) != fmt.Sprint(id) { + t.Fatalf("server request %s failed: %s", method, mustIntegrationJSON(t, response)) + } + return response + } + } + request("initialize", map[string]any{ + "protocolVersion": "2024-11-05", "capabilities": map[string]any{}, + "clientInfo": map[string]string{"name": "dscpkg-integration", "version": "1.0.0"}, + }) + if err := encoder.Encode(map[string]string{"jsonrpc": "2.0", "method": "notifications/initialized"}); err != nil { + t.Fatal(err) + } + invoke := func(operation, configuration string) { + t.Helper() + response := request("tools/call", map[string]any{ + "name": "invoke_dsc_config", + "arguments": map[string]string{ + "operation": operation, "configuration": configuration, + }, + }) + if operation == "get" && mode == "preDeployment" { + if !bytes.Contains(response["error"], []byte("Resource not found")) { + t.Fatalf("expected stale preDeployment server discovery: %s", mustIntegrationJSON(t, response)) + } + t.Log("DSC 3.3.0 server retained stale preDeployment discovery between configurations") + return + } + if response["error"] != nil { + t.Fatalf("server configuration failed: %s", mustIntegrationJSON(t, response)) + } + var result struct { + IsError bool `json:"isError"` + StructuredContent struct { + Result json.RawMessage `json:"result"` + } `json:"structuredContent"` + } + if err := json.Unmarshal(response["result"], &result); err != nil || result.IsError || len(result.StructuredContent.Result) == 0 { + t.Fatalf("server configuration failed: %s %v", mustIntegrationJSON(t, response), err) + } + count := 1 + if operation == "set" { + count = 2 + } + assertConfigurationSuccess(t, result.StructuredContent.Result, count) + } + invoke("set", config) + invoke("get", fmt.Sprintf(`$schema: https://aka.ms/dsc/schemas/v3/bundled/config/document.json +directives: + resourceDiscovery: %s +resources: + - name: NewlyInstalled + type: DscPkg.Test/Two +`, mode)) + if mode == "duringDeployment" { + t.Log("DSC 3.3.0 server refreshed resources between configurations with duringDeployment") + } +} + +func mustIntegrationJSON(t *testing.T, value any) string { + t.Helper() + data, err := json.Marshal(value) + if err != nil { + t.Fatal(err) + } + return strings.TrimSpace(string(data)) +} diff --git a/tests/test-discovery.ps1 b/tests/test-discovery.ps1 index caf57f7..92a1a2f 100644 --- a/tests/test-discovery.ps1 +++ b/tests/test-discovery.ps1 @@ -14,6 +14,11 @@ try { if (-not (Get-Content -LiteralPath ./dscpkg.dsc.extension.json -Raw | Test-Json -Schema ($schema -join "`n"))) { throw 'The discovery extension does not validate against the DSC 3.3.0 schema.' } + $schema = & $dsc schema --type resource-manifest --output-format json + if ($LASTEXITCODE -ne 0) { throw 'Could not export the DSC resource manifest schema.' } + if (-not (Get-Content -LiteralPath ./dscpkg.dsc.resource.json -Raw | Test-Json -Schema ($schema -join "`n"))) { + throw 'The resource manifest does not validate against the DSC 3.3.0 schema.' + } $env:DSC_TEST_BIN = $dsc & go test -tags=integration -count=1 -v ./tests if ($LASTEXITCODE -ne 0) { throw 'DSC discovery integration tests failed.' }