From 9f412287463b33d67bf7e6f3901ac04032a45a67 Mon Sep 17 00:00:00 2001 From: reddevillg Date: Tue, 1 Sep 2026 19:06:52 +0800 Subject: [PATCH] feat: add linglong.lock for reproducible builds Introduce a project lock file that pins the resolved source hashes and the selected base/runtime references: - ll-builder build writes linglong.lock on the first build once sources are verified and dependencies are acquired, and reuses it on later builds; --locked requires an existing, complete lock and never rewrites it, while the new generate-lockfile/update-lockfile subcommands create or refresh the lock on demand - source entries record the normalized declaration together with the resolved hash (sha256 for file/archive/dsc sources, the commit for git sources); dependency entries record the requested declaration and the resolved reference/modules - the digest of file/archive/dsc sources is now optional: when omitted, the first build pins the hash of the fetched source and later builds verify against it Rework source fetching into per-kind handlers (git/file/archive/dsc) backed by a content-addressed cache under linglong/cache/sources/v1/. Cache entries are guarded by a per-entry file lock, published via atomic rename, and re-validated on every read; the fetch scripts move to an action-based interface (download/materialize/verify). As a result, --offline no longer implies --skip-fetch-source: it re-materializes sources from the validated cache and discards manual edits under linglong/sources/; use --skip-fetch-source to keep them. Build integration: - dependencies are pulled at the locked reference and modules; the module set now comes from the lock instead of best-effort mounting - when base is omitted from linglong.yaml it follows the base declared by the runtime layer metadata, and non-locked builds may rewrite the base entry of an existing lock when that metadata changes - build container overlay reuse is keyed on the locked dependency state (build_overlay.state), so dependency changes reset the overlays Signed-off-by: reddevillg --- api/schema/v1.json | 193 +- api/schema/v1.yaml | 115 +- apps/ll-builder/src/main.cpp | 158 +- .../reference/commands/ll-builder/build.md | 17 +- .../commands/ll-builder/generate-lockfile.md | 53 + .../commands/ll-builder/ll-builder.md | 12 +- .../commands/ll-builder/update-lockfile.md | 56 + .../en/guide/start/ll-builder-workflow.md | 23 +- .../reference/commands/ll-builder/build.md | 17 +- .../commands/ll-builder/generate-lockfile.md | 53 + .../commands/ll-builder/ll-builder.md | 12 +- .../commands/ll-builder/update-lockfile.md | 56 + docs/pages/guide/start/ll-builder-workflow.md | 23 +- external/ytj/src/ytj/ytj.hpp.in | 4 +- .../tests/ytj-test/src/ytj/to_yaml.test.cpp | 18 + libs/api/CMakeLists.txt | 4 + .../api/types/v1/BuilderProjectLock.hpp | 46 + .../types/v1/BuilderProjectLockDependency.hpp | 45 + .../BuilderProjectLockDependencyResolved.hpp | 46 + .../api/types/v1/BuilderSourceLockEntry.hpp | 45 + .../types/v1/BuilderSourceLockResolved.hpp | 44 + .../src/linglong/api/types/v1/Generators.hpp | 109 +- .../linglong/api/types/v1/LinglongAPIV1.hpp | 12 + libs/linglong/CMakeLists.txt | 6 + .../linglong/builder/builder_directory.cpp | 232 ++ .../src/linglong/builder/builder_directory.h | 100 + .../src/linglong/builder/linglong_builder.cpp | 1013 +++++--- .../src/linglong/builder/linglong_builder.h | 56 +- .../src/linglong/builder/project_lock.cpp | 580 +++++ .../src/linglong/builder/project_lock.h | 103 + .../src/linglong/builder/source_cache.cpp | 77 + .../src/linglong/builder/source_cache.h | 36 + .../src/linglong/builder/source_fetcher.cpp | 986 +++++++- .../src/linglong/builder/source_fetcher.h | 49 +- .../src/linglong/repo/ostree_repo.cpp | 38 +- .../src/linglong/runtime/run_context.cpp | 65 +- .../src/linglong/runtime/run_context.h | 11 +- libs/linglong/tests/ll-tests/CMakeLists.txt | 1 + .../builder/linglong_builder_test.cpp | 695 +++++- .../linglong/builder/project_lock_test.cpp | 1875 +++++++++++++++ .../linglong/builder/pull_dependency_test.cpp | 2059 ++++++++++++++++- .../linglong/builder/source_fetcher_test.cpp | 1162 ++++++++-- .../linglong/mocks/linglong_builder_mock.h | 9 + .../src/linglong/repo/ostree_repo_test.cpp | 84 + .../src/linglong/runtime/run_context_test.cpp | 162 ++ .../ll-tests/src/linglong/utils/file_test.cpp | 152 +- libs/utils/src/linglong/utils/file.cpp | 94 +- libs/utils/src/linglong/utils/file.h | 14 +- misc/libexec/linglong/fetch-archive-source | 65 +- misc/libexec/linglong/fetch-dsc-source | 82 +- misc/libexec/linglong/fetch-file-source | 37 +- misc/libexec/linglong/fetch-git-source | 51 +- 52 files changed, 10067 insertions(+), 988 deletions(-) create mode 100644 docs/pages/en/guide/reference/commands/ll-builder/generate-lockfile.md create mode 100644 docs/pages/en/guide/reference/commands/ll-builder/update-lockfile.md create mode 100644 docs/pages/guide/reference/commands/ll-builder/generate-lockfile.md create mode 100644 docs/pages/guide/reference/commands/ll-builder/update-lockfile.md create mode 100644 libs/api/src/linglong/api/types/v1/BuilderProjectLock.hpp create mode 100644 libs/api/src/linglong/api/types/v1/BuilderProjectLockDependency.hpp create mode 100644 libs/api/src/linglong/api/types/v1/BuilderProjectLockDependencyResolved.hpp create mode 100644 libs/api/src/linglong/api/types/v1/BuilderSourceLockEntry.hpp create mode 100644 libs/api/src/linglong/api/types/v1/BuilderSourceLockResolved.hpp create mode 100644 libs/linglong/src/linglong/builder/builder_directory.cpp create mode 100644 libs/linglong/src/linglong/builder/builder_directory.h create mode 100644 libs/linglong/src/linglong/builder/project_lock.cpp create mode 100644 libs/linglong/src/linglong/builder/project_lock.h create mode 100644 libs/linglong/src/linglong/builder/source_cache.cpp create mode 100644 libs/linglong/src/linglong/builder/source_cache.h create mode 100644 libs/linglong/tests/ll-tests/src/linglong/builder/project_lock_test.cpp diff --git a/api/schema/v1.json b/api/schema/v1.json index 599e65ec7..a2574a46e 100644 --- a/api/schema/v1.json +++ b/api/schema/v1.json @@ -287,6 +287,144 @@ } } }, + "BuilderSourceLockResolved": { + "title": "BuilderSourceLockResolved", + "description": "resolved identity of a locked builder source", + "type": "object", + "required": [ + "hash" + ], + "properties": { + "hash": { + "type": "string", + "description": "locked content identity: sha256: or git-commit:" + } + } + }, + "BuilderProjectSource": { + "title": "BuilderProjectSource", + "description": "items of sources of builder project", + "type": "object", + "required": [ + "kind" + ], + "properties": { + "kind": { + "type": "string", + "description": "kind of source" + }, + "url": { + "type": "string", + "description": "url of source" + }, + "digest": { + "type": "string", + "description": "digest of source" + }, + "name": { + "type": "string", + "description": "name of source" + }, + "commit": { + "type": "string", + "description": "commit of source" + }, + "version": { + "type": "string", + "description": "version of source" + }, + "submodules": { + "type": "boolean", + "description": "whether to checkout submodules" + } + } + }, + "BuilderSourceLockEntry": { + "title": "BuilderSourceLockEntry", + "description": "one source lock entry", + "type": "object", + "required": [ + "source", + "resolved" + ], + "properties": { + "source": { + "$ref": "#/$defs/BuilderProjectSource" + }, + "resolved": { + "$ref": "#/$defs/BuilderSourceLockResolved" + } + } + }, + "BuilderProjectLockDependency": { + "title": "BuilderProjectLockDependency", + "description": "requested and resolved builder dependency", + "type": "object", + "required": [ + "type", + "requested", + "resolved" + ], + "properties": { + "type": { + "type": "string" + }, + "requested": { + "type": "string" + }, + "resolved": { + "$ref": "#/$defs/BuilderProjectLockDependencyResolved" + } + } + }, + "BuilderProjectLockDependencyResolved": { + "title": "BuilderProjectLockDependencyResolved", + "description": "selected reference and modules of a builder dependency", + "type": "object", + "required": [ + "reference", + "modules" + ], + "properties": { + "reference": { + "type": "string", + "description": "locked channel, package ID and exact version; architecture preserves the declaration or uses unknown when unspecified" + }, + "modules": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "BuilderProjectLock": { + "title": "BuilderProjectLock", + "description": "content lock for builder project sources and dependencies", + "type": "object", + "required": [ + "version", + "sources", + "dependencies" + ], + "properties": { + "version": { + "type": "integer" + }, + "sources": { + "type": "array", + "items": { + "$ref": "#/$defs/BuilderSourceLockEntry" + } + }, + "dependencies": { + "type": "array", + "items": { + "$ref": "#/$defs/BuilderProjectLockDependency" + } + } + } + }, "BuilderProject": { "title": "BuilderProject", "description": "Linglong project build file.", @@ -397,42 +535,7 @@ "type": "array", "description": "sources of package", "items": { - "title": "BuilderProjectSource", - "description": "items of sources of builder project", - "type": "object", - "required": [ - "kind" - ], - "properties": { - "kind": { - "type": "string", - "description": "kind of source" - }, - "url": { - "type": "string", - "description": "url of source" - }, - "digest": { - "type": "string", - "description": "digest of source" - }, - "name": { - "type": "string", - "description": "name of source" - }, - "commit": { - "type": "string", - "description": "commit of source" - }, - "version": { - "type": "string", - "description": "version of source" - }, - "submodules": { - "type": "boolean", - "description": "whether to checkout submodules" - } - } + "$ref": "#/$defs/BuilderProjectSource" } }, "build": { @@ -1659,6 +1762,24 @@ "UABMetaInfo": { "$ref": "#/$defs/UABMetaInfo" }, + "BuilderSourceLockResolved": { + "$ref": "#/$defs/BuilderSourceLockResolved" + }, + "BuilderProjectSource": { + "$ref": "#/$defs/BuilderProjectSource" + }, + "BuilderSourceLockEntry": { + "$ref": "#/$defs/BuilderSourceLockEntry" + }, + "BuilderProjectLockDependency": { + "$ref": "#/$defs/BuilderProjectLockDependency" + }, + "BuilderProjectLockDependencyResolved": { + "$ref": "#/$defs/BuilderProjectLockDependencyResolved" + }, + "BuilderProjectLock": { + "$ref": "#/$defs/BuilderProjectLock" + }, "BuilderProject": { "$ref": "#/$defs/BuilderProject" }, diff --git a/api/schema/v1.yaml b/api/schema/v1.yaml index f20812c21..03353e2de 100644 --- a/api/schema/v1.yaml +++ b/api/schema/v1.yaml @@ -234,6 +234,93 @@ $defs: minified: description: Whether this layer file is minified or not. type: boolean + BuilderSourceLockResolved: + title: BuilderSourceLockResolved + description: resolved identity of a locked builder source + type: object + required: [hash] + properties: + hash: + type: string + description: "locked content identity: sha256: or git-commit:" + BuilderProjectSource: + title: BuilderProjectSource + description: items of sources of builder project + type: object + required: [kind] + properties: + kind: + type: string + description: kind of source + url: + type: string + description: url of source + digest: + type: string + description: digest of source + name: + type: string + description: name of source + commit: + type: string + description: commit of source + version: + type: string + description: version of source + submodules: + type: boolean + description: whether to checkout submodules + BuilderSourceLockEntry: + title: BuilderSourceLockEntry + description: one source lock entry + type: object + required: [source, resolved] + properties: + source: + $ref: '#/$defs/BuilderProjectSource' + resolved: + $ref: '#/$defs/BuilderSourceLockResolved' + BuilderProjectLockDependency: + title: BuilderProjectLockDependency + description: requested and resolved builder dependency + type: object + required: [type, requested, resolved] + properties: + type: + type: string + requested: + type: string + resolved: + $ref: '#/$defs/BuilderProjectLockDependencyResolved' + BuilderProjectLockDependencyResolved: + title: BuilderProjectLockDependencyResolved + description: selected reference and modules of a builder dependency + type: object + required: [reference, modules] + properties: + reference: + type: string + description: "locked channel, package ID and exact version; architecture preserves the declaration or uses unknown when unspecified" + modules: + type: array + items: + type: string + BuilderProjectLock: + title: BuilderProjectLock + description: content lock for builder project sources and dependencies + type: object + required: [version, sources, dependencies] + properties: + version: + type: integer + sources: + type: array + items: + $ref: '#/$defs/BuilderSourceLockEntry' + dependencies: + type: array + items: + $ref: '#/$defs/BuilderProjectLockDependency' BuilderProject: title: BuilderProject description: Linglong project build file. @@ -326,33 +413,7 @@ $defs: type: array description: sources of package items: - title: BuilderProjectSource - description: items of sources of builder project - type: object - required: - - kind - properties: - kind: - type: string - description: kind of source - url: - type: string - description: url of source - digest: - type: string - description: digest of source - name: - type: string - description: name of source - commit: - type: string - description: commit of source - version: - type: string - description: version of source - submodules: - type: boolean - description: whether to checkout submodules + $ref: '#/$defs/BuilderProjectSource' build: title: BuilderProjectBuildScript description: build script of builder project diff --git a/apps/ll-builder/src/main.cpp b/apps/ll-builder/src/main.cpp index 79ddaef17..0bde4e4e1 100644 --- a/apps/ll-builder/src/main.cpp +++ b/apps/ll-builder/src/main.cpp @@ -6,9 +6,11 @@ #include "command_options.h" #include "configure.h" +#include "linglong/builder/builder_directory.h" #include "linglong/builder/builder_script.h" #include "linglong/builder/config.h" #include "linglong/builder/linglong_builder.h" +#include "linglong/builder/project_lock.h" #include "linglong/cli/cli.h" #include "linglong/cli/cli_printer.h" #include "linglong/common/global/initialize.h" @@ -29,8 +31,8 @@ #include #include -#include +#include #include #include #include @@ -118,42 +120,6 @@ parseProjectConfig(const std::filesystem::path &filename) return project; } -linglong::utils::error::Result -getProjectYAMLPath(const std::filesystem::path &projectDir, const std::string &usePath) -{ - LINGLONG_TRACE("get project yaml path"); - - std::error_code ec; - if (!usePath.empty()) { - std::filesystem::path path = std::filesystem::canonical(usePath, ec); - if (ec) { - return LINGLONG_ERR( - fmt::format("invalid file path {} error: {}", usePath, ec.message())); - } - return path; - } - - std::filesystem::path path = projectDir - / ("linglong." + linglong::package::Architecture::currentCPUArchitecture().toString() - + ".yaml"); - if (std::filesystem::exists(path, ec)) { - return path; - } - if (ec) { - return LINGLONG_ERR(fmt::format("path {} error: {}", path, ec.message())); - } - - path = projectDir / "linglong.yaml"; - if (std::filesystem::exists(path, ec)) { - return path; - } - if (ec) { - return LINGLONG_ERR(fmt::format("path {} error: {}", path, ec.message())); - } - - return LINGLONG_ERR("project yaml file not found"); -} - int handleCreate(const CreateCommandOptions &options) { LogI("Handling create for project: {}", options.projectName); @@ -208,6 +174,18 @@ int handleCreate(const CreateCommandOptions &options) return 0; } +int handleProjectLock(linglong::builder::Builder &builder, linglong::builder::ProjectLockMode mode) +{ + auto result = builder.processProjectLock(mode); + if (!result) { + LogE("failed to process project lock: {}", result.error()); + return result.error().code(); + } + + LogI("Project lock processed successfully."); + return 0; +} + int handleBuild(linglong::builder::Builder &builder, const BuildCommandOptions &options) { LogI("Handling build command"); @@ -216,27 +194,15 @@ int handleBuild(linglong::builder::Builder &builder, const BuildCommandOptions & auto finalBuildOptions = options.builderSpecificOptions; - // offline means skip fetch source and pull dependency - if (options.buildOffline || cfg.offline) { - finalBuildOptions.skipFetchSource = true; + // Offline builds still prepare sources from the verified source cache. + if (options.buildOffline || cfg.offline.value_or(false)) { finalBuildOptions.skipPullDepend = true; + finalBuildOptions.offline = true; } builder.setBuildOptions(finalBuildOptions); - QStringList commandList; - if (!options.commands.empty()) { - for (const auto &command : options.commands) { - commandList.append(QString::fromStdString(command)); - } - } - - linglong::utils::error::Result ret; - if (!commandList.isEmpty()) { - ret = builder.build(commandList); - } else { - ret = builder.build(); - } + auto ret = builder.build(options.commands); if (!ret) { LogE("Build failed: {}", ret.error()); return ret.error().code(); @@ -393,7 +359,8 @@ int handleExtract(const ExtractCommandOptions &options) return 0; } -int handleClean(linglong::builder::Builder &builder, const CleanCommandOptions &options) +int handleClean(linglong::builder::Builder &builder, + [[maybe_unused]] const CleanCommandOptions &options) { auto result = builder.cleanBuildArtifacts(); if (!result) { @@ -521,16 +488,21 @@ You can report bugs to the linyaps team under this project: https://github.com/O _("Enter the container to execute command instead of building applications")); buildBuilder->add_flag("--offline", buildOpts.buildOffline, - _("Only use local files. This implies --skip-fetch-source and " - "--skip-pull-depend will be set")); + _("Only use verified cached sources and local dependencies")); + auto *lockedFlag = + buildBuilder->add_flag("--locked", + buildOpts.builderSpecificOptions.locked, + _("Require an existing up-to-date project lock file")); buildBuilder ->add_flag("--full-develop-module", buildOpts.builderSpecificOptions.fullDevelop, _("Build full develop packages, runtime requires")) ->group(hiddenGroup); - buildBuilder->add_flag("--skip-fetch-source", - buildOpts.builderSpecificOptions.skipFetchSource, - _("Skip fetch sources")); + buildBuilder + ->add_flag("--skip-fetch-source", + buildOpts.builderSpecificOptions.skipFetchSource, + _("Skip fetching sources and use existing sources as-is")) + ->excludes(lockedFlag); buildBuilder->add_flag("--skip-pull-depend", buildOpts.builderSpecificOptions.skipPullDepend, _("Skip pull dependency")); @@ -550,6 +522,19 @@ You can report bugs to the linyaps team under this project: https://github.com/O buildOpts.builderSpecificOptions.isolateNetWork, _("Build in an isolated network environment")); + auto generateLock = + commandParser.add_subcommand("generate-lockfile", + _("Generate a project lock file without building")); + generateLock->add_option("-f, --file", filePath, _("File path of the linglong.yaml")) + ->type_name("FILE") + ->check(CLI::ExistingFile); + + auto updateLock = + commandParser.add_subcommand("update-lockfile", _("Update the project lock file")); + updateLock->add_option("-f, --file", filePath, _("File path of the linglong.yaml")) + ->type_name("FILE") + ->check(CLI::ExistingFile); + // add builder run auto buildRun = commandParser.add_subcommand("run", _("Run built linyaps app")); buildRun->usage(_("Usage: ll-builder run [OPTIONS] [COMMAND...]")); @@ -853,17 +838,29 @@ You can report bugs to the linyaps team under this project: https://github.com/O return -1; } - auto canonicalYamlPath = getProjectYAMLPath(cwd, filePath); - if (canonicalYamlPath && canonicalYamlPath->string().rfind(cwd.string(), 0) != 0) { + if (buildClean->parsed()) { + linglong::builder::Builder builder(std::nullopt, + cwd, + **repo, + *containerBuilder, + *builderCfg); + return handleClean(builder, cleanOpts); + } + + auto projectYamlPath = linglong::builder::getProjectYamlPath(cwd, filePath); + if (projectYamlPath + && std::mismatch(cwd.begin(), cwd.end(), projectYamlPath->begin(), projectYamlPath->end()) + .first + != cwd.end()) { LogE("the project file {} is not under the current working directory {}", - canonicalYamlPath->string(), + projectYamlPath->string(), cwd.string()); return -1; } std::optional project; - if (canonicalYamlPath && std::filesystem::exists(*canonicalYamlPath, ec)) { - auto projectRet = parseProjectConfig(*canonicalYamlPath); + if (projectYamlPath && std::filesystem::exists(*projectYamlPath, ec)) { + auto projectRet = parseProjectConfig(*projectYamlPath); if (!projectRet) { LogE("{}", projectRet.error()); return -1; @@ -872,16 +869,29 @@ You can report bugs to the linyaps team under this project: https://github.com/O project = std::move(projectRet).value(); } - linglong::builder::Builder builder(std::move(project), - cwd, - **repo, - *containerBuilder, - *builderCfg); + linglong::builder::Builder builder(project, cwd, **repo, *containerBuilder, *builderCfg); + + if (project) { + builder.projectYamlFile = projectYamlPath->string(); + } if (buildExport->parsed()) { return handleExport(builder, exportOpts); } + if (!project) { + LogE("the project file is not found"); + return -1; + } + + if (generateLock->parsed()) { + return handleProjectLock(builder, linglong::builder::ProjectLockMode::Generate); + } + + if (updateLock->parsed()) { + return handleProjectLock(builder, linglong::builder::ProjectLockMode::Update); + } + if (buildPush->parsed()) { if (!pushModule.empty()) { pushOpts.pushModules = { pushModule }; @@ -891,16 +901,6 @@ You can report bugs to the linyaps team under this project: https://github.com/O return handlePush(builder, pushOpts); } - if (buildClean->parsed()) { - return handleClean(builder, cleanOpts); - } - - if (!canonicalYamlPath) { - LogE("the project file is not found"); - return -1; - } - - builder.projectYamlFile = std::move(canonicalYamlPath).value(); if (buildBuilder->parsed()) { return handleBuild(builder, buildOpts); } diff --git a/docs/pages/en/guide/reference/commands/ll-builder/build.md b/docs/pages/en/guide/reference/commands/ll-builder/build.md index 8f92c69a5..25f4d8f80 100644 --- a/docs/pages/en/guide/reference/commands/ll-builder/build.md +++ b/docs/pages/en/guide/reference/commands/ll-builder/build.md @@ -12,6 +12,16 @@ ll-builder-build - Build Linyaps project The `ll-builder build` command is used to build Linyaps applications. After the build is complete, the build content will be automatically committed to the local `ostree` cache. +The lock does not record the build architecture. Dependency references lock the channel, package ID, and exact version while preserving any explicit architecture in the original declaration. An unspecified architecture is recorded as `unknown` and uses the current build environment architecture, independently of the output target specified by `package.architecture`. A Base derived from Runtime follows the same rule using the Runtime's Base declaration. Dependencies without an explicit architecture can follow the build environment across architectures using the same lock. Those architectures must support the same locked sources, dependency versions, and module sets. A missing locked version or module for the selected architecture causes an error; another version is never selected automatically. + +Normal builds reuse entries in `linglong.lock` that match the current source declarations. Matching compares `kind` and `url` for all sources. For `file`, `archive`, and `dsc`, it also compares the normalized `digest`. For `git`, it also compares `commit` and `submodules`; omitted `submodules` defaults to enabled. The `commit` and `submodules` fields do not affect matching for non-Git sources. The `name` and `version` fields do not affect matching; output names come from the current project configuration. + +New or changed sources are downloaded again, even when content with the same digest is cached. A declared `digest` must match before a new lock entry is generated. Offline builds require matching source lock entries and valid cached content. With `--locked`, mismatched source declarations cause an error. + +For Git sources, `commit` accepts a branch name, tag, or commit SHA. The lock stores the resolved commit as `resolved.hash: git-commit:`. When the YAML declaration stays unchanged, builds use that locked SHA even if the remote branch or tag moves; run `ll-builder update-lockfile` to resolve it again. Changing the branch name or tag in YAML changes the source declaration: normal builds fetch it again, while `--locked` builds report an error. + +File changes in the build container persist between builds. The build environment resets automatically when the exact base/runtime references or module lists change; `ll-builder clean` also removes it. Build scripts must create the temporary data they need and must not rely on files left by a previous build. Each build runs the APT installation process for the current `buildext.apt.build_depends` list; changes to that list do not reset the environment. + ## OPTIONS **-h, --help** @@ -24,10 +34,13 @@ The `ll-builder build` command is used to build Linyaps applications. After the : Specify a project configuration file under the current working directory. If omitted, `linglong..yaml` is preferred when it exists; otherwise, `linglong.yaml` is used **--offline** -: Use only local files. This implies --skip-fetch-source and --skip-pull-depend +: Use only local source caches and dependencies, without fetching either remotely; sources are still prepared and verified + +**--locked** +: Require a project lock matching the declarations; cannot be used with --skip-fetch-source **--skip-fetch-source** -: Skip fetching source code +: Skip obtaining sources and use the existing `linglong/sources/` as-is; useful for locally modified sources. Project lock and dependency handling still follow the normal build rules **--skip-pull-depend** : Skip pulling dependencies diff --git a/docs/pages/en/guide/reference/commands/ll-builder/generate-lockfile.md b/docs/pages/en/guide/reference/commands/ll-builder/generate-lockfile.md new file mode 100644 index 000000000..1f1ce6655 --- /dev/null +++ b/docs/pages/en/guide/reference/commands/ll-builder/generate-lockfile.md @@ -0,0 +1,53 @@ +% ll-builder-generate-lockfile 1 + +## NAME + +ll-builder-generate-lockfile - Generate a build resource lock file + +## SYNOPSIS + +**ll-builder generate-lockfile** [*options*] + +## DESCRIPTION + +`ll-builder generate-lockfile` acquires the sources and Base/Runtime dependencies declared by the project, then records source hashes, Git commits, exact dependency references, and module lists. It writes a complete resource lock file after resource preparation succeeds. This command does not execute the project's build script. + +The corresponding lock file must not exist. If it already exists, the command fails; use `ll-builder update-lockfile` instead. Failed resource preparation does not produce a complete lock, but acquired content may remain in the cache for later use. + +The lock is stored beside the YAML entry file, with its extension replaced by `.lock`: `linglong.yaml` uses `linglong.lock`, and `config/app.yaml` uses `config/app.lock`. See [ll-builder(1)](./ll-builder.md) for file symbolic link and project locking rules. + +Commit the generated lock alongside the project configuration. Subsequent builds can use `ll-builder build --locked` to build with the locked resources. + +## OPTIONS + +**-h, --help** +: Print help information and exit + +**--help-all** +: Expand all help + +**-f, --file** _FILE_ +: Select a project YAML file within the current working directory, including its subdirectories. If omitted, use `linglong..yaml` when present, otherwise `linglong.yaml` + +## EXAMPLES + +Generate a resource lock for a project that has none, then build: + +```bash +ll-builder generate-lockfile +ll-builder build --locked +``` + +Generate `config/app.lock` for a configuration in a subdirectory: + +```bash +ll-builder generate-lockfile -f config/app.yaml +``` + +## SEE ALSO + +**[ll-builder(1)](./ll-builder.md)**, **[ll-builder-build(1)](./build.md)**, **[ll-builder-update-lockfile(1)](./update-lockfile.md)** + +## HISTORY + +Developed in 2026 by UnionTech Software Technology Co., Ltd. diff --git a/docs/pages/en/guide/reference/commands/ll-builder/ll-builder.md b/docs/pages/en/guide/reference/commands/ll-builder/ll-builder.md index d80a65ab9..ff9837b1a 100644 --- a/docs/pages/en/guide/reference/commands/ll-builder/ll-builder.md +++ b/docs/pages/en/guide/reference/commands/ll-builder/ll-builder.md @@ -17,6 +17,14 @@ Main features include: - Building applications in isolated containers - Complete application build, test, export, and publish workflow +Only one `build`, `clean`, `run`, `export`, `push`, `generate-lockfile`, or `update-lockfile` command may operate in a project working directory at a time. A competing command fails immediately without waiting. For example, while an application is running through `ll-builder run`, the same project cannot be built or cleaned. Selecting different configuration files with `-f` in the same working directory shares the project lock. Commands from different working directories also exclude each other when they use the same YAML entry directory. Projects whose working and YAML entry directories do not overlap may be used concurrently. + +The project lock uses `.ll-builder.lock` in both the working directory and the YAML entry directory. When these are the same directory, the lock is acquired once. It is released automatically when the command ends or its process exits. Keep this file out of version control. `clean` preserves it, and its presence does not mean the project is busy. Do not delete it while a command is running. Commit the resource lock file `linglong.lock` to version control. + +Commands that do not load a YAML file, such as `clean` or `export --ref` in a directory without a configuration file, only lock the working directory. + +When a YAML file is a symbolic link, its contents are read from the target, but the resource lock is stored beside the link, with the link filename’s extension replaced by `.lock`. For example, `linglong.yaml -> /shared/app.yaml` uses `linglong.lock` beside the entry file. Automatic discovery and explicit `-f` selection follow the same rule; symbolic links in parent directories are resolved first. + ## OPTIONS **-h, --help** @@ -34,6 +42,8 @@ Main features include: | ------- | ------------------------------------- | --------------------------------------------- | | create | [ll-builder-create(1)](./create.md) | Create Linyaps build template | | build | [ll-builder-build(1)](./build.md) | Build Linyaps project | +| generate-lockfile | [ll-builder-generate-lockfile(1)](./generate-lockfile.md) | Generate a build resource lock without compiling | +| update-lockfile | [ll-builder-update-lockfile(1)](./update-lockfile.md) | Resolve resources again and update an existing lock | | run | [ll-builder-run(1)](./run.md) | Run the built application | | list | [ll-builder-list(1)](./list.md) | List built applications | | remove | [ll-builder-remove(1)](./remove.md) | Remove built applications | @@ -45,7 +55,7 @@ Main features include: ## SEE ALSO -**[ll-builder-create(1)](./create.md)**, **[ll-builder-build(1)](./build.md)**, **[ll-builder-run(1)](./run.md)**, **[ll-builder-export(1)](./export.md)**, **[ll-builder-push(1)](./push.md)** +**[ll-builder-generate-lockfile(1)](./generate-lockfile.md)**, **[ll-builder-update-lockfile(1)](./update-lockfile.md)**, **[ll-builder-create(1)](./create.md)**, **[ll-builder-build(1)](./build.md)**, **[ll-builder-run(1)](./run.md)**, **[ll-builder-export(1)](./export.md)**, **[ll-builder-push(1)](./push.md)** ## HISTORY diff --git a/docs/pages/en/guide/reference/commands/ll-builder/update-lockfile.md b/docs/pages/en/guide/reference/commands/ll-builder/update-lockfile.md new file mode 100644 index 000000000..e07e48de8 --- /dev/null +++ b/docs/pages/en/guide/reference/commands/ll-builder/update-lockfile.md @@ -0,0 +1,56 @@ +% ll-builder-update-lockfile 1 + +## NAME + +ll-builder-update-lockfile - Update a build resource lock file + +## SYNOPSIS + +**ll-builder update-lockfile** [*options*] + +## DESCRIPTION + +`ll-builder update-lockfile` resolves all sources and Base/Runtime dependencies from the current project YAML again, then replaces the corresponding resource lock file in a single step after resource preparation succeeds. It resolves Git branches, tags, and dependency version requirements again even when their declarations have not changed. Any YAML `digest` remains a mandatory verification condition. This command does not execute the project's build script. + +The corresponding lock file must already exist. If it does not, the command fails; use `ll-builder generate-lockfile` first. A failed update preserves the old lock, but acquired content may remain in the cache for later use. + +The lock is stored beside the YAML entry file, with its extension replaced by `.lock`. See [ll-builder(1)](./ll-builder.md) for file symbolic link and project locking rules. + +The lock does not record the build architecture. Dependency references preserve any explicit architecture in the original declaration; an unspecified architecture is recorded as `unknown` and uses the current build environment architecture, independently of the output target specified by `package.architecture`. A Base derived from Runtime follows the same rule using the Runtime's Base declaration. Dependencies without an explicit architecture can reuse the same lock after changing build architectures, provided the locked dependency versions and modules are available for the selected architecture. + +Ordinary builds reuse locked resources whose declarations have not changed. Use this command to refresh resources explicitly, then review and commit the lock changes before rebuilding. + +## OPTIONS + +**-h, --help** +: Print help information and exit + +**--help-all** +: Expand all help + +**-f, --file** _FILE_ +: Select a project YAML file within the current working directory, including its subdirectories. If omitted, use `linglong..yaml` when present, otherwise `linglong.yaml` + +## EXAMPLES + +Update an existing resource lock, review its changes, then build: + +```bash +ll-builder update-lockfile +git diff -- linglong.lock +ll-builder build --locked +``` + +Update `config/app.lock` for a configuration in a subdirectory: + +```bash +ll-builder update-lockfile -f config/app.yaml +``` + +## SEE ALSO + +**[ll-builder(1)](./ll-builder.md)**, **[ll-builder-build(1)](./build.md)**, **[ll-builder-generate-lockfile(1)](./generate-lockfile.md)** + +## HISTORY + +Developed in 2026 by UnionTech Software Technology Co., Ltd. diff --git a/docs/pages/en/guide/start/ll-builder-workflow.md b/docs/pages/en/guide/start/ll-builder-workflow.md index a6f719b5d..094f4b157 100644 --- a/docs/pages/en/guide/start/ll-builder-workflow.md +++ b/docs/pages/en/guide/start/ll-builder-workflow.md @@ -104,6 +104,12 @@ See [Repository Management](../publishing/repositories.md) for repository config ## How ll-builder Works +Only one `build`, `clean`, `run`, `export`, `push`, `generate-lockfile`, or `update-lockfile` command may run in a project working directory at a time; a competing command fails immediately. `run` holds the lock until the application exits, so the same project cannot be built or cleaned while it is running. Different configuration files selected with `-f` in the same working directory share the project lock. Commands from different working directories also exclude each other when they use the same YAML entry directory; projects whose working and YAML entry directories do not overlap may run concurrently. Keep each working and YAML entry directory's `.ll-builder.lock` out of version control and do not delete it while a command is running. The lock is released automatically on process exit, and `clean` preserves the file. + +Commands that do not load a YAML file, such as `clean` or `export --ref` in a directory without a configuration file, only lock the working directory. + +When a YAML file is a symbolic link, its contents are read from the target, but the resource lock is stored beside the link, with the link filename’s extension replaced by `.lock`. For example, `linglong.yaml -> /shared/app.yaml` uses `linglong.lock` beside the entry file. Automatic discovery and explicit `-f` selection follow the same rule; symbolic links in parent directories are resolved first. + You can think of `ll-builder build` as an automated assembly line: `linglong.yaml` is the assembly specification, source code is the raw material, Base and Runtime provide a consistent work environment, and the resulting artifacts are organized and stored in the local repository. A complete build proceeds through these stages: @@ -121,6 +127,16 @@ Read configuration → Prepare sources and dependencies → Create isolated envi `ll-builder` downloads or expands sources according to `sources` and prepares them in `linglong/sources/`. The corresponding path in the build container is `/project/linglong/sources/`. Downloaded content can be reused from cache, but a normal build prepares the source directory again each time so files modified by a previous build cannot affect the result. Obtaining sources in a separate step instead of in the build instructions makes the build reproducible, traceable, and auditable. +Source output names must be unique, and the same fetch request cannot be declared more than once, even with different `name` or `version` values. Within the same source kind, Git requests are compared by `url`, `commit`, and `submodules`; other kinds are compared by `url` and `digest`. The `commit` and `submodules` fields are ignored for non-Git sources. To use the same resource in multiple directories, declare it once and copy it in the build script. + +The first build resolves sources, Base, and Runtime and writes `linglong.lock` once sources have been verified and dependencies have been acquired. Commit this file to version control; later builds verify file sources with SHA-256 and prepare Git sources at their locked commits. A normal build updates only resources whose declarations changed and keeps the locked versions of unchanged resources. Use `ll-builder generate-lockfile` to create a lock explicitly, `ll-builder update-lockfile` to refresh resources without changing their declarations, and `ll-builder build --locked` to require an existing, complete lock that matches the declarations. + +The lock has a `dependencies` list. Each Base and Runtime entry records its original declaration, selected locked reference, and available module names. Modules may include `binary`, `develop`, and other names. Normal builds acquire unchanged dependencies at their locked references and resolve only dependencies whose declarations changed. `--locked` builds always use the lock without rewriting it. A file source whose content differs from the locked hash fails verification until `update-lockfile` is run explicitly. Git sources are prepared at their locked commits, so changes to cached working trees do not enter the build sources. If Git cache verification fails, online builds fetch the locked sources again and offline builds fail. Dependency content is checked by the repository's other mechanisms. Updates replace the old file once all resources are prepared. The `buildext.apt.build_depends` overlay is reset when the actual full reference (including architecture) or module list changes. APT runs on that overlay for every build. + +Version 1 of the lock includes sources and `dependencies`; each dependency uses `type: base` or `type: runtime`. Dependency references lock the channel, package ID, and exact version while preserving any explicit architecture in the original declaration. An unspecified architecture is recorded as `unknown`, such as `main:org.example.Base/1.0.0.0/unknown`, and uses the current build environment architecture, independently of the output target specified by `package.architecture`. A Base derived from Runtime follows the same rule using the Runtime's Base declaration. Dependencies without an explicit architecture can reuse the same lock across build architectures without an update solely for an architecture change. All architectures must use the same locked sources, dependency versions, and module sets. Missing locked dependencies for the selected architecture cause an error rather than an automatic upgrade or fallback. + +`--skip-fetch-source` only skips obtaining sources and uses the existing `linglong/sources/` as-is, which supports locally modified sources. Reading the project lock and resolving and locking Base and Runtime still follow the normal build rules. It cannot be combined with `--locked`; use `--locked --offline` to verify cached sources against the lock using only local caches. + It then prepares `base` and `runtime` for the build environment. ### Create the Isolated Build Environment @@ -129,7 +145,7 @@ Before compiling, `ll-builder`: 1. Cleans temporary directories from the previous build, such as `linglong/output/`. The `linglong` directory is an internal `ll-builder` work directory and must not be used to store other files. 2. Combines the Base, Runtime, project directory, sources, and output directory into the build environment, and creates temporary writable layers for the Base and Runtime so the original dependencies are not modified. -3. Installs build-time dependencies such as `buildext.apt.buildDepends`. +3. Installs build-time dependencies such as `buildext.apt.build_depends`. The environment is a container isolated from the host, but it can access the current project and prepared sources. Two commonly used variables are available inside it: @@ -161,8 +177,9 @@ The following options alter build inputs or how stages are executed. `-h, --help | Option | Affected stage | Effect | | --- | --- | --- | | `-f, --file ` | Read configuration | Use the specified file instead of an automatically discovered `linglong..yaml` or `linglong.yaml`. The file must be in the current project directory or one of its subdirectories | -| `--offline` | Obtain sources and dependencies | Do not fetch sources or pull Base and Runtime from remote repositories. All required sources and dependencies must already exist locally. `offline: true` in `linglong.yaml` has the same effect | -| `--skip-fetch-source` | Obtain sources | Preserve and use the existing `linglong/sources/` without cleaning or fetching `sources` | +| `--offline` | Obtain sources and dependencies | Use only source caches verified by the lock and do not pull Base or Runtime remotely. All required content must already exist locally. `offline: true` in the ll-builder configuration file (default: `~/.config/linglong/builder/config.yaml`) has the same effect | +| `--locked` | Obtain sources and dependencies | Require an existing project lock matching the source, Base, and Runtime declarations; never update the lock during the build | +| `--skip-fetch-source` | Obtain sources | Skip obtaining sources and use the existing `linglong/sources/` as-is; project lock and dependency handling still follow the normal build rules; cannot be combined with `--locked` | | `--skip-pull-depend` | Obtain dependencies | Do not pull new content from remotes, but still resolve and merge locally available Base and Runtime modules. The build fails if a dependency is missing locally | | `--skip-run-container` | Run build | Prepare the configuration, sources, and dependencies, but do not clean output, create the build container, or run the build script. The command then exits without organizing or checking output | | `--skip-commit-output` | Save artifacts | Run the container build normally, but do not generate modules, desktop-integration content, or application configuration, and do not save to the local repository. The runtime check is also skipped | diff --git a/docs/pages/guide/reference/commands/ll-builder/build.md b/docs/pages/guide/reference/commands/ll-builder/build.md index 38bbea84e..f18d9f7c4 100644 --- a/docs/pages/guide/reference/commands/ll-builder/build.md +++ b/docs/pages/guide/reference/commands/ll-builder/build.md @@ -12,6 +12,16 @@ ll-builder-build - 构建如意玲珑项目 `ll-builder build` 命令用来构建如意玲珑应用。构建完成后，构建内容将自动提交到本地 `ostree` 缓存中。 +lock 不记录构建架构。依赖引用锁定 channel、ID 和精确版本，并保留原声明中的显式架构；未指定架构时记录 `unknown`，构建时使用当前构建环境架构，与 `package.architecture` 指定的输出目标架构独立。Base 由 Runtime 推导时，同样遵循 Runtime 的 Base 声明。未指定架构的依赖可以随构建环境切换架构，复用同一份 lock；各架构必须能够使用同一套锁定源码、依赖版本和模块集合。缺少所选架构的锁定版本或模块时构建报错，不会自动选择其他版本。 + +普通构建优先复用 `linglong.lock` 中与当前源码声明匹配的条目。所有源码比较 `kind` 和 `url`；`file`、`archive` 和 `dsc` 还比较规范化后的 `digest`；`git` 还比较 `commit` 和 `submodules`，未指定 `submodules` 时按默认启用处理。非 Git 源码的 `commit` 和 `submodules` 不参与匹配。`name` 和 `version` 不参与匹配，输出名称使用当前项目配置。 + +新增或声明变化的源码会重新下载，即使缓存中已有相同摘要的内容；配置了 `digest` 时必须校验通过，之后生成新的锁定条目。离线构建要求源码已有匹配的锁定条目和有效缓存。使用 `--locked` 时，源码声明不匹配会直接报错。 + +Git 源码的 `commit` 可以填写分支名、标签或提交 SHA。lock 使用 `resolved.hash: git-commit:` 记录确定的提交。YAML 声明不变时，即使远端分支或标签已经移动，构建仍使用锁定的 SHA；执行 `ll-builder update-lockfile` 才会重新解析该分支或标签。修改 YAML 中的分支名或标签则属于声明变化，普通构建会重新获取，`--locked` 构建会报错。 + +构建容器中的文件修改会跨构建保留。当 base/runtime 的实际完整引用（包括架构）或模块列表变化时，构建环境会自动重置；执行 `ll-builder clean` 也会清除它。构建脚本应自行生成所需的临时数据，不应依赖上一次构建留下的文件。每次构建都会根据当前 `buildext.apt.build_depends` 执行 APT 安装流程，该列表变化不会触发环境重置。 + ## OPTIONS **-h, --help** @@ -24,10 +34,13 @@ ll-builder-build - 构建如意玲珑项目 : 指定当前工作目录下的项目配置文件。未指定时优先使用 `linglong.<当前架构>.yaml`，不存在时使用 `linglong.yaml` **--offline** -: 仅使用本地文件。这意味着将设置 --skip-fetch-source 和 --skip-pull-depend +: 仅使用本地源码缓存和依赖，不从远程获取它们；仍会准备和校验源码 + +**--locked** +: 要求项目 lock 存在且与项目声明一致；不能与 --skip-fetch-source 同时使用 **--skip-fetch-source** -: 跳过获取源代码 +: 跳过源码获取，原样使用现有的 `linglong/sources/`，适合构建手动修改的源码；项目 lock 和依赖仍按普通构建规则处理 **--skip-pull-depend** : 跳过拉取依赖项 diff --git a/docs/pages/guide/reference/commands/ll-builder/generate-lockfile.md b/docs/pages/guide/reference/commands/ll-builder/generate-lockfile.md new file mode 100644 index 000000000..f94107f17 --- /dev/null +++ b/docs/pages/guide/reference/commands/ll-builder/generate-lockfile.md @@ -0,0 +1,53 @@ +% ll-builder-generate-lockfile 1 + +## NAME + +ll-builder-generate-lockfile - 生成构建资源 lock 文件 + +## SYNOPSIS + +**ll-builder generate-lockfile** [*options*] + +## DESCRIPTION + +`ll-builder generate-lockfile` 获取项目声明的源码和 Base/Runtime 依赖，记录源码摘要、Git commit、依赖精确引用及模块列表，在资源准备成功后生成完整的资源 lock 文件。此命令不执行项目构建脚本。 + +对应的 lock 文件必须不存在；已存在时命令报错，应使用 `ll-builder update-lockfile` 更新。资源准备失败时不会生成完整 lock，但已获取的内容可以留在缓存中供后续使用。 + +lock 位于 YAML 入口旁，文件名由 YAML 的扩展名替换为 `.lock`，例如 `linglong.yaml` 对应 `linglong.lock`，`config/app.yaml` 对应 `config/app.lock`。文件软链接和项目互斥锁的规则见 [ll-builder(1)](./ll-builder.md)。 + +应将生成的 lock 与项目配置一起提交到版本控制，后续可使用 `ll-builder build --locked` 按锁定资源构建。 + +## OPTIONS + +**-h, --help** +: 打印帮助信息并退出 + +**--help-all** +: 展开所有帮助 + +**-f, --file** _FILE_ +: 指定当前工作目录内的项目 YAML 文件，也可以位于其子目录。未指定时优先使用 `linglong.<当前构建架构>.yaml`，不存在时使用 `linglong.yaml` + +## EXAMPLES + +在项目尚无 lock 时生成资源 lock，然后构建： + +```bash +ll-builder generate-lockfile +ll-builder build --locked +``` + +为子目录中的配置生成对应的 `config/app.lock`： + +```bash +ll-builder generate-lockfile -f config/app.yaml +``` + +## SEE ALSO + +**[ll-builder(1)](./ll-builder.md)**, **[ll-builder-build(1)](./build.md)**, **[ll-builder-update-lockfile(1)](./update-lockfile.md)** + +## HISTORY + +2026年，由 UnionTech Software Technology Co., Ltd. 开发 diff --git a/docs/pages/guide/reference/commands/ll-builder/ll-builder.md b/docs/pages/guide/reference/commands/ll-builder/ll-builder.md index 2662900f5..775d65741 100644 --- a/docs/pages/guide/reference/commands/ll-builder/ll-builder.md +++ b/docs/pages/guide/reference/commands/ll-builder/ll-builder.md @@ -17,6 +17,14 @@ ll-builder 是为应用开发者提供的一款构建如意玲珑应用的工具 - 在独立容器内构建应用 - 完整的应用构建、测试、导出和发布流程 +同一项目工作目录同时只允许执行一个 `build`、`clean`、`run`、`export`、`push`、`generate-lockfile` 或 `update-lockfile` 命令。另一个命令会立即报错，不等待；例如，应用通过 `ll-builder run` 运行期间不能构建或清理同一项目。使用 `-f` 选择同工作目录下的不同配置仍共用项目锁；从不同工作目录使用同一 YAML 入口目录时也会互斥。工作目录和 YAML 入口目录均不重叠的项目可以并行操作。 + +项目锁使用工作目录和 YAML 入口目录各自的 `.ll-builder.lock`；两个目录相同时只获取一次锁。命令结束或进程退出后自动释放。该文件不应提交到版本控制，`clean` 会保留它；文件存在不代表项目正在被占用，不要在命令执行期间删除它。资源锁定文件 `linglong.lock` 应提交到版本控制。 + +没有加载 YAML 的命令（例如 `clean`，或目录中没有配置文件时的 `export --ref`）只锁工作目录。 + +YAML 为文件软链接时，读取链接目标的内容，但资源 lock 写在软链接所在目录，并使用软链接的文件名替换扩展名。例如 `linglong.yaml -> /shared/app.yaml` 对应入口旁的 `linglong.lock`。自动查找与显式 `-f` 使用相同规则；父目录的软链接会先解析为实际目录。 + ## OPTIONS **-h, --help** @@ -34,6 +42,8 @@ ll-builder 是为应用开发者提供的一款构建如意玲珑应用的工具 | ------- | ------------------------------------- | --------------------------------- | | create | [ll-builder-create(1)](./create.md) | 创建如意玲珑构建模板 | | build | [ll-builder-build(1)](./build.md) | 构建如意玲珑项目 | +| generate-lockfile | [ll-builder-generate-lockfile(1)](./generate-lockfile.md) | 获取资源并生成 lock，不执行项目构建脚本 | +| update-lockfile | [ll-builder-update-lockfile(1)](./update-lockfile.md) | 重新解析资源并更新已有 lock | | run | [ll-builder-run(1)](./run.md) | 运行构建好的应用程序 | | list | [ll-builder-list(1)](./list.md) | 列出已构建的应用程序 | | remove | [ll-builder-remove(1)](./remove.md) | 删除已构建的应用程序 | @@ -45,7 +55,7 @@ ll-builder 是为应用开发者提供的一款构建如意玲珑应用的工具 ## SEE ALSO -**[ll-builder-create(1)](./create.md)**, **[ll-builder-build(1)](./build.md)**, **[ll-builder-run(1)](./run.md)**, **[ll-builder-export(1)](./export.md)**, **[ll-builder-push(1)](./push.md)** +**[ll-builder-generate-lockfile(1)](./generate-lockfile.md)**, **[ll-builder-update-lockfile(1)](./update-lockfile.md)**, **[ll-builder-create(1)](./create.md)**, **[ll-builder-build(1)](./build.md)**, **[ll-builder-run(1)](./run.md)**, **[ll-builder-export(1)](./export.md)**, **[ll-builder-push(1)](./push.md)** ## HISTORY diff --git a/docs/pages/guide/reference/commands/ll-builder/update-lockfile.md b/docs/pages/guide/reference/commands/ll-builder/update-lockfile.md new file mode 100644 index 000000000..0ff8a8c7f --- /dev/null +++ b/docs/pages/guide/reference/commands/ll-builder/update-lockfile.md @@ -0,0 +1,56 @@ +% ll-builder-update-lockfile 1 + +## NAME + +ll-builder-update-lockfile - 更新构建资源 lock 文件 + +## SYNOPSIS + +**ll-builder update-lockfile** [*options*] + +## DESCRIPTION + +`ll-builder update-lockfile` 根据当前项目 YAML 重新解析全部源码和 Base/Runtime 依赖，在资源准备成功后一次性替换对应的资源 lock 文件。即使资源声明没有变化，也会重新解析 Git 分支、标签和依赖版本要求；YAML 中的 `digest` 仍是必须满足的校验条件。此命令不执行项目构建脚本。 + +对应的 lock 文件必须已存在；不存在时命令报错，应先使用 `ll-builder generate-lockfile`。更新失败时保留旧 lock，但已获取的内容可以留在缓存中供后续使用。 + +lock 位于 YAML 入口旁，文件名由 YAML 的扩展名替换为 `.lock`。文件软链接和项目互斥锁的规则见 [ll-builder(1)](./ll-builder.md)。 + +lock 不记录构建架构。依赖引用保留原声明中的显式架构，未指定时记录 `unknown`，构建时使用当前构建环境架构，与 `package.architecture` 指定的输出目标架构独立。从 Runtime 推导的 Base 同样按 Runtime 的 Base 声明处理。未指定架构的依赖可以在更换构建架构后复用同一份 lock；所选架构必须提供锁定的依赖版本和模块。 + +普通构建会复用声明未变化的锁定资源。需要主动刷新资源时使用此命令，并审阅、提交 lock 的变化后重新构建。 + +## OPTIONS + +**-h, --help** +: 打印帮助信息并退出 + +**--help-all** +: 展开所有帮助 + +**-f, --file** _FILE_ +: 指定当前工作目录内的项目 YAML 文件，也可以位于其子目录。未指定时优先使用 `linglong.<当前构建架构>.yaml`，不存在时使用 `linglong.yaml` + +## EXAMPLES + +更新已有资源 lock，审阅变化后构建： + +```bash +ll-builder update-lockfile +git diff -- linglong.lock +ll-builder build --locked +``` + +更新子目录中的配置对应的 `config/app.lock`： + +```bash +ll-builder update-lockfile -f config/app.yaml +``` + +## SEE ALSO + +**[ll-builder(1)](./ll-builder.md)**, **[ll-builder-build(1)](./build.md)**, **[ll-builder-generate-lockfile(1)](./generate-lockfile.md)** + +## HISTORY + +2026年，由 UnionTech Software Technology Co., Ltd. 开发 diff --git a/docs/pages/guide/start/ll-builder-workflow.md b/docs/pages/guide/start/ll-builder-workflow.md index 315274958..b8113367d 100644 --- a/docs/pages/guide/start/ll-builder-workflow.md +++ b/docs/pages/guide/start/ll-builder-workflow.md @@ -103,6 +103,12 @@ ll-builder push ## 理解 ll-builder 工作原理 +同一项目工作目录同时只允许一个 `build`、`clean`、`run`、`export`、`push`、`generate-lockfile` 或 `update-lockfile` 命令执行，冲突时立即报错。`run` 持锁到应用退出，因此运行期间不能再次构建或清理该项目。同工作目录下通过 `-f` 选择不同配置仍共用项目锁；从不同工作目录使用同一 YAML 入口目录也会互斥。工作目录和 YAML 入口目录均不重叠时可以并行操作。工作目录和 YAML 入口目录各自的 `.ll-builder.lock` 不应提交到版本控制，也不要在命令执行期间删除它；命令退出后锁自动释放，`clean` 保留该文件。 + +没有加载 YAML 的命令（例如 `clean`，或目录中没有配置文件时的 `export --ref`）只锁工作目录。 + +YAML 为文件软链接时，读取链接目标的内容，但资源 lock 写在软链接所在目录，并使用软链接的文件名替换扩展名。例如 `linglong.yaml -> /shared/app.yaml` 对应入口旁的 `linglong.lock`。自动查找与显式 `-f` 使用相同规则；父目录的软链接会先解析为实际目录。 + 可以把 `ll-builder build` 理解为一条自动化装配线：`linglong.yaml` 是装配说明，源码是待加工的材料，Base 和 Runtime 是统一的工作环境，最终产物则会被整理后存入本地仓库。 一次完整构建会依次经过以下阶段： @@ -119,6 +125,16 @@ ll-builder push `ll-builder` 根据 `sources` 下载或展开源码，并把它们准备到 `linglong/sources/`。构建容器中对应的路径是 `/project/linglong/sources/`。下载过的内容可以复用缓存，但每次正常构建都会重新准备源码目录，避免上一次构建修改过的文件影响结果。将获取源码放在单独的步骤而不是放到构建指令中，是为了使得构建过程可重复/可追溯/可审计。 +源码的输出名字不能重复，也不能重复声明相同的获取请求，即使 `name` 或 `version` 不同。同一种源码类型中，Git 按 `url`、`commit` 和 `submodules` 判断请求是否相同，其他类型按 `url` 和 `digest` 判断；非 Git 源码的 `commit`、`submodules` 不参与判断。需要在多个目录使用同一资源时，声明一次，再在构建脚本中复制。 + +首次构建会解析 `sources`、Base 和 Runtime，源码校验及依赖获取成功后，才一次性生成 `linglong.lock`。应将 lock 文件提交到版本控制；后续构建会按 SHA-256 校验文件类源码，并按锁定的 commit 准备 Git 源码。普通构建在资源声明变化时只重新解析变化的资源并更新 lock，未变化的资源继续使用已锁定版本。可用 `ll-builder generate-lockfile` 显式创建 lock；要在声明未变化时主动刷新资源，可使用 `ll-builder update-lockfile`。`ll-builder build --locked` 要求 lock 已存在、完整且与声明一致。 + +lock 的 `dependencies` 列表为 Base 和 Runtime 记录原始声明、选定的锁定引用以及可用模块名。模块可以是 `binary`、`develop` 或其他名称。普通构建会补齐架构后按完整引用获取声明未变化的依赖，不会主动升级；声明变化时只重新解析相应依赖。`--locked` 构建始终按 lock 获取资源，不会改写 lock。同一声明对应的文件类源码内容若与 lock 中的哈希不符，构建会失败，需要显式运行 `update-lockfile`；Git 源码会按锁定 commit 重新准备，缓存工作区中的修改不会进入构建源码。Git 缓存校验失败时，在线构建会重新获取锁定源码，离线构建会失败。依赖内容的正确性由仓库的其他机制保证。更新会在所有资源准备成功后一次性替换旧文件。`buildext.apt.build_depends` 的 overlay 在实际完整引用（包括架构）或模块列表变化时重置，每次构建都会在该 overlay 上运行 APT。 + +版本 1 的 lock 包含源码和 `dependencies`；每项依赖通过 `type: base` 或 `type: runtime` 区分。依赖引用锁定 channel、ID 和精确版本，并保留原声明中的显式架构。未指定架构时记录 `unknown`，例如 `main:org.example.Base/1.0.0.0/unknown`，构建时使用当前构建环境架构，与 `package.architecture` 指定的输出目标架构独立。从 Runtime 推导的 Base 同样遵循 Runtime 的 Base 声明。未指定架构的依赖可以在不同构建架构上复用同一份 lock，无需仅因更换构建架构而更新。各架构需使用相同的锁定源码、依赖版本和模块集合；缺少所选架构的锁定依赖时构建报错，不会自动升级或回退。 + +`--skip-fetch-source` 只跳过获取源码的步骤，原样使用现有的 `linglong/sources/`，适合构建手动修改的源码。项目 lock 的读取、Base 和 Runtime 的解析及锁定仍按普通构建规则处理。它不能与 `--locked` 同时使用；需要只使用本地缓存并按 lock 校验源码时，使用 `--locked --offline`。 + 随后会准备 `base` 和 `runtime`，用于创建构建环境。 ### 创建隔离的构建环境 @@ -127,7 +143,7 @@ ll-builder push 1. 清理上一次构建的临时目录，比如 `linglong/output/`。需要注意 `linglong` 目录是 ll-builder 内部使用工作目录，用户不应该在此目录存放其他文件。 2. 将 Base、Runtime、项目目录、源码和输出目录组合成构建环境，并为 Base 和 Runtime 创建临时可写层，避免修改原始依赖。 -3. 处理 `buildext.apt.buildDepends` 等构建期依赖的安装。 +3. 处理 `buildext.apt.build_depends` 等构建期依赖的安装。 这个环境是与宿主机相互隔离的容器，但仍能访问当前项目和已经准备好的源码。容器内会提供两个常用变量： @@ -159,8 +175,9 @@ TRIPLET=<当前 CPU 架构的 GNU triplet> | 参数 | 影响阶段 | 对构建过程的影响 | | --- | --- | --- | | `-f, --file ` | 读取配置 | 使用指定配置代替自动查找的 `linglong..yaml` 或 `linglong.yaml`。文件必须位于当前项目目录或其子目录中 | -| `--offline` | 获取源码和依赖 | 不重新获取源码，也不从远程拉取 Base、Runtime；所需源码和依赖必须已经存在于本地。`linglong.yaml` 中的 `offline: true` 作用相同 | -| `--skip-fetch-source` | 获取源码 | 保留并使用现有的 `linglong/sources/`，不会清理或重新获取 `sources` | +| `--offline` | 获取源码和依赖 | 仅使用经过 lock 校验的源码缓存，也不从远程拉取 Base、Runtime；所需内容必须已经存在于本地。ll-builder 配置文件（默认 `~/.config/linglong/builder/config.yaml`）中的 `offline: true` 作用相同 | +| `--locked` | 获取源码和依赖 | 要求项目 lock 已存在且 `sources`、Base、Runtime 声明一致；构建期间不会更新 lock | +| `--skip-fetch-source` | 获取源码 | 跳过源码获取，原样使用现有的 `linglong/sources/`；项目 lock 和依赖仍按普通构建规则处理，不能与 `--locked` 同时使用 | | `--skip-pull-depend` | 获取依赖 | 不从远程拉取新内容，但仍会解析和合并本地已有的 Base、Runtime 模块；本地缺少依赖时构建失败 | | `--skip-run-container` | 执行构建 | 仍会完成配置、源码和依赖准备，但不会清理输出、创建构建容器或执行构建脚本；命令随后结束，不再整理和检查输出 | | `--skip-commit-output` | 保存产物 | 正常执行容器构建，但不生成模块、桌面集成内容或应用配置，也不保存到本地仓库；随后不执行运行检查 | diff --git a/external/ytj/src/ytj/ytj.hpp.in b/external/ytj/src/ytj/ytj.hpp.in index 344b492b9..f8dbc7045 100644 --- a/external/ytj/src/ytj/ytj.hpp.in +++ b/external/ytj/src/ytj/ytj.hpp.in @@ -89,7 +89,7 @@ inline YAML::Node to_yaml(const nlohmann::json &json) } if (json.is_object()) { - YAML::Node object; + YAML::Node object(YAML::NodeType::Map); for (auto it = json.begin(); it != json.end(); it++) { object[it.key()] = to_yaml(it.value()); } @@ -97,7 +97,7 @@ inline YAML::Node to_yaml(const nlohmann::json &json) } if (json.is_array()) { - YAML::Node array; + YAML::Node array(YAML::NodeType::Sequence); for (const auto &item : json) { array.push_back(to_yaml(item)); } diff --git a/external/ytj/tests/ytj-test/src/ytj/to_yaml.test.cpp b/external/ytj/tests/ytj-test/src/ytj/to_yaml.test.cpp index e85e76e9b..ff1168a7d 100644 --- a/external/ytj/tests/ytj-test/src/ytj/to_yaml.test.cpp +++ b/external/ytj/tests/ytj-test/src/ytj/to_yaml.test.cpp @@ -10,6 +10,7 @@ TEST_CASE("to_yaml") SECTION("Scalar") { const std::vector cases{ + R"(null)"_json, R"(1)"_json, R"("1")"_json, R"(true)"_json, R"(false)"_json, R"(1.1)"_json, R"(-1.1)"_json, }; @@ -24,6 +25,7 @@ TEST_CASE("to_yaml") SECTION("Sequence") { const std::vector cases{ + R"([])"_json, R"([1])"_json, R"([1,1])"_json, R"(["1","1"])"_json, R"([true])"_json, R"([true,true])"_json, R"([1.1])"_json, @@ -41,6 +43,7 @@ TEST_CASE("to_yaml") SECTION("Map") { const std::vector cases{ + R"({})"_json, R"({"a":"b"})"_json, R"([{"a":"b"}])"_json, R"([{"a":1}])"_json, @@ -52,4 +55,19 @@ TEST_CASE("to_yaml") REQUIRE(json == to_json(yaml)); } } + + SECTION("Nested empty containers survive serialization") + { + const std::vector cases{ + R"([[],{}])"_json, + R"({"sources":[],"metadata":{}})"_json, + }; + + for (const auto &json : cases) { + INFO(json.dump()); + auto yaml = to_yaml(json); + REQUIRE(json == to_json(yaml)); + REQUIRE(json == to_json(YAML::Load(YAML::Dump(yaml)))); + } + } } diff --git a/libs/api/CMakeLists.txt b/libs/api/CMakeLists.txt index f4703149e..264fb9fd2 100644 --- a/libs/api/CMakeLists.txt +++ b/libs/api/CMakeLists.txt @@ -21,6 +21,10 @@ pfl_add_library( src/linglong/api/types/v1/BuilderProjectModules.hpp src/linglong/api/types/v1/BuilderProjectPackage.hpp src/linglong/api/types/v1/BuilderProjectSource.hpp + src/linglong/api/types/v1/BuilderProjectLock.hpp + src/linglong/api/types/v1/BuilderProjectLockDependency.hpp + src/linglong/api/types/v1/BuilderSourceLockEntry.hpp + src/linglong/api/types/v1/BuilderSourceLockResolved.hpp src/linglong/api/types/v1/CliContainer.hpp src/linglong/api/types/v1/CommonOptions.hpp src/linglong/api/types/v1/CommonResult.hpp diff --git a/libs/api/src/linglong/api/types/v1/BuilderProjectLock.hpp b/libs/api/src/linglong/api/types/v1/BuilderProjectLock.hpp new file mode 100644 index 000000000..f1beb2684 --- /dev/null +++ b/libs/api/src/linglong/api/types/v1/BuilderProjectLock.hpp @@ -0,0 +1,46 @@ +// This file is generated by tools/codegen.sh +// DO NOT EDIT IT. + +// clang-format off + +// To parse this JSON data, first install +// +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// BuilderProjectLock.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "linglong/api/types/v1/helper.hpp" + +#include "linglong/api/types/v1/BuilderProjectLockDependency.hpp" +#include "linglong/api/types/v1/BuilderSourceLockEntry.hpp" + +namespace linglong { +namespace api { +namespace types { +namespace v1 { +/** +* content lock for builder project sources and dependencies +*/ + +using nlohmann::json; + +/** +* content lock for builder project sources and dependencies +*/ +struct BuilderProjectLock { +std::vector dependencies; +std::vector sources; +int64_t version; +}; +} +} +} +} + +// clang-format on diff --git a/libs/api/src/linglong/api/types/v1/BuilderProjectLockDependency.hpp b/libs/api/src/linglong/api/types/v1/BuilderProjectLockDependency.hpp new file mode 100644 index 000000000..eea895990 --- /dev/null +++ b/libs/api/src/linglong/api/types/v1/BuilderProjectLockDependency.hpp @@ -0,0 +1,45 @@ +// This file is generated by tools/codegen.sh +// DO NOT EDIT IT. + +// clang-format off + +// To parse this JSON data, first install +// +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// BuilderProjectLockDependency.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "linglong/api/types/v1/helper.hpp" + +#include "linglong/api/types/v1/BuilderProjectLockDependencyResolved.hpp" + +namespace linglong { +namespace api { +namespace types { +namespace v1 { +/** +* requested and resolved builder dependency +*/ + +using nlohmann::json; + +/** +* requested and resolved builder dependency +*/ +struct BuilderProjectLockDependency { +std::string requested; +BuilderProjectLockDependencyResolved resolved; +std::string type; +}; +} +} +} +} + +// clang-format on diff --git a/libs/api/src/linglong/api/types/v1/BuilderProjectLockDependencyResolved.hpp b/libs/api/src/linglong/api/types/v1/BuilderProjectLockDependencyResolved.hpp new file mode 100644 index 000000000..9ef6768d9 --- /dev/null +++ b/libs/api/src/linglong/api/types/v1/BuilderProjectLockDependencyResolved.hpp @@ -0,0 +1,46 @@ +// This file is generated by tools/codegen.sh +// DO NOT EDIT IT. + +// clang-format off + +// To parse this JSON data, first install +// +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// BuilderProjectLockDependencyResolved.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "linglong/api/types/v1/helper.hpp" + +namespace linglong { +namespace api { +namespace types { +namespace v1 { +/** +* selected reference and modules of a builder dependency +*/ + +using nlohmann::json; + +/** +* selected reference and modules of a builder dependency +*/ +struct BuilderProjectLockDependencyResolved { +std::vector modules; +/** +* locked channel, package ID and exact version; architecture preserves the declaration or +* uses unknown when unspecified +*/ +std::string reference; +}; +} +} +} +} + +// clang-format on diff --git a/libs/api/src/linglong/api/types/v1/BuilderSourceLockEntry.hpp b/libs/api/src/linglong/api/types/v1/BuilderSourceLockEntry.hpp new file mode 100644 index 000000000..deabc821c --- /dev/null +++ b/libs/api/src/linglong/api/types/v1/BuilderSourceLockEntry.hpp @@ -0,0 +1,45 @@ +// This file is generated by tools/codegen.sh +// DO NOT EDIT IT. + +// clang-format off + +// To parse this JSON data, first install +// +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// BuilderSourceLockEntry.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "linglong/api/types/v1/helper.hpp" + +#include "linglong/api/types/v1/BuilderSourceLockResolved.hpp" +#include "linglong/api/types/v1/BuilderProjectSource.hpp" + +namespace linglong { +namespace api { +namespace types { +namespace v1 { +/** +* one source lock entry +*/ + +using nlohmann::json; + +/** +* one source lock entry +*/ +struct BuilderSourceLockEntry { +BuilderSourceLockResolved resolved; +BuilderProjectSource source; +}; +} +} +} +} + +// clang-format on diff --git a/libs/api/src/linglong/api/types/v1/BuilderSourceLockResolved.hpp b/libs/api/src/linglong/api/types/v1/BuilderSourceLockResolved.hpp new file mode 100644 index 000000000..9fa54c9fa --- /dev/null +++ b/libs/api/src/linglong/api/types/v1/BuilderSourceLockResolved.hpp @@ -0,0 +1,44 @@ +// This file is generated by tools/codegen.sh +// DO NOT EDIT IT. + +// clang-format off + +// To parse this JSON data, first install +// +// json.hpp https://github.com/nlohmann/json +// +// Then include this file, and then do +// +// BuilderSourceLockResolved.hpp data = nlohmann::json::parse(jsonString); + +#pragma once + +#include +#include +#include "linglong/api/types/v1/helper.hpp" + +namespace linglong { +namespace api { +namespace types { +namespace v1 { +/** +* resolved identity of a locked builder source +*/ + +using nlohmann::json; + +/** +* resolved identity of a locked builder source +*/ +struct BuilderSourceLockResolved { +/** +* locked content identity: sha256: or git-commit: +*/ +std::string hash; +}; +} +} +} +} + +// clang-format on diff --git a/libs/api/src/linglong/api/types/v1/Generators.hpp b/libs/api/src/linglong/api/types/v1/Generators.hpp index 7961f44be..c7eec38da 100644 --- a/libs/api/src/linglong/api/types/v1/Generators.hpp +++ b/libs/api/src/linglong/api/types/v1/Generators.hpp @@ -68,6 +68,11 @@ #include "linglong/api/types/v1/CliContainer.hpp" #include "linglong/api/types/v1/CdiDeviceEntry.hpp" #include "linglong/api/types/v1/CdiSpec.hpp" +#include "linglong/api/types/v1/BuilderProjectLock.hpp" +#include "linglong/api/types/v1/BuilderSourceLockEntry.hpp" +#include "linglong/api/types/v1/BuilderSourceLockResolved.hpp" +#include "linglong/api/types/v1/BuilderProjectLockDependency.hpp" +#include "linglong/api/types/v1/BuilderProjectLockDependencyResolved.hpp" #include "linglong/api/types/v1/BuilderProject.hpp" #include "linglong/api/types/v1/BuilderProjectSource.hpp" #include "linglong/api/types/v1/BuilderProjectPackage.hpp" @@ -129,6 +134,21 @@ void to_json(json & j, const BuilderProjectSource & x); void from_json(const json & j, BuilderProject & x); void to_json(json & j, const BuilderProject & x); +void from_json(const json & j, BuilderProjectLockDependencyResolved & x); +void to_json(json & j, const BuilderProjectLockDependencyResolved & x); + +void from_json(const json & j, BuilderProjectLockDependency & x); +void to_json(json & j, const BuilderProjectLockDependency & x); + +void from_json(const json & j, BuilderSourceLockResolved & x); +void to_json(json & j, const BuilderSourceLockResolved & x); + +void from_json(const json & j, BuilderSourceLockEntry & x); +void to_json(json & j, const BuilderSourceLockEntry & x); + +void from_json(const json & j, BuilderProjectLock & x); +void to_json(json & j, const BuilderProjectLock & x); + void from_json(const json & j, CdiSpec & x); void to_json(json & j, const CdiSpec & x); @@ -558,6 +578,63 @@ j["strip"] = x.strip; j["version"] = x.version; } +inline void from_json(const json & j, BuilderProjectLockDependencyResolved& x) { +x.modules = j.at("modules").get>(); +x.reference = j.at("reference").get(); +} + +inline void to_json(json & j, const BuilderProjectLockDependencyResolved & x) { +j = json::object(); +j["modules"] = x.modules; +j["reference"] = x.reference; +} + +inline void from_json(const json & j, BuilderProjectLockDependency& x) { +x.requested = j.at("requested").get(); +x.resolved = j.at("resolved").get(); +x.type = j.at("type").get(); +} + +inline void to_json(json & j, const BuilderProjectLockDependency & x) { +j = json::object(); +j["requested"] = x.requested; +j["resolved"] = x.resolved; +j["type"] = x.type; +} + +inline void from_json(const json & j, BuilderSourceLockResolved& x) { +x.hash = j.at("hash").get(); +} + +inline void to_json(json & j, const BuilderSourceLockResolved & x) { +j = json::object(); +j["hash"] = x.hash; +} + +inline void from_json(const json & j, BuilderSourceLockEntry& x) { +x.resolved = j.at("resolved").get(); +x.source = j.at("source").get(); +} + +inline void to_json(json & j, const BuilderSourceLockEntry & x) { +j = json::object(); +j["resolved"] = x.resolved; +j["source"] = x.source; +} + +inline void from_json(const json & j, BuilderProjectLock& x) { +x.dependencies = j.at("dependencies").get>(); +x.sources = j.at("sources").get>(); +x.version = j.at("version").get(); +} + +inline void to_json(json & j, const BuilderProjectLock & x) { +j = json::object(); +j["dependencies"] = x.dependencies; +j["sources"] = x.sources; +j["version"] = x.version; +} + inline void from_json(const json & j, CdiSpec& x) { x.checksum = j.at("checksum").get(); x.path = j.at("path").get(); @@ -1434,6 +1511,12 @@ x.applicationConfigurationPermissions = get_stack_optional(j, "ApplicationPermissionsRequest"); x.builderConfig = get_stack_optional(j, "BuilderConfig"); x.builderProject = get_stack_optional(j, "BuilderProject"); +x.builderProjectLock = get_stack_optional(j, "BuilderProjectLock"); +x.builderProjectLockDependency = get_stack_optional(j, "BuilderProjectLockDependency"); +x.builderProjectLockDependencyResolved = get_stack_optional(j, "BuilderProjectLockDependencyResolved"); +x.builderProjectSource = get_stack_optional(j, "BuilderProjectSource"); +x.builderSourceLockEntry = get_stack_optional(j, "BuilderSourceLockEntry"); +x.builderSourceLockResolved = get_stack_optional(j, "BuilderSourceLockResolved"); x.cdiDeviceEntry = get_stack_optional(j, "CDIDeviceEntry"); x.cliContainer = get_stack_optional(j, "CLIContainer"); x.commonOptions = get_stack_optional(j, "CommonOptions"); @@ -1499,6 +1582,24 @@ j["BuilderConfig"] = x.builderConfig; if (x.builderProject) { j["BuilderProject"] = x.builderProject; } +if (x.builderProjectLock) { +j["BuilderProjectLock"] = x.builderProjectLock; +} +if (x.builderProjectLockDependency) { +j["BuilderProjectLockDependency"] = x.builderProjectLockDependency; +} +if (x.builderProjectLockDependencyResolved) { +j["BuilderProjectLockDependencyResolved"] = x.builderProjectLockDependencyResolved; +} +if (x.builderProjectSource) { +j["BuilderProjectSource"] = x.builderProjectSource; +} +if (x.builderSourceLockEntry) { +j["BuilderSourceLockEntry"] = x.builderSourceLockEntry; +} +if (x.builderSourceLockResolved) { +j["BuilderSourceLockResolved"] = x.builderSourceLockResolved; +} if (x.cdiDeviceEntry) { j["CDIDeviceEntry"] = x.cdiDeviceEntry; } @@ -1641,7 +1742,7 @@ j["XDGDirectoryPermissions"] = x.xdgDirectoryPermissions; inline void from_json(const json & j, DeviceOption & x) { if (j == "passthru") x = DeviceOption::Passthru; -else { throw std::runtime_error("Cannot deserialize to enumeration \"DeviceOption\""); } +else { throw std::runtime_error("Input JSON does not conform to schema!"); } } inline void to_json(json & j, const DeviceOption & x) { @@ -1657,7 +1758,7 @@ else if (j == "Install") x = InteractionMessageType::Install; else if (j == "Uninstall") x = InteractionMessageType::Uninstall; else if (j == "Unknown") x = InteractionMessageType::Unknown; else if (j == "Upgrade") x = InteractionMessageType::Upgrade; -else { throw std::runtime_error("Cannot deserialize to enumeration \"InteractionMessageType\""); } +else { throw std::runtime_error("Input JSON does not conform to schema!"); } } inline void to_json(json & j, const InteractionMessageType & x) { @@ -1679,7 +1780,7 @@ else if (j == "Processing") x = State::Processing; else if (j == "Queued") x = State::Queued; else if (j == "Succeed") x = State::Succeed; else if (j == "Unknown") x = State::Unknown; -else { throw std::runtime_error("Cannot deserialize to enumeration \"State\""); } +else { throw std::runtime_error("Input JSON does not conform to schema!"); } } inline void to_json(json & j, const State & x) { @@ -1697,7 +1798,7 @@ default: throw std::runtime_error("Unexpected value in enumeration \"State\": " inline void from_json(const json & j, Version & x) { if (j == "1") x = Version::The1; -else { throw std::runtime_error("Cannot deserialize to enumeration \"Version\""); } +else { throw std::runtime_error("Input JSON does not conform to schema!"); } } inline void to_json(json & j, const Version & x) { diff --git a/libs/api/src/linglong/api/types/v1/LinglongAPIV1.hpp b/libs/api/src/linglong/api/types/v1/LinglongAPIV1.hpp index 9872a4bda..2168fc326 100644 --- a/libs/api/src/linglong/api/types/v1/LinglongAPIV1.hpp +++ b/libs/api/src/linglong/api/types/v1/LinglongAPIV1.hpp @@ -22,6 +22,12 @@ #include "linglong/api/types/v1/ApplicationPermissionsRequest.hpp" #include "linglong/api/types/v1/BuilderConfig.hpp" #include "linglong/api/types/v1/BuilderProject.hpp" +#include "linglong/api/types/v1/BuilderProjectLock.hpp" +#include "linglong/api/types/v1/BuilderProjectLockDependency.hpp" +#include "linglong/api/types/v1/BuilderProjectLockDependencyResolved.hpp" +#include "linglong/api/types/v1/BuilderProjectSource.hpp" +#include "linglong/api/types/v1/BuilderSourceLockEntry.hpp" +#include "linglong/api/types/v1/BuilderSourceLockResolved.hpp" #include "linglong/api/types/v1/CdiDeviceEntry.hpp" #include "linglong/api/types/v1/CliContainer.hpp" #include "linglong/api/types/v1/CommonOptions.hpp" @@ -97,6 +103,12 @@ std::optional applicationConfigurationPermi std::optional applicationPermissionsRequest; std::optional builderConfig; std::optional builderProject; +std::optional builderProjectLock; +std::optional builderProjectLockDependency; +std::optional builderProjectLockDependencyResolved; +std::optional builderProjectSource; +std::optional builderSourceLockEntry; +std::optional builderSourceLockResolved; std::optional cdiDeviceEntry; std::optional cliContainer; std::optional commonOptions; diff --git a/libs/linglong/CMakeLists.txt b/libs/linglong/CMakeLists.txt index 7d8be065c..a951faa26 100644 --- a/libs/linglong/CMakeLists.txt +++ b/libs/linglong/CMakeLists.txt @@ -13,6 +13,8 @@ pfl_add_library( src/linglong/adaptors/package_manager/package_manager1.h src/linglong/adaptors/task/task1.cpp src/linglong/adaptors/task/task1.h + src/linglong/builder/builder_directory.cpp + src/linglong/builder/builder_directory.h src/linglong/builder/builder_script.cpp src/linglong/builder/builder_script.h src/linglong/builder/config.cpp @@ -20,8 +22,12 @@ pfl_add_library( src/linglong/builder/linglong_builder.cpp src/linglong/builder/linglong_builder.h src/linglong/builder/printer.h + src/linglong/builder/source_cache.cpp + src/linglong/builder/source_cache.h src/linglong/builder/source_fetcher.cpp src/linglong/builder/source_fetcher.h + src/linglong/builder/project_lock.cpp + src/linglong/builder/project_lock.h src/linglong/cli/cli.cpp src/linglong/cli/cli.h src/linglong/cli/cli_printer.cpp diff --git a/libs/linglong/src/linglong/builder/builder_directory.cpp b/libs/linglong/src/linglong/builder/builder_directory.cpp new file mode 100644 index 000000000..deb6295b1 --- /dev/null +++ b/libs/linglong/src/linglong/builder/builder_directory.cpp @@ -0,0 +1,232 @@ +// SPDX-FileCopyrightText: 2026 UnionTech Software Technology Co., Ltd. +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "builder_directory.h" + +#include "linglong/package/architecture.h" +#include "linglong/utils/file.h" + +#include + +#include + +namespace linglong::builder { + +auto getProjectYamlPath(const std::filesystem::path &projectDir, + const std::filesystem::path &usePath) noexcept + -> utils::error::Result +{ + LINGLONG_TRACE("get project yaml path"); + + std::error_code ec; + auto path = usePath; + if (path.empty()) { + path = projectDir + / ("linglong." + package::Architecture::currentCPUArchitecture().toString() + ".yaml"); + const bool exists = std::filesystem::exists(path, ec); + if (ec) { + return LINGLONG_ERR(fmt::format("path {} error: {}", path.string(), ec.message())); + } + if (!exists) { + path = projectDir / "linglong.yaml"; + } + } + + // Resolve directory aliases, but keep a YAML file symlink as the project entry point. + const auto parent = path.has_parent_path() ? path.parent_path() : std::filesystem::path("."); + const auto directory = std::filesystem::canonical(parent, ec); + if (ec) { + return LINGLONG_ERR( + fmt::format("invalid file path {} error: {}", path.string(), ec.message())); + } + path = directory / path.filename(); + + const bool exists = std::filesystem::exists(path, ec); + if (ec) { + return LINGLONG_ERR(fmt::format("path {} error: {}", path.string(), ec.message())); + } + if (!exists) { + return LINGLONG_ERR("project yaml file not found: " + path.string()); + } + + return path; +} + +const std::filesystem::path BuilderDirectoryLayout::containerProject = "/project"; +const std::filesystem::path BuilderDirectoryLayout::containerRoot = containerProject / "linglong"; +const std::filesystem::path BuilderDirectoryLayout::containerOutput = containerRoot / "output"; +const std::filesystem::path BuilderDirectoryLayout::containerOverlays = containerRoot / "overlay"; +const std::filesystem::path BuilderDirectoryLayout::containerEntryScript = + containerRoot / "entry.sh"; +const std::filesystem::path BuilderDirectoryLayout::containerBuildExtScript = + containerRoot / "buildext.sh"; + +BuilderDirectoryLayout::BuilderDirectoryLayout(std::filesystem::path project) + : projectRoot(std::move(project)) + , stateRoot(projectRoot / "linglong") + , cacheRoot(stateRoot / "cache") + , sourceRoot(stateRoot / "sources") + , outputRoot(stateRoot / "output") + , buildOutputRoot(outputRoot / "_build") + , overlayRoot(stateRoot / "overlay") + , buildBaseOverlayPath(overlayRoot / "build_base") + , buildRuntimeOverlayPath(overlayRoot / "build_runtime") + , prepareBaseOverlayPath(overlayRoot / "prepare_base") + , prepareRuntimeOverlayPath(overlayRoot / "prepare_runtime") + , buildOverlayStatePath(overlayRoot / "build_overlay.state") + , entryScriptPath(stateRoot / "entry.sh") + , buildExtScriptPath(stateRoot / "buildext.sh") +{ +} + +auto BuilderDirectoryLayout::module(std::string_view name) const -> std::filesystem::path +{ + return outputRoot / name; +} + +auto BuilderDirectoryLayout::lockProject() const noexcept + -> utils::error::Result +{ + LINGLONG_TRACE("lock builder project"); + + std::error_code ec; + const auto project = std::filesystem::canonical(projectRoot, ec); + if (ec) { + return LINGLONG_ERR("failed to resolve project directory: " + ec.message()); + } + + // Keep the lock outside the directories removed by clean, and never unlink it. + auto lock = utils::filelock::FileLock::create(project / ".ll-builder.lock", + utils::filelock::LockType::Write); + if (!lock) { + return LINGLONG_ERR("failed to create project lock", lock); + } + + auto locked = lock->tryLock(utils::filelock::LockType::Write); + if (!locked) { + return LINGLONG_ERR("failed to acquire project lock", locked); + } + if (!*locked) { + return LINGLONG_ERR("project is already in use by another ll-builder process: " + + project.string()); + } + + return lock; +} + +auto BuilderDirectoryLayout::ensureRoot() const noexcept -> utils::error::Result +{ + LINGLONG_TRACE("ensure builder state directory"); + + std::error_code ec; + std::filesystem::create_directories(stateRoot, ec); + if (ec) { + return LINGLONG_ERR( + fmt::format("failed to create directory {}: {}", stateRoot.string(), ec.message())); + } + + return LINGLONG_OK; +} + +auto BuilderDirectoryLayout::resetSources() const noexcept -> utils::error::Result +{ + LINGLONG_TRACE("reset builder source directory"); + + return reset(sourceRoot, true); +} + +auto BuilderDirectoryLayout::resetOutput() const noexcept -> utils::error::Result +{ + LINGLONG_TRACE("reset builder output directory"); + + auto result = reset(outputRoot, false); + if (!result) { + return LINGLONG_ERR(result); + } + + std::error_code ec; + std::filesystem::create_directories(buildOutputRoot, ec); + if (ec) { + return LINGLONG_ERR( + fmt::format("failed to create directory {}: {}", buildOutputRoot.string(), ec.message())); + } + + return LINGLONG_OK; +} + +auto BuilderDirectoryLayout::resetOverlay(const std::filesystem::path &directory, + bool createUpperdir) const noexcept + -> utils::error::Result +{ + LINGLONG_TRACE("reset builder overlay directory"); + + auto result = reset(directory, false); + if (!result) { + return LINGLONG_ERR(result); + } + + if (createUpperdir) { + std::error_code ec; + std::filesystem::create_directories(directory / "upperdir", ec); + if (ec) { + return LINGLONG_ERR("failed to create overlay: " + ec.message()); + } + } + + return LINGLONG_OK; +} + +auto BuilderDirectoryLayout::clean() const noexcept -> utils::error::Result +{ + LINGLONG_TRACE("clean builder state directory"); + + std::error_code ec; + const auto exists = std::filesystem::exists(stateRoot, ec); + if (ec) { + return LINGLONG_ERR( + fmt::format("failed to check directory {}: {}", stateRoot.string(), ec.message())); + } + if (!exists) { + return LINGLONG_OK; + } + + auto removable = utils::makeDirectoryTreeRemovable(stateRoot); + if (!removable) { + return LINGLONG_ERR("failed to make directory tree removable", removable); + } + + std::filesystem::remove_all(stateRoot, ec); + if (ec) { + return LINGLONG_ERR( + fmt::format("failed to remove {}: {}", stateRoot.string(), ec.message())); + } + + return LINGLONG_OK; +} + +auto BuilderDirectoryLayout::reset(const std::filesystem::path &directory, + bool recreate) const noexcept -> utils::error::Result +{ + LINGLONG_TRACE("reset builder directory"); + + std::error_code ec; + std::filesystem::remove_all(directory, ec); + if (ec) { + return LINGLONG_ERR( + fmt::format("failed to reset directory {}: {}", directory.string(), ec.message())); + } + + if (!recreate) { + return LINGLONG_OK; + } + + std::filesystem::create_directories(directory, ec); + if (ec) { + return LINGLONG_ERR( + fmt::format("failed to create directory {}: {}", directory.string(), ec.message())); + } + + return LINGLONG_OK; +} + +} // namespace linglong::builder diff --git a/libs/linglong/src/linglong/builder/builder_directory.h b/libs/linglong/src/linglong/builder/builder_directory.h new file mode 100644 index 000000000..de2f6fa80 --- /dev/null +++ b/libs/linglong/src/linglong/builder/builder_directory.h @@ -0,0 +1,100 @@ +// SPDX-FileCopyrightText: 2026 UnionTech Software Technology Co., Ltd. +// SPDX-License-Identifier: LGPL-3.0-or-later + +#pragma once + +#include "linglong/utils/error/error.h" +#include "linglong/utils/filelock.h" + +#include +#include + +namespace linglong::builder { + +auto getProjectYamlPath(const std::filesystem::path &projectDir, + const std::filesystem::path &usePath) noexcept + -> utils::error::Result; + +class BuilderDirectoryLayout +{ +public: + explicit BuilderDirectoryLayout(std::filesystem::path projectRoot); + + const std::filesystem::path &project() const noexcept { return projectRoot; } + + const std::filesystem::path &root() const noexcept { return stateRoot; } + + const std::filesystem::path &cache() const noexcept { return cacheRoot; } + + const std::filesystem::path &sources() const noexcept { return sourceRoot; } + + const std::filesystem::path &output() const noexcept { return outputRoot; } + + const std::filesystem::path &buildOutput() const noexcept { return buildOutputRoot; } + + const std::filesystem::path &overlays() const noexcept { return overlayRoot; } + + const std::filesystem::path &buildBaseOverlay() const noexcept { return buildBaseOverlayPath; } + + const std::filesystem::path &buildRuntimeOverlay() const noexcept + { + return buildRuntimeOverlayPath; + } + + const std::filesystem::path &prepareBaseOverlay() const noexcept + { + return prepareBaseOverlayPath; + } + + const std::filesystem::path &prepareRuntimeOverlay() const noexcept + { + return prepareRuntimeOverlayPath; + } + + const std::filesystem::path &buildOverlayState() const noexcept + { + return buildOverlayStatePath; + } + + const std::filesystem::path &entryScript() const noexcept { return entryScriptPath; } + + const std::filesystem::path &buildExtScript() const noexcept { return buildExtScriptPath; } + + auto module(std::string_view name) const -> std::filesystem::path; + + auto lockProject() const noexcept -> utils::error::Result; + auto ensureRoot() const noexcept -> utils::error::Result; + auto resetSources() const noexcept -> utils::error::Result; + auto resetOutput() const noexcept -> utils::error::Result; + auto resetOverlay(const std::filesystem::path &directory, + bool createUpperdir = false) const noexcept -> utils::error::Result; + auto clean() const noexcept -> utils::error::Result; + + static const std::filesystem::path containerProject; + static const std::filesystem::path containerRoot; + static const std::filesystem::path containerOutput; + static const std::filesystem::path containerOverlays; + static const std::filesystem::path containerEntryScript; + static const std::filesystem::path containerBuildExtScript; + +private: + auto reset(const std::filesystem::path &directory, bool recreate) const noexcept + -> utils::error::Result; + + std::filesystem::path projectRoot; + std::filesystem::path stateRoot; + std::filesystem::path cacheRoot; + std::filesystem::path sourceRoot; + std::filesystem::path outputRoot; + std::filesystem::path buildOutputRoot; + std::filesystem::path overlayRoot; + std::filesystem::path buildBaseOverlayPath; + std::filesystem::path buildRuntimeOverlayPath; + std::filesystem::path prepareBaseOverlayPath; + std::filesystem::path prepareRuntimeOverlayPath; + std::filesystem::path buildOverlayStatePath; + std::filesystem::path entryScriptPath; + std::filesystem::path buildExtScriptPath; +}; + +} // namespace linglong::builder diff --git a/libs/linglong/src/linglong/builder/linglong_builder.cpp b/libs/linglong/src/linglong/builder/linglong_builder.cpp index a6821341c..e0cc940d6 100644 --- a/libs/linglong/src/linglong/builder/linglong_builder.cpp +++ b/libs/linglong/src/linglong/builder/linglong_builder.cpp @@ -11,6 +11,7 @@ #include "linglong/api/types/v1/Generators.hpp" #include "linglong/builder/builder_script.h" #include "linglong/builder/printer.h" +#include "linglong/builder/source_cache.h" #include "linglong/common/global/initialize.h" #include "linglong/common/strings.h" #include "linglong/common/version.h" @@ -30,8 +31,9 @@ #include "linglong/utils/log/log.h" #include "linglong/utils/serialize/json.h" #include "linglong/utils/serialize/packageinfo_handler.h" +#include "linglong/utils/temporary_directory.h" #include "ocppi/runtime/RunOption.hpp" -#include "source_fetcher.h" +#include "project_lock.h" #include #include @@ -41,6 +43,7 @@ #include #include +#include #include #include #include @@ -48,6 +51,7 @@ #include #include #include +#include #include #include @@ -72,42 +76,46 @@ currentReference(const api::types::v1::BuilderProject &project) return package::Reference::fromBuilderProject(project); } -utils::error::Result -fetchSources(const std::vector &sources, - const QDir &cacheDir, - const QDir &destination, - [[maybe_unused]] const api::types::v1::BuilderConfig &cfg) noexcept +void printSourceFetchStatus(std::size_t, + const api::types::v1::BuilderProjectSource &source, + SourceFetchStatus status) { - LINGLONG_TRACE("fetch sources to " + destination.absolutePath().toStdString()); - - for (decltype(sources.size()) pos = 0; pos < sources.size(); ++pos) { - if (!sources.at(pos).url.has_value()) { - return LINGLONG_ERR("source missing url"); - } - auto url = QString::fromStdString(*(sources.at(pos).url)); - if (url.length() > 75) { // NOLINT - url = "..." + url.right(70); // NOLINT - } - printReplacedText(fmt::format("{:<20}{:<15}{:<75}downloading ...", - "Source " + std::to_string(pos), - sources.at(pos).kind, - url.toStdString()), - 2); - SourceFetcher fetcher(sources.at(pos), cacheDir); - auto result = fetcher.fetch(QDir(destination)); - if (!result) { - return LINGLONG_ERR(result); - } - printReplacedText(QString("%1%2%3%4") - .arg("Source " + QString::number(pos), -20) // NOLINT - .arg(QString::fromStdString(sources.at(pos).kind), -15) // NOLINT - .arg(url, -75) // NOLINT - .arg("complete\n") - .toStdString(), - 2); - } + auto url = QString::fromStdString(source.url.value_or("")); + if (url.length() > 75) { + url = "..." + url.right(70); + } + + std::string_view statusText; + switch (status) { + case SourceFetchStatus::Fetching: + statusText = "fetching ..."; + break; + case SourceFetchStatus::Complete: + statusText = "complete\n"; + break; + case SourceFetchStatus::Failed: + statusText = "failed\n"; + break; + } + + printReplacedText(fmt::format("{:<20}{:<15}{:<75}{}", + sourceOutputName(source), + source.kind, + url.toStdString(), + statusText), + 2); +} - return LINGLONG_OK; +void printDependencyStatus(const package::Reference &reference, + std::string_view module, + std::string_view status) +{ + printReplacedText(fmt::format("{:<35}{:<15}{:<15}{}\n", + reference.id, + reference.version.toString(), + module, + status), + 2); } } // namespace @@ -242,11 +250,13 @@ utils::error::Result pullDependency(const std::string &fuzzy return std::move(refRepo->reference); } -void mergeOutput(const std::vector &src, - const std::filesystem::path &dest, - const std::vector &targets, - const std::vector &excludes) +utils::error::Result mergeOutput(const std::vector &src, + const std::filesystem::path &dest, + const std::vector &targets, + const std::vector &excludes) { + LINGLONG_TRACE("merge build output"); + for (const auto &dir : src) { LogD("merge {} to {}", dir, dest); auto matcher = [&dir, &targets, &excludes](const std::filesystem::path &path) { @@ -276,8 +286,56 @@ void mergeOutput(const std::vector &src, return false; }; - utils::copyDirectory(dir, dest, matcher); + auto copied = utils::copyDirectory(dir, dest, matcher); + if (!copied) { + return LINGLONG_ERR(copied); + } + } + + return LINGLONG_OK; +} + +auto prepareBuildOverlays(const BuilderDirectoryLayout &directories, + const std::string &dependencyState, + bool hasRuntime) -> utils::error::Result +{ + LINGLONG_TRACE("prepare build overlays"); + + const auto &marker = directories.buildOverlayState(); + auto previousState = utils::readFile(marker); + std::error_code ec; + if (previousState && *previousState == dependencyState + && std::filesystem::is_directory(directories.buildBaseOverlay() / "upperdir", ec) + && (!hasRuntime + || std::filesystem::is_directory(directories.buildRuntimeOverlay() / "upperdir", ec))) { + return true; + } + + std::filesystem::remove(marker, ec); + if (ec) { + return LINGLONG_ERR("failed to invalidate build overlay state: " + ec.message()); + } + + auto baseReset = directories.resetOverlay(directories.buildBaseOverlay(), true); + if (!baseReset) { + return LINGLONG_ERR(baseReset); + } + auto runtimeReset = directories.resetOverlay(directories.buildRuntimeOverlay(), hasRuntime); + if (!runtimeReset) { + return LINGLONG_ERR(runtimeReset); + } + + auto temporaryMarker = marker; + temporaryMarker += ".tmp"; + auto written = utils::writeFile(temporaryMarker, dependencyState); + if (!written) { + return LINGLONG_ERR("failed to write build overlay state", written); + } + std::filesystem::rename(temporaryMarker, marker, ec); + if (ec) { + return LINGLONG_ERR("failed to publish build overlay state: " + ec.message()); } + return false; } } // namespace detail @@ -398,14 +456,13 @@ Builder::Builder(std::optional project, runtime::ContainerBuilder &containerBuilder, const api::types::v1::BuilderConfig &cfg) : repo(repo) - , workingDir(std::move(projectDir)) + , directories(std::move(projectDir)) , project(project) , containerBuilder(containerBuilder) , cfg(cfg) , buildContext(repo) { - LogD("builder working dir: {}", workingDir); - internalDir = workingDir / "linglong"; + LogD("builder working dir: {}", directories.project()); } auto Builder::getConfig() const noexcept -> api::types::v1::BuilderConfig @@ -423,10 +480,78 @@ void Builder::setConfig(const api::types::v1::BuilderConfig &cfg) noexcept this->cfg = cfg; } +auto Builder::lockProject() noexcept -> utils::error::Result +{ + LINGLONG_TRACE("lock builder project operation"); + + if (projectMutex) { + return LINGLONG_OK; + } + + auto lock = directories.lockProject(); + if (!lock) { + return LINGLONG_ERR(lock); + } + + std::optional yamlLock; + if (!projectYamlFile.empty()) { + std::error_code ec; + const auto projectDirectory = std::filesystem::canonical(directories.project(), ec); + if (ec) { + return LINGLONG_ERR("failed to resolve project directory: " + ec.message()); + } + + const auto manifest = std::filesystem::path(projectYamlFile); + const auto parent = + manifest.has_parent_path() ? manifest.parent_path() : std::filesystem::path("."); + const auto yamlDirectory = std::filesystem::canonical(parent, ec); + if (ec) { + return LINGLONG_ERR("failed to resolve project YAML directory: " + ec.message()); + } + + if (yamlDirectory != projectDirectory) { + auto acquired = BuilderDirectoryLayout(yamlDirectory).lockProject(); + if (!acquired) { + return LINGLONG_ERR(acquired); + } + yamlLock.emplace(std::move(*acquired)); + } + } + + // Publish ownership only after every required lock has been acquired. + projectMutex.emplace(std::move(*lock)); + projectYamlMutex = std::move(yamlLock); + return LINGLONG_OK; +} + utils::error::Result Builder::buildStagePrepare() noexcept { LINGLONG_TRACE("build prepare"); + if (buildOptions.locked && buildOptions.skipFetchSource) { + return LINGLONG_ERR("--locked cannot be combined with --skip-fetch-source"); + } + + projectLock.reset(); + pendingSources.reset(); + + const auto path = projectLockPath(projectYamlFile); + std::error_code ec; + const bool exists = std::filesystem::exists(path, ec); + if (ec) { + return LINGLONG_ERR("failed to check project lock file: " + ec.message()); + } + + if (exists) { + auto loaded = readProjectLock(path); + if (!loaded) { + return LINGLONG_ERR(loaded); + } + projectLock = std::move(*loaded); + } else if (buildOptions.locked) { + return LINGLONG_ERR(path.string() + " does not exist; run ll-builder generate-lockfile"); + } + uid = getuid(); gid = getgid(); @@ -468,12 +593,9 @@ utils::error::Result Builder::buildStagePrepare() noexcept printRepo(); - std::error_code ec; - std::filesystem::create_directory(this->internalDir, ec); - if (ec) { - return LINGLONG_ERR( - fmt::format("failed to create internal dir {} {}", this->internalDir, ec.message()) - .c_str()); + auto ensured = directories.ensureRoot(); + if (!ensured) { + return LINGLONG_ERR(ensured); } return LINGLONG_OK; @@ -483,10 +605,23 @@ utils::error::Result Builder::buildStageFetchSource() noexcept { LINGLONG_TRACE("build stage fetch source"); - if (!this->project->sources || this->buildOptions.skipFetchSource) { + if (this->buildOptions.skipFetchSource) { + LogW("using existing sources without source lock verification"); return LINGLONG_OK; } + utils::error::Result plan; + if (this->buildOptions.locked) { + // buildStagePrepare() requires and loads the lock for locked builds. + assert(projectLock.has_value()); + plan = verifyProjectLockSources(this->project->sources, *projectLock); + } else { + plan = resolveProjectLockSources(this->project->sources, projectLock); + } + if (!plan) { + return LINGLONG_ERR(plan); + } + printMessage("[Processing Sources]"); printMessage(QString("%1%2%3%4") .arg("Name", -20) // NOLINT @@ -495,22 +630,93 @@ utils::error::Result Builder::buildStageFetchSource() noexcept .arg("Status") .toStdString(), 2); - auto fetchCacheDir = QString::fromStdString(internalDir / "cache"); - if (!qEnvironmentVariableIsEmpty("LINGLONG_FETCH_CACHE")) { - fetchCacheDir = qgetenv("LINGLONG_FETCH_CACHE"); + + auto reset = directories.resetSources(); + if (!reset) { + return LINGLONG_ERR(reset); } - // clean sources directory on every build - auto fetchSourcesDir = QDir(QString::fromStdString(internalDir / "sources")); - fetchSourcesDir.removeRecursively(); - auto result = fetchSources(*this->project->sources, fetchCacheDir, fetchSourcesDir, this->cfg); - if (!result) { - return LINGLONG_ERR(result); + auto entries = fetchProjectSources(*plan, + sourceCachePath(), + directories.sources(), + this->buildOptions.offline, + printSourceFetchStatus); + if (!entries) { + return LINGLONG_ERR(entries); + } + + if (!this->buildOptions.locked && plan->changed) { + pendingSources = std::move(*entries); } return LINGLONG_OK; } +utils::error::Result Builder::processProjectLock(ProjectLockMode mode) noexcept +{ + LINGLONG_TRACE("process project lock"); + + auto locked = lockProject(); + if (!locked) { + return LINGLONG_ERR(locked); + } + + projectLock.reset(); + pendingSources.reset(); + + const auto path = projectLockPath(projectYamlFile); + std::error_code ec; + const bool exists = std::filesystem::exists(path, ec); + if (ec) { + return LINGLONG_ERR("failed to check project lock file: " + ec.message()); + } + + if (mode == ProjectLockMode::Generate && exists) { + return LINGLONG_ERR(path.string() + " already exists; run ll-builder update-lockfile"); + } + if (mode == ProjectLockMode::Update && !exists) { + return LINGLONG_ERR(path.string() + " does not exist; run ll-builder generate-lockfile"); + } + + auto plan = resolveProjectLockSources(this->project->sources, projectLock); + if (!plan) { + return LINGLONG_ERR(plan); + } + + const auto cache = sourceCachePath(); + SourceCacheLayout cacheLayout(cache); + auto ensured = cacheLayout.ensure(); + if (!ensured) { + return LINGLONG_ERR(ensured); + } + + auto temporaryDestination = + utils::TemporaryDirectory::create("fetch-", cacheLayout.temporary()); + if (!temporaryDestination) { + return LINGLONG_ERR(temporaryDestination); + } + + auto entries = fetchProjectSources(*plan, cache, temporaryDestination->path(), false); + if (!entries) { + return LINGLONG_ERR(entries); + } + + if (plan->changed) { + pendingSources = std::move(*entries); + } + + return buildStagePullDependency(); +} + +auto Builder::sourceCachePath() const -> std::filesystem::path +{ + if (const auto *cacheEnv = std::getenv("LINGLONG_FETCH_CACHE"); + cacheEnv && cacheEnv[0] != '\0') { + return std::filesystem::path(cacheEnv); + } + return directories.cache(); +} + utils::error::Result Builder::ensureUtils(const std::string &id, const package::Architecture &arch) noexcept { @@ -569,8 +775,11 @@ utils::error::Result Builder::buildStagePullDependency() noexcept { LINGLONG_TRACE("build stage pull dependency"); - if (!this->project->base && !this->project->runtime) { - return LINGLONG_ERR("at least one of base or runtime must be specified"); + assert(!buildOptions.locked || projectLock.has_value()); + auto plan = + planProjectDependencies(project->base, project->runtime, projectLock, buildOptions.locked); + if (!plan) { + return LINGLONG_ERR(plan); } printMessage("[Processing Dependency]"); @@ -582,134 +791,291 @@ utils::error::Result Builder::buildStagePullDependency() noexcept .toStdString(), 2); - auto handleDependency = - [this](const std::string &refStr) -> utils::error::Result { - LINGLONG_TRACE("handle dependency " + refStr); - - auto printStatus = [](const package::Reference &ref, - std::string_view module, - std::string_view status) { - printReplacedText( - fmt::format("{:<35}{:<15}{:<15}{}\n", ref.id, ref.version.toString(), module, status), - 2); - }; + std::optional runtimeBase; + std::optional runtimeRef; + if (plan->runtime) { + auto fetched = fetchProjectDependency(*plan->runtime); + if (!fetched) { + return LINGLONG_ERR(fetched); + } - auto ref = detail::clearDependency(refStr, this->repo, !this->buildOptions.skipPullDepend); - if (!ref) { - return LINGLONG_ERR(ref); + runtimeRef = std::move(*fetched); + auto runtimeItem = repo.getLayerItem(*runtimeRef); + if (!runtimeItem) { + return LINGLONG_ERR("failed to get runtime information", runtimeItem); } - const package::Reference *resolvedRef = nullptr; - auto &[refRepo, localRef] = *ref; - if (!refRepo) { - const auto &local = *localRef; - resolvedRef = &local; - // binary module is install - printStatus(local, "binary", "complete"); + auto requiredBase = package::FuzzyReference::parse(runtimeItem->info.base); + if (!requiredBase) { + return LINGLONG_ERR("failed to parse base required by runtime", requiredBase); + } + runtimeBase = std::move(*requiredBase); - // try pull develop module if skipPullDepend is not set - if (!this->buildOptions.skipPullDepend) { - auto res = detail::pullDependency(localRef->toString(), this->repo, "develop"); - if (!res) { - LogW("failed to pull develop module of {}: {}", refStr, res.error().message()); + if (!plan->base) { + // Preserve the runtime's base declaration, including any explicit architecture. + const auto &baseDeclaration = runtimeItem->info.base; + auto oldBase = projectLock ? lockedDependency(*projectLock, "base") : std::nullopt; + if (oldBase && oldBase->requested == baseDeclaration) { + plan->base = LockedDependencyRequest{ std::move(*oldBase) }; + } else { + if (buildOptions.locked) { + return LINGLONG_ERR("base differs from linglong.lock; run ll-builder " + "update-lockfile"); } + + plan->base = ResolveDependencyRequest{ baseDeclaration, "base" }; + plan->changed = true; } - } else { - // use remote reference - resolvedRef = &refRepo->reference; - auto res = detail::pullResolvedRef(*refRepo, this->repo, "binary"); - if (!res) { - if (!localRef) { - return LINGLONG_ERR(res); - } + } + } - LogW("failed to pull binary module of {}, use local version {}: {}", - refRepo->reference.toString(), - localRef->toString(), - res.error()); - printStatus(*localRef, "binary", "complete"); - - auto layerDir = this->repo.getLayerDir(*localRef, "develop"); - if (!layerDir) { - auto developRes = - detail::pullDependency(localRef->toString(), this->repo, "develop"); - if (!developRes) { - LogW("failed to pull develop module of {}: {}", - refStr, - developRes.error().message()); - } - layerDir = this->repo.getLayerDir(*localRef, "develop"); - } + assert(plan->base.has_value()); + auto baseRef = fetchProjectDependency(*plan->base); + if (!baseRef) { + return LINGLONG_ERR(baseRef); + } - printStatus(*localRef, "develop", layerDir ? "complete" : "missing"); + if (runtimeBase) { + if (!baseRef->semanticMatch(*runtimeBase)) { + return LINGLONG_ERR( + fmt::format("base is not compatible with runtime. \n - Current base: {}\n - " + "Current runtime: {}\n - base required by runtime: {}", + baseRef->toString(), + runtimeRef->toString(), + runtimeBase->toString())); + } + } - return *localRef; - } - printStatus(refRepo->reference, "binary", "complete"); + auto merged = repo.mergeModules(); + if (!merged) { + return LINGLONG_ERR("failed to merge modules", merged); + } + + auto finished = finishProjectLockDependencies(*plan, *baseRef, runtimeRef); + if (!finished) { + return LINGLONG_ERR(finished); + } + + return LINGLONG_OK; +} + +auto Builder::fetchProjectDependency(const DependencyFetchRequest &request) noexcept + -> utils::error::Result +{ + LINGLONG_TRACE("fetch project dependency"); - res = detail::pullResolvedRef(*refRepo, this->repo, "develop"); - if (!res) { - LogW("failed to pull develop module of {}: {}", - refRepo->reference.toString(), - res.error().message()); + if (const auto *lockedRequest = std::get_if(&request)) { + const auto &lockedEntry = lockedRequest->entry; + auto reference = lockedDependencyReference(lockedEntry); + if (!reference) { + return LINGLONG_ERR("invalid locked dependency reference", reference); + } + + std::optional remote; + for (const auto &module : lockedEntry.resolved.modules) { + auto fetched = ensureDependencyModule(*reference, module, remote); + if (!fetched) { + printDependencyStatus(*reference, module, "failed"); + return LINGLONG_ERR(fetched); } + printDependencyStatus(*reference, module, "complete"); } + return *reference; + } - auto layerDir = this->repo.getLayerDir(*resolvedRef, "develop"); - printStatus(*resolvedRef, "develop", layerDir ? "complete" : "missing"); + return fetchResolvedProjectDependency(std::get(request)); +} - return *resolvedRef; - }; +auto Builder::fetchResolvedProjectDependency(const ResolveDependencyRequest &request) noexcept + -> utils::error::Result +{ + LINGLONG_TRACE("resolve and fetch dependency " + request.requested); - std::optional runtimeBase; - std::optional runtimeRef; - if (this->project->runtime) { - auto resolvedRuntime = handleDependency(*this->project->runtime); - if (!resolvedRuntime) { - return LINGLONG_ERR(resolvedRuntime); + auto fuzzy = package::FuzzyReference::parse(request.requested); + if (!fuzzy) { + return LINGLONG_ERR("invalid dependency reference " + request.requested, fuzzy); + } + + if (!fuzzy->arch) { + fuzzy->arch = package::Architecture::currentCPUArchitecture(); + } + + auto ref = detail::clearDependency(fuzzy->toString(), repo, !buildOptions.skipPullDepend); + if (!ref) { + return LINGLONG_ERR(ref); + } + + auto &[remote, localRef] = *ref; + auto reference = remote ? remote->reference : *localRef; + + auto binary = ensureDependencyModule(reference, "binary", remote); + if (!binary && localRef && *localRef != reference) { + LogW("failed to fetch binary module of {}, use local version {}: {}", + reference.toString(), + localRef->toString(), + binary.error().message()); + reference = *localRef; + remote.reset(); + binary = ensureDependencyModule(reference, "binary", remote); + } + if (!binary) { + printDependencyStatus(reference, "binary", "failed"); + return LINGLONG_ERR(binary); + } + printDependencyStatus(reference, "binary", "complete"); + + auto develop = ensureDependencyModule(reference, "develop", remote); + if (!develop && !buildOptions.skipPullDepend) { + LogW("failed to fetch develop module of {}: {}", + reference.toString(), + develop.error().message()); + } + printDependencyStatus(reference, "develop", develop ? "complete" : "missing"); + return reference; +} + +auto Builder::ensureDependencyModule(const package::Reference &reference, + const std::string &module, + std::optional &remote) noexcept + -> utils::error::Result +{ + LINGLONG_TRACE("ensure dependency module " + reference.toString() + "/" + module); + + if (repo.getLayerDir(reference, module)) { + return LINGLONG_OK; + } + + if (buildOptions.skipPullDepend) { + return LINGLONG_ERR("dependency is unavailable locally: " + reference.toString() + "/" + + module); + } + + if (!remote) { + auto fuzzy = package::FuzzyReference::parse(reference.toString()); + if (!fuzzy) { + return LINGLONG_ERR(fuzzy); } - runtimeRef = *resolvedRuntime; - auto runtimeItem = this->repo.getLayerItem(*runtimeRef); - if (!runtimeItem) { - return LINGLONG_ERR("failed to get runtime information", runtimeItem); + auto candidates = repo.matchRemoteByPriority(*fuzzy); + if (!candidates) { + return LINGLONG_ERR("failed to find dependency remotely: " + reference.toString(), + candidates); } - if (this->project->base) { - runtimeBase = runtimeItem->info.base; - } else { - this->project->base = runtimeItem->info.base; + + for (const auto &[candidateRepo, packages] : candidates->getRepoPackages()) { + for (const auto &packageInfo : packages) { + auto candidate = package::Reference::fromPackageInfo(packageInfo); + if (candidate && *candidate == reference) { + remote = package::ReferenceWithRepo{ candidateRepo, *candidate }; + break; + } + } + if (remote) { + break; + } + } + if (!remote) { + return LINGLONG_ERR("dependency is unavailable remotely: " + reference.toString()); } } - auto res = handleDependency(*this->project->base); - if (!res) { - return LINGLONG_ERR(res); + assert(remote->reference == reference); + auto pulled = detail::pullResolvedRef(*remote, repo, module); + if (!pulled) { + return LINGLONG_ERR(pulled); } - if (runtimeBase) { - auto runtimeBaseFuzzyRef = package::FuzzyReference::parse(*runtimeBase); - if (!runtimeBaseFuzzyRef) { - return LINGLONG_ERR("failed to parse base required by runtime", runtimeBaseFuzzyRef); - } - if (!runtimeBaseFuzzyRef->version) { - return LINGLONG_ERR("base version required by runtime is missing"); - } + if (!repo.getLayerDir(reference, module)) { + return LINGLONG_ERR("dependency module is unavailable: " + reference.toString() + "/" + + module); + } + return LINGLONG_OK; +} - if (!res->semanticMatch(*runtimeBaseFuzzyRef)) { - return LINGLONG_ERR( - fmt::format("base is not compatible with runtime. \n - Current base: {}\n - " - "Current runtime: {}\n - base required by runtime: {}", - res->toString(), - runtimeRef->toString(), - *runtimeBase)); +auto Builder::recordProjectDependency(const DependencyFetchRequest &request, + const package::Reference &reference) noexcept + -> utils::error::Result +{ + LINGLONG_TRACE("record locked dependency"); + + if (const auto *lockedRequest = std::get_if(&request)) { + return lockedRequest->entry; + } + + const auto &resolved = std::get(request); + auto fuzzy = package::FuzzyReference::parse(resolved.requested); + if (!fuzzy) { + return LINGLONG_ERR(fuzzy); + } + if (!reference.semanticMatch(*fuzzy)) { + return LINGLONG_ERR("resolved dependency does not match its declaration"); + } + + fuzzy->channel = reference.channel; + fuzzy->id = reference.id; + fuzzy->version = reference.version.toString(); + + LockedDependency locked; + locked.requested = resolved.requested; + locked.type = resolved.type; + locked.resolved.reference = fuzzy->toString(); + locked.resolved.modules = repo.getModuleList(reference); + + std::sort(locked.resolved.modules.begin(), locked.resolved.modules.end()); + locked.resolved.modules.erase( + std::unique(locked.resolved.modules.begin(), locked.resolved.modules.end()), + locked.resolved.modules.end()); + + if (locked.resolved.modules.empty()) { + return LINGLONG_ERR("dependency has no available modules: " + reference.toString()); + } + + return locked; +} + +auto Builder::finishProjectLockDependencies( + const ProjectDependencyPlan &plan, + const package::Reference &base, + const std::optional &runtime) noexcept -> utils::error::Result +{ + LINGLONG_TRACE("finish project dependency lock"); + + if (!plan.changed && !pendingSources) { + return LINGLONG_OK; + } + + assert(plan.base.has_value()); + auto baseEntry = recordProjectDependency(*plan.base, base); + if (!baseEntry) { + return LINGLONG_ERR(baseEntry); + } + + std::optional runtimeEntry; + if (runtime) { + assert(plan.runtime.has_value()); + auto recorded = recordProjectDependency(*plan.runtime, *runtime); + if (!recorded) { + return LINGLONG_ERR(recorded); } + runtimeEntry = std::move(*recorded); } - auto ret = this->repo.mergeModules(); - if (!ret) { - return LINGLONG_ERR("failed to merge modules", ret); + std::vector sources; + if (pendingSources) { + sources = *pendingSources; + } else if (projectLock) { + sources = projectLock->sources; } + auto complete = + completeProjectLock(std::move(sources), std::move(*baseEntry), std::move(runtimeEntry)); + auto written = writeCompleteProjectLock(projectYamlFile, complete); + if (!written) { + return LINGLONG_ERR("failed to write complete project lock", written); + } + + projectLock = std::move(complete); + pendingSources.reset(); return LINGLONG_OK; } @@ -782,13 +1148,15 @@ utils::error::Result Builder::processBuildDepends() noexcept runtime::BuilderContainerOptions options{ .common = runtime::CommonContainerOptions{ - .containerCachePath = this->workingDir / "linglong/cache", + .containerCachePath = directories.cache(), .extraMounts = std::vector{ - ocppi::runtime::config::types::Mount{ .destination = "/project", - .options = { { "rbind", "ro" } }, - .source = this->workingDir, - .type = "bind" }, + ocppi::runtime::config::types::Mount{ + .destination = directories.containerProject.string(), + .options = { { "rbind", "ro" } }, + .source = directories.project(), + .type = "bind", + }, }, }, .basePath = baseOverlay->mergedDirPath(), @@ -803,8 +1171,8 @@ utils::error::Result Builder::processBuildDepends() noexcept } auto process = ocppi::runtime::config::types::Process{}; - process.args = { "/bin/bash", "/project/linglong/buildext.sh" }; - process.cwd = "/project"; + process.args = { "/bin/bash", directories.containerBuildExtScript.string() }; + process.cwd = directories.containerProject.string(); process.noNewPrivileges = true; ocppi::runtime::RunOption opt{}; @@ -820,20 +1188,14 @@ utils::error::Result Builder::buildStagePreBuild() noexcept { LINGLONG_TRACE("build stage pre build"); - // clean output - QDir(QString::fromStdString(internalDir / "output")).removeRecursively(); - - buildOutput = internalDir / "output" / "_build"; - std::error_code ec; - std::filesystem::create_directories(buildOutput, ec); - if (ec) { - return LINGLONG_ERR( - fmt::format("failed to create directory {}: {}", buildOutput.string(), ec.message()) - .c_str()); + auto resetOutput = directories.resetOutput(); + if (!resetOutput) { + return LINGLONG_ERR(resetOutput); } LogD("create develop output success"); - auto res = buildContext.resolve(this->project.value(), buildOutput); + assert(projectLock.has_value()); + auto res = buildContext.resolve(this->project.value(), directories.buildOutput(), *projectLock); if (!res) { return LINGLONG_ERR(res); } @@ -842,20 +1204,38 @@ utils::error::Result Builder::buildStagePreBuild() noexcept if (!baseLayerPath) { return LINGLONG_ERR(baseLayerPath); } - // prepare overlayfs - auto overlayDir = internalDir / "overlay"; - baseOverlay = makeOverlay(*baseLayerPath / "files", overlayDir / "build_base"); + + std::optional runtimeLayerPath; + if (buildContext.hasRuntime()) { + auto resolved = buildContext.getRuntimeLayerPath(); + if (!resolved) { + return LINGLONG_ERR(resolved); + } + runtimeLayerPath = std::move(*resolved); + } + + auto state = projectLockDependencyState(*projectLock); + if (!state) { + return LINGLONG_ERR("failed to get project lock dependencies", state); + } + + baseOverlay.reset(); + runtimeOverlay.reset(); + auto prepared = detail::prepareBuildOverlays(directories, *state, runtimeLayerPath.has_value()); + if (!prepared) { + return LINGLONG_ERR(prepared); + } + LogD("build overlays: {}", *prepared ? "reusing existing state" : "reset for dependencies"); + + baseOverlay = makeOverlay(*baseLayerPath / "files", directories.buildBaseOverlay()); if (!baseOverlay) { return LINGLONG_ERR("failed to mount build base overlayfs"); } fixLocaltimeInOverlay(baseOverlay); - if (buildContext.hasRuntime()) { - auto runtimeLayerPath = buildContext.getRuntimeLayerPath(); - if (!runtimeLayerPath) { - return LINGLONG_ERR(runtimeLayerPath); - } - runtimeOverlay = makeOverlay(*runtimeLayerPath / "files", overlayDir / "build_runtime"); + if (runtimeLayerPath) { + runtimeOverlay = + makeOverlay(*runtimeLayerPath / "files", directories.buildRuntimeOverlay()); if (!runtimeOverlay) { return LINGLONG_ERR("failed to mount build runtime overlayfs"); } @@ -869,7 +1249,7 @@ utils::error::Result Builder::buildStagePreBuild() noexcept return LINGLONG_OK; } -utils::error::Result Builder::buildStageBuild(const QStringList &args) noexcept +utils::error::Result Builder::buildStageBuild(std::vector args) noexcept { LINGLONG_TRACE("build stage build"); @@ -895,12 +1275,14 @@ utils::error::Result Builder::buildStageBuild(const QStringList &args) noe runtime::BuilderContainerOptions containerOptions{ .common = runtime::CommonContainerOptions{ - .containerCachePath = this->workingDir / "linglong/cache", + .containerCachePath = directories.cache(), .extraMounts = std::vector{ - ocppi::runtime::config::types::Mount{ .destination = "/project", - .options = { { "rbind", "rw" } }, - .source = this->workingDir, - .type = "bind" }, + ocppi::runtime::config::types::Mount{ + .destination = directories.containerProject.string(), + .options = { { "rbind", "rw" } }, + .source = directories.project(), + .type = "bind", + }, ocppi::runtime::config::types::Mount{ .destination = LINGLONG_BUILDER_HELPER, .options = { { "rbind", "ro" } }, .source = LINGLONG_BUILDER_HELPER, @@ -910,8 +1292,8 @@ utils::error::Result Builder::buildStageBuild(const QStringList &args) noe .basePath = baseOverlay->mergedDirPath(), .isolateNetWork = this->buildOptions.isolateNetWork, .masks = { - "/project/linglong/output", - "/project/linglong/overlay", + directories.containerOutput.string(), + directories.containerOverlays.string(), }, .startContainerHooks = std::vector{ ocppi::runtime::config::types::Hook{ .path = "/sbin/ldconfig" }, @@ -928,14 +1310,9 @@ utils::error::Result Builder::buildStageBuild(const QStringList &args) noe } LogD("create container success"); - auto arguments = std::vector{}; - for (const auto &arg : args) { - arguments.push_back(arg.toStdString()); - } - auto process = ocppi::runtime::config::types::Process{}; - process.args = std::move(arguments); - process.cwd = "/project"; + process.args = std::move(args); + process.cwd = directories.containerProject.string(); process.env = { { "PREFIX=" + installPrefix, // During the build stage, we use overlayfs where /etc/ld.so.cache is safe @@ -977,16 +1354,23 @@ utils::error::Result Builder::buildStagePreCommit() noexcept takeTerminalForeground(); // clean prepare overlay - auto overlayDir = internalDir / "overlay"; - QDir(QString::fromStdString(overlayDir / "prepare_base")).removeRecursively(); - QDir(QString::fromStdString(overlayDir / "prepare_runtime")).removeRecursively(); + baseOverlay.reset(); + runtimeOverlay.reset(); + auto baseReset = directories.resetOverlay(directories.prepareBaseOverlay()); + if (!baseReset) { + return LINGLONG_ERR(baseReset); + } + auto runtimeReset = directories.resetOverlay(directories.prepareRuntimeOverlay()); + if (!runtimeReset) { + return LINGLONG_ERR(runtimeReset); + } // prepare overlay auto baseLayerPath = buildContext.getBaseLayerPath(); if (!baseLayerPath) { return LINGLONG_ERR(baseLayerPath); } - baseOverlay = makeOverlay(*baseLayerPath / "files", overlayDir / "prepare_base"); + baseOverlay = makeOverlay(*baseLayerPath / "files", directories.prepareBaseOverlay()); if (!baseOverlay) { return LINGLONG_ERR("failed to mount prepare base overlayfs"); } @@ -997,7 +1381,8 @@ utils::error::Result Builder::buildStagePreCommit() noexcept if (!runtimeLayerPath) { return LINGLONG_ERR(runtimeLayerPath); } - runtimeOverlay = makeOverlay(*runtimeLayerPath / "files", overlayDir / "prepare_runtime"); + runtimeOverlay = + makeOverlay(*runtimeLayerPath / "files", directories.prepareRuntimeOverlay()); if (!runtimeOverlay) { return LINGLONG_ERR("failed to mount prepare runtime overlayfs"); } @@ -1006,13 +1391,15 @@ utils::error::Result Builder::buildStagePreCommit() noexcept runtime::BuilderContainerOptions containerOptions{ .common = runtime::CommonContainerOptions{ - .containerCachePath = this->workingDir / "linglong/cache", + .containerCachePath = directories.cache(), .extraMounts = std::vector{ - ocppi::runtime::config::types::Mount{ .destination = "/project", - .options = { { "rbind", "rw" } }, - .source = this->workingDir, - .type = "bind" }, + ocppi::runtime::config::types::Mount{ + .destination = directories.containerProject.string(), + .options = { { "rbind", "rw" } }, + .source = directories.project(), + .type = "bind", + }, }, }, .basePath = baseOverlay->mergedDirPath(), @@ -1028,7 +1415,7 @@ utils::error::Result Builder::buildStagePreCommit() noexcept } ocppi::runtime::config::types::Process process; - process.args = std::vector{ std::string("bash"), std::string("/project/linglong/buildext.sh") }; + process.args = { "bash", directories.containerBuildExtScript.string() }; ocppi::runtime::RunOption opt{}; auto result = (*container)->run(process, opt); if (!result) { @@ -1044,7 +1431,28 @@ utils::error::Result Builder::buildStagePreCommit() noexcept return LINGLONG_ERR("base overlay missing upperdir"); } - std::vector src = { *baseOverlay->upperDirPath() / "usr" }; + const auto baseUpper = *baseOverlay->upperDirPath(); + std::error_code ec; + const auto baseUpperStatus = std::filesystem::status(baseUpper, ec); + if (ec) { + return LINGLONG_ERR(fmt::format("failed to inspect base overlay upperdir {}", baseUpper), + ec); + } + if (!std::filesystem::is_directory(baseUpperStatus)) { + return LINGLONG_ERR(fmt::format("base overlay upperdir is not a directory: {}", baseUpper)); + } + + // /usr can remain entirely in the lower layer if no changes were made there. + const auto baseUsr = baseUpper / "usr"; + const auto hasUsrChanges = std::filesystem::exists(baseUsr, ec); + if (ec) { + return LINGLONG_ERR(fmt::format("failed to inspect base overlay directory {}", baseUsr), + ec); + } + std::vector src; + if (hasUsrChanges) { + src.push_back(baseUsr); + } if (project.package.kind == "app" || project.package.kind == "extension") { if (runtimeOverlay) { if (!runtimeOverlay->upperDirPath()) { @@ -1053,12 +1461,10 @@ utils::error::Result Builder::buildStagePreCommit() noexcept src.push_back(*runtimeOverlay->upperDirPath()); } } - detail::mergeOutput(src, - buildOutput, - { "bin/", "sbin/", "lib/" }, - { "lib/systemd", "share/systemd" }); - - return LINGLONG_OK; + return detail::mergeOutput(src, + directories.buildOutput(), + { "bin/", "sbin/", "lib/" }, + { "lib/systemd", "share/systemd" }); } utils::error::Result Builder::generateAppConf() noexcept @@ -1076,7 +1482,7 @@ utils::error::Result Builder::generateAppConf() noexcept return LINGLONG_ERR(scriptFile); } auto output = utils::Cmd("bash").exec( - { "-e", scriptFile->string(), project.package.id, buildOutput.string() }); + { "-e", scriptFile->string(), project.package.id, directories.buildOutput().string() }); if (!output) { return LINGLONG_ERR(output); } @@ -1099,7 +1505,11 @@ utils::error::Result Builder::installFiles() noexcept 2); // 保存全量的develop, runtime需要对旧的ll-builder保持兼容 if (this->buildOptions.fullDevelop) { - utils::copyDirectory(buildOutput, internalDir / "output" / "develop" / "files"); + auto copied = + utils::copyDirectory(directories.buildOutput(), directories.module("develop") / "files"); + if (!copied) { + return LINGLONG_ERR("failed to copy develop module files", copied); + } } std::vector projectModules; @@ -1137,7 +1547,7 @@ utils::error::Result Builder::installFiles() noexcept for (const auto &module : projectModules) { std::unordered_set installRules; - auto moduleDir = internalDir / "output" / module.name; + auto moduleDir = directories.module(module.name); auto moduleFilesDir = moduleDir / "files"; std::error_code ec; std::filesystem::create_directories(moduleFilesDir, ec); @@ -1156,7 +1566,7 @@ utils::error::Result Builder::installFiles() noexcept for (const auto &file : module.files) { installRules.emplace(file); } - auto ret = installModule(buildOutput, moduleFilesDir, installRules); + auto ret = installModule(directories.buildOutput(), moduleFilesDir, installRules); if (!ret.has_value()) { return LINGLONG_ERR("install module", ret); } @@ -1202,7 +1612,14 @@ utils::error::Result Builder::generateEntries() noexcept // configuration file(linglong.yaml). // 仅导出名单中的目录，以避免意外文件影响系统功能 const std::filesystem::path exportDirConfigPath = LINGLONG_DATA_DIR "/export-dirs.json"; - if (!std::filesystem::exists(exportDirConfigPath)) { + std::error_code ec; + const auto exportDirConfigExists = std::filesystem::exists(exportDirConfigPath, ec); + if (ec) { + return LINGLONG_ERR(fmt::format("failed to check export config file {}: {}", + exportDirConfigPath, + ec.message())); + } + if (!exportDirConfigExists) { return LINGLONG_ERR( fmt::format("this export config file doesn't exist: {}", exportDirConfigPath)); } @@ -1215,8 +1632,8 @@ utils::error::Result Builder::generateEntries() noexcept exportDirConfig); } - QDir binaryFiles(QString::fromStdString(internalDir / "output" / "binary" / "files")); - QDir binaryEntries(QString::fromStdString(internalDir / "output" / "binary" / "entries")); + QDir binaryFiles(QString::fromStdString(directories.module("binary") / "files")); + QDir binaryEntries(QString::fromStdString(directories.module("binary") / "entries")); if (!binaryEntries.mkpath(".")) { return LINGLONG_ERR("make path " + binaryEntries.absolutePath().toStdString() @@ -1235,14 +1652,13 @@ utils::error::Result Builder::generateEntries() noexcept continue; } // appdata是旧版本的metainfo - if (path == "share/appdata") { - utils::copyDirectory(binaryFiles.absoluteFilePath(path).toStdString(), - binaryEntries.filePath("share/metainfo").toStdString()); - continue; + const auto entryPath = path == "share/appdata" ? binaryEntries.filePath("share/metainfo") + : binaryEntries.absoluteFilePath(path); + auto copied = utils::copyDirectory(binaryFiles.absoluteFilePath(path).toStdString(), + entryPath.toStdString()); + if (!copied) { + return LINGLONG_ERR("failed to copy application entries", copied); } - - utils::copyDirectory(binaryFiles.absoluteFilePath(path).toStdString(), - binaryEntries.absoluteFilePath(path).toStdString()); } if (binaryFiles.exists("lib/systemd/user")) { @@ -1256,8 +1672,12 @@ utils::error::Result Builder::generateEntries() noexcept LogW("mkpath files/lib/systemd/user: failed"); } - utils::copyDirectory(binaryFiles.filePath("lib/systemd/user").toStdString(), - binaryEntries.absoluteFilePath("lib/systemd/user").toStdString()); + auto copied = + utils::copyDirectory(binaryFiles.filePath("lib/systemd/user").toStdString(), + binaryEntries.absoluteFilePath("lib/systemd/user").toStdString()); + if (!copied) { + return LINGLONG_ERR("failed to copy systemd user entries", copied); + } } return LINGLONG_OK; @@ -1279,15 +1699,17 @@ utils::error::Result Builder::commitToLocalRepo() noexcept .kind = project.package.kind, .name = project.package.name, .permissions = project.permissions, - .runtime = {}, + .runtime = project.runtime, .schemaVersion = PACKAGE_INFO_VERSION, .version = project.package.version, }; - info.base = *project.base; - if (project.runtime) { - info.runtime = project.runtime; + assert(projectLock.has_value()); + const auto base = lockedDependency(*projectLock, "base"); + if (!base) { + return LINGLONG_ERR("project lock base dependency is missing"); } + info.base = base->requested; if (info.kind == "extension") { info.extImpl = api::types::v1::ExtensionImpl{ .deviceNodes = project.package.deviceNodes, @@ -1313,7 +1735,7 @@ utils::error::Result Builder::commitToLocalRepo() noexcept .toStdString(), 2); for (const auto &module : std::as_const(packageModules)) { - auto moduleOutput = internalDir / "output" / module; + auto moduleOutput = directories.module(module); info.packageInfoV2Module = module; auto ret = linglong::utils::calculateDirectorySize(moduleOutput); if (!ret) { @@ -1394,11 +1816,20 @@ utils::error::Result Builder::buildStageCommit() noexcept return true; } -utils::error::Result Builder::build(const QStringList &args) noexcept +utils::error::Result Builder::build(std::vector args) noexcept { - LINGLONG_TRACE(fmt::format("build project {}", this->workingDir / "linglong.yaml")); + LINGLONG_TRACE(fmt::format("build project {}", directories.project() / "linglong.yaml")); + + auto locked = lockProject(); + if (!locked) { + return LINGLONG_ERR(locked); + } auto &project = *this->project; + if (args.empty()) { + args.push_back(directories.containerEntryScript.string()); + } + utils::error::Result res; if (!(res = buildStagePrepare())) { return LINGLONG_ERR("stage prepare error", res); @@ -1413,7 +1844,7 @@ utils::error::Result Builder::build(const QStringList &args) noexcept } utils::error::Result success; - if (!(success = buildStageBuild(args))) { + if (!(success = buildStageBuild(std::move(args)))) { return LINGLONG_ERR("stage build error", success); } // skip build stage means skip all following stage @@ -1443,7 +1874,12 @@ utils::error::Result Builder::exportUAB(const ExportOption &option, { LINGLONG_TRACE("export uab file"); - auto exportWorkingDir = this->workingDir / ".uabBuild"; + auto locked = lockProject(); + if (!locked) { + return LINGLONG_ERR(locked); + } + + auto exportWorkingDir = directories.project() / ".uabBuild"; auto res = utils::ensureDirectory(exportWorkingDir); if (!res) { return LINGLONG_ERR("failed to ensure export working directory", res); @@ -1552,11 +1988,13 @@ utils::error::Result Builder::exportUAB(const ExportOption &option, LINGLONG_TRACE("use utils to bundle file"); std::error_code ec; - const auto relativeBundleFile = std::filesystem::relative(bundleFile, workingDir, ec); + const auto relativeBundleFile = + std::filesystem::relative(bundleFile, directories.project(), ec); if (ec) { return LINGLONG_ERR(fmt::format("failed to get relative path {}", bundleFile), ec); } - const auto relativeBundleDir = std::filesystem::relative(bundleDir, workingDir, ec); + const auto relativeBundleDir = + std::filesystem::relative(bundleDir, directories.project(), ec); if (ec) { return LINGLONG_ERR(fmt::format("failed to get relative path {}", bundleDir), ec); } @@ -1569,8 +2007,8 @@ utils::error::Result Builder::exportUAB(const ExportOption &option, "/opt/apps/cn.org.linyaps.builder.utils/files/bin/ll-builder-export", "--packdir", fmt::format("{}:{}", - std::filesystem::path{ "/project" } / relativeBundleDir, - std::filesystem::path{ "/project" } / relativeBundleFile), + directories.containerProject / relativeBundleDir, + directories.containerProject / relativeBundleFile), "-z", exportOpts.compressor, }; @@ -1593,7 +2031,7 @@ utils::error::Result Builder::exportUAB(const ExportOption &option, } } } else { - uabFile = workingDir / uabExportFilename(*curRef, exportOpts.mode); + uabFile = directories.project() / uabExportFilename(*curRef, exportOpts.mode); } std::optional temporaryMergedModuleDir; @@ -1667,6 +2105,11 @@ utils::error::Result Builder::exportLayer(const ExportOption &option) { LINGLONG_TRACE("export layer file"); + auto locked = lockProject(); + if (!locked) { + return LINGLONG_ERR(locked); + } + auto &project = *this->project; auto ref = currentReference(project); if (!ref) { @@ -1696,7 +2139,8 @@ utils::error::Result Builder::exportLayer(const ExportOption &option) continue; } - auto layerFile = QString::fromStdString(workingDir / layerExportFilename(*ref, module)); + auto layerFile = + QString::fromStdString(directories.project() / layerExportFilename(*ref, module)); auto ret = pkger.pack(*layerDir, layerFile); if (!ret) { LogE("export layer {}/{} failed: {}", ref->toString(), module, ret.error()); @@ -1742,6 +2186,11 @@ linglong::utils::error::Result Builder::push(const std::string &module, { LINGLONG_TRACE("push reference to remote repository"); + auto locked = lockProject(); + if (!locked) { + return LINGLONG_ERR(locked); + } + if (!this->project) { return LINGLONG_ERR("not under project"); } @@ -1803,6 +2252,11 @@ utils::error::Result Builder::run(std::vector modules, { LINGLONG_TRACE("run application"); + auto locked = lockProject(); + if (!locked) { + return LINGLONG_ERR(locked); + } + auto &project = *this->project; if (project.package.kind != "app") { return LINGLONG_ERR("only app can run"); @@ -1826,17 +2280,19 @@ utils::error::Result Builder::run(std::vector modules, } std::vector applicationMounts{ - ocppi::runtime::config::types::Mount{ .destination = "/project", - .options = { { "rbind", "rw" } }, - .source = this->workingDir, - .type = "bind" }, + ocppi::runtime::config::types::Mount{ + .destination = directories.containerProject.string(), + .options = { { "rbind", "rw" } }, + .source = directories.project(), + .type = "bind", + }, ocppi::runtime::config::types::Mount{ .destination = LINGLONG_BUILDER_HELPER, .options = { { "rbind", "ro" } }, .source = LINGLONG_BUILDER_HELPER, .type = "bind" }, }; - auto appCache = internalDir / "cache"; + auto appCache = directories.cache(); { // run init container @@ -1860,10 +2316,11 @@ utils::error::Result Builder::run(std::vector modules, if (debug) { // 生成 host_gdbinit 可使用 gdb --init-command=linglong/host_gdbinit 从宿主机调试 { - auto gdbinit = internalDir / "host_gdbinit"; - auto debugDir = internalDir / "output/develop/files/lib/debug"; + auto gdbinit = directories.root() / "host_gdbinit"; + auto debugDir = directories.module("develop") / "files/lib/debug"; std::ofstream hostConf(gdbinit); - hostConf << "set substitute-path /project " << workingDir << std::endl; + hostConf << "set substitute-path " << directories.containerProject << ' ' + << directories.project() << std::endl; hostConf << "set debug-file-directory " << debugDir << std::endl; hostConf << "# target remote :12345"; } @@ -1876,7 +2333,7 @@ utils::error::Result Builder::run(std::vector modules, std::string appPrefix = "/opt/apps/" + project.package.id + "/files"; std::string debugDir = "/usr/lib/debug:/runtime/lib/debug:" + appPrefix + "/lib/debug"; - auto gdbinit = internalDir / "gdbinit"; + auto gdbinit = directories.root() / "gdbinit"; std::ofstream f(gdbinit); f << "set debug-file-directory " << debugDir << std::endl; @@ -1939,7 +2396,7 @@ utils::error::Result Builder::runFromRepo(const package::Reference &ref, uid = getuid(); gid = getgid(); - auto appCache = internalDir / "cache" / ref.id; + auto appCache = directories.cache() / ref.id; { auto container = this->containerBuilder.createInitContainer( @@ -1967,9 +2424,9 @@ utils::error::Result Builder::runFromRepo(const package::Reference &ref, .extraMounts = std::vector{ ocppi::runtime::config::types::Mount{ - .destination = "/project", + .destination = directories.containerProject.string(), .options = { { "rbind", "rw" } }, - .source = this->workingDir, + .source = directories.project(), .type = "bind", }, }, @@ -2026,7 +2483,7 @@ utils::error::Result Builder::generateEntryScript() noexcept LINGLONG_TRACE("generate entry script"); auto &project = *this->project; - auto entry = internalDir / "entry.sh"; + const auto &entry = directories.entryScript(); std::string scriptContent = R"(#!/bin/bash set -e @@ -2083,6 +2540,7 @@ utils::error::Result Builder::generateBuildDependsScript() noexcept } if (!packages.empty()) { + content.append("set -e\n"); // ll-box current support single-user mode only, force apt use user content.append("echo 'APT::Sandbox::User \"root\";' > " "/etc/apt/apt.conf.d/99linglong-builder.conf\n"); @@ -2095,7 +2553,7 @@ utils::error::Result Builder::generateBuildDependsScript() noexcept } if (!content.empty()) { - auto res = utils::writeFile(internalDir / "buildext.sh", content); + auto res = utils::writeFile(directories.buildExtScript(), content); return res ? utils::error::Result(true) : LINGLONG_ERR(res); } @@ -2133,7 +2591,7 @@ utils::error::Result Builder::generateDependsScript() noexcept } if (!content.empty()) { - auto res = utils::writeFile(internalDir / "buildext.sh", content); + auto res = utils::writeFile(directories.buildExtScript(), content); return res ? utils::error::Result(true) : LINGLONG_ERR(res); } @@ -2195,7 +2653,7 @@ void Builder::printRepo() bool Builder::checkDeprecatedInstallFile() { - auto installFilepath = this->workingDir / (this->project->package.id + ".install"); + auto installFilepath = directories.project() / (this->project->package.id + ".install"); std::error_code ec; return !std::filesystem::exists(installFilepath, ec); } @@ -2224,23 +2682,14 @@ utils::error::Result Builder::cleanBuildArtifacts() noexcept { LINGLONG_TRACE("clean build artifacts"); - std::error_code ec; - if (!std::filesystem::exists(this->internalDir, ec)) { - return LINGLONG_OK; + auto locked = lockProject(); + if (!locked) { + return LINGLONG_ERR(locked); } - auto ret = utils::makeDirectoryTreeRemovable(this->internalDir); - if (!ret) { - return LINGLONG_ERR("failed to make directory tree removable", ret); - } - - std::filesystem::remove_all(this->internalDir, ec); - if (ec) { - return LINGLONG_ERR( - fmt::format("failed to remove {}: {}", this->internalDir.string(), ec.message())); - } - - return LINGLONG_OK; + baseOverlay.reset(); + runtimeOverlay.reset(); + return directories.clean(); } } // namespace linglong::builder diff --git a/libs/linglong/src/linglong/builder/linglong_builder.h b/libs/linglong/src/linglong/builder/linglong_builder.h index f7c484738..e384d934e 100644 --- a/libs/linglong/src/linglong/builder/linglong_builder.h +++ b/libs/linglong/src/linglong/builder/linglong_builder.h @@ -8,6 +8,8 @@ #include "linglong/api/types/v1/BuilderConfig.hpp" #include "linglong/api/types/v1/BuilderProject.hpp" +#include "linglong/builder/builder_directory.h" +#include "linglong/builder/project_lock.h" #include "linglong/repo/ostree_repo.h" #include "linglong/runtime/container_builder.h" #include "linglong/runtime/run_context.h" @@ -51,6 +53,8 @@ struct BuilderBuildOptions bool skipCheckOutput{ false }; bool skipStripSymbols{ false }; bool isolateNetWork{ false }; + bool locked{ false }; + bool offline{ false }; }; utils::error::Result> @@ -63,10 +67,10 @@ utils::error::Result cmdRemoveApp(repo::OSTreeRepo &repo, bool prune); namespace detail { -void mergeOutput(const std::vector &src, - const std::filesystem::path &dest, - const std::vector &targets, - const std::vector &excludes); +utils::error::Result mergeOutput(const std::vector &src, + const std::filesystem::path &dest, + const std::vector &targets, + const std::vector &excludes); utils::error::Result pullResolvedRef(const package::ReferenceWithRepo &refRepo, repo::OSTreeRepo &repo, const std::string &module) noexcept; @@ -80,6 +84,9 @@ clearDependency(const std::string &fuzzyRefStr, utils::error::Result pullDependency(const std::string &fuzzyRefStr, repo::OSTreeRepo &repo, const std::string &module) noexcept; +auto prepareBuildOverlays(const BuilderDirectoryLayout &directories, + const std::string &dependencyState, + bool hasRuntime) -> utils::error::Result; } // namespace detail class Builder @@ -98,8 +105,9 @@ class Builder auto create(const QString &projectName) -> utils::error::Result; - auto build(const QStringList &args = { "/project/linglong/entry.sh" }) noexcept - -> utils::error::Result; + auto build(std::vector args = {}) noexcept -> utils::error::Result; + + auto processProjectLock(ProjectLockMode mode) noexcept -> utils::error::Result; auto exportUAB(const ExportOption &option, const std::filesystem::path &outputFile = {}) -> utils::error::Result; @@ -123,8 +131,6 @@ class Builder const std::string &workdir = "", std::vector extensions = {}) -> utils::error::Result; auto runtimeCheck() -> utils::error::Result; - auto runFromRepo(const package::Reference &ref, const std::vector &args) - -> utils::error::Result; void setBuildOptions(const BuilderBuildOptions &options) noexcept { buildOptions = options; } @@ -134,12 +140,29 @@ class Builder std::string uabExportFilename(const linglong::package::Reference &ref, ExportMode mode); std::string layerExportFilename(const linglong::package::Reference &ref, const std::string &module); + auto buildStageFetchSource() noexcept -> utils::error::Result; private: + auto lockProject() noexcept -> utils::error::Result; auto buildStagePrepare() noexcept -> utils::error::Result; - auto buildStageFetchSource() noexcept -> utils::error::Result; + auto sourceCachePath() const -> std::filesystem::path; utils::error::Result buildStagePullDependency() noexcept; - utils::error::Result buildStageBuild(const QStringList &args) noexcept; + auto fetchProjectDependency(const DependencyFetchRequest &request) noexcept + -> utils::error::Result; + auto fetchResolvedProjectDependency(const ResolveDependencyRequest &request) noexcept + -> utils::error::Result; + auto ensureDependencyModule(const package::Reference &reference, + const std::string &module, + std::optional &remote) noexcept + -> utils::error::Result; + auto recordProjectDependency(const DependencyFetchRequest &request, + const package::Reference &reference) noexcept + -> utils::error::Result; + auto finishProjectLockDependencies(const ProjectDependencyPlan &plan, + const package::Reference &base, + const std::optional &runtime) noexcept + -> utils::error::Result; + utils::error::Result buildStageBuild(std::vector args) noexcept; utils::error::Result buildStagePreBuild() noexcept; utils::error::Result buildStagePreCommit() noexcept; utils::error::Result buildStageCommit() noexcept; @@ -154,6 +177,8 @@ class Builder void fixLocaltimeInOverlay(std::unique_ptr &base); utils::error::Result ensureUtils(const std::string &id, const package::Architecture &arch) noexcept; + auto runFromRepo(const package::Reference &ref, const std::vector &args) + -> utils::error::Result; auto generateEntryScript() noexcept -> utils::error::Result; auto generateBuildDependsScript() noexcept -> utils::error::Result; auto generateDependsScript() noexcept -> utils::error::Result; @@ -164,12 +189,18 @@ class Builder private: repo::OSTreeRepo &repo; - std::filesystem::path workingDir; - std::filesystem::path internalDir; + BuilderDirectoryLayout directories; std::optional project; runtime::ContainerBuilder &containerBuilder; api::types::v1::BuilderConfig cfg; BuilderBuildOptions buildOptions; + // Acquired lazily and kept until this Builder and its overlays are destroyed. + std::optional projectMutex; + std::optional projectYamlMutex; + // An existing or fully resolved lock; absent until fresh sources and dependencies are ready. + std::optional projectLock; + // Source changes waiting for dependency resolution, including an empty source list. + std::optional> pendingSources; int64_t uid; int64_t gid; @@ -178,7 +209,6 @@ class Builder std::vector packageModules; std::unique_ptr baseOverlay; std::unique_ptr runtimeOverlay; - std::filesystem::path buildOutput; std::string installPrefix; runtime::RunContext buildContext; diff --git a/libs/linglong/src/linglong/builder/project_lock.cpp b/libs/linglong/src/linglong/builder/project_lock.cpp new file mode 100644 index 000000000..d34c38064 --- /dev/null +++ b/libs/linglong/src/linglong/builder/project_lock.cpp @@ -0,0 +1,580 @@ +// SPDX-FileCopyrightText: 2026 UnionTech Software Technology Co., Ltd. +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "project_lock.h" + +#include "linglong/api/types/v1/Generators.hpp" +#include "linglong/builder/source_cache.h" +#include "linglong/builder/source_fetcher.h" +#include "linglong/package/architecture.h" +#include "linglong/package/fuzzy_reference.h" +#include "linglong/package/reference.h" +#include "linglong/utils/log/log.h" +#include "linglong/utils/serialize/yaml.h" +#include "ytj/ytj.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace linglong::builder { +namespace { + +using api::types::v1::BuilderProjectLock; +using api::types::v1::BuilderProjectSource; +using api::types::v1::BuilderSourceLockEntry; +using api::types::v1::BuilderSourceLockResolved; + +BuilderProjectSource normalizedSource(const BuilderProjectSource &source) +{ + auto normalized = source; + if (normalized.digest) { + if (normalized.digest->rfind("sha256:", 0) != 0) { + *normalized.digest = "sha256:" + *normalized.digest; + } + std::transform(normalized.digest->begin(), + normalized.digest->end(), + normalized.digest->begin(), + [](unsigned char character) { + return static_cast(std::tolower(character)); + }); + } + + if (source.kind == "git") { + normalized.submodules = source.submodules.value_or(true); + } else { + normalized.commit.reset(); + normalized.submodules.reset(); + } + + return normalized; +} + +bool sourceFetchEqual(const BuilderProjectSource &left, const BuilderProjectSource &right) +{ + // Output names and version descriptions do not affect the fetched content. + return left.commit == right.commit && left.digest == right.digest && left.kind == right.kind + && left.submodules == right.submodules && left.url == right.url; +} + +auto matchLockEntries(const std::vector &sources, + const BuilderProjectLock &lock) + -> utils::error::Result> +{ + LINGLONG_TRACE("match project sources with project lock"); + + if (lock.sources.size() != sources.size()) { + return LINGLONG_ERR("sources differ from linglong.lock; run ll-builder update-lockfile"); + } + + std::vector used(lock.sources.size(), false); + std::vector resolvedSources; + resolvedSources.reserve(sources.size()); + for (const auto &source : sources) { + const auto normalized = normalizedSource(source); + bool found = false; + for (std::size_t index = 0; index < lock.sources.size(); ++index) { + const auto &entry = lock.sources[index]; + if (used[index] || !sourceFetchEqual(normalizedSource(entry.source), normalized)) { + continue; + } + if (source.kind == "git" && !gitCommitFromHash(entry.resolved.hash)) { + return LINGLONG_ERR("git source has an invalid hash in linglong.lock; run " + "ll-builder update-lockfile"); + } + + used[index] = true; + resolvedSources.push_back(entry.resolved); + found = true; + break; + } + + if (!found) { + return LINGLONG_ERR( + "sources differ from linglong.lock; run ll-builder update-lockfile"); + } + } + + return resolvedSources; +} + +bool validDependencies(const BuilderProjectLock &lock) +{ + std::set types; + for (const auto &entry : lock.dependencies) { + const auto reference = lockedDependencyReference(entry); + if ((entry.type != "base" && entry.type != "runtime") || !types.insert(entry.type).second + || !reference || entry.resolved.modules.empty()) { + return false; + } + + std::set modules; + for (const auto &module : entry.resolved.modules) { + if (module.empty() || !modules.insert(module).second) { + return false; + } + } + + if (modules.count("binary") == 0 && modules.count("runtime") == 0) { + return false; + } + } + + return types.count("base") == 1; +} + +utils::error::Result validateSource(const api::types::v1::BuilderProjectSource &source) +{ + LINGLONG_TRACE("validate locked source"); + + if (source.kind != "git" && source.kind != "file" && source.kind != "archive" + && source.kind != "dsc") { + return LINGLONG_ERR("unsupported source kind: " + source.kind); + } + if (!source.url) { + return LINGLONG_ERR("source URL is missing"); + } + if (source.kind == "git" && !source.commit) { + return LINGLONG_ERR("git source commit is missing"); + } + if (source.kind == "git" && source.digest) { + return LINGLONG_ERR("git source does not support digest"); + } + + if (source.digest) { + auto value = *source.digest; + if (value.rfind("sha256:", 0) == 0) { + value.erase(0, 7); + } + if (value.size() != 64 + || !std::all_of(value.begin(), value.end(), [](unsigned char character) { + return std::isxdigit(character) != 0; + })) { + return LINGLONG_ERR("source digest must be a SHA-256 value"); + } + } + + const auto name = sourceOutputName(source); + if (name.empty() || name == "." || name == ".." || name == ".git" + || name.find('/') != std::string::npos || name.find('\\') != std::string::npos) { + return LINGLONG_ERR("source name must be a single path component"); + } + + return name; +} + +auto validateSources(const std::vector &sources) -> utils::error::Result +{ + LINGLONG_TRACE("validate project sources"); + + std::set outputNames; + std::vector requests; + requests.reserve(sources.size()); + for (const auto &source : sources) { + auto name = validateSource(source); + if (!name) { + return LINGLONG_ERR(name); + } + if (!outputNames.insert(*name).second) { + return LINGLONG_ERR("multiple sources use the same output name: " + *name); + } + + auto normalized = normalizedSource(source); + const auto duplicate = std::find_if(requests.begin(), requests.end(), [&](const auto &old) { + return sourceFetchEqual(old, normalized); + }); + if (duplicate != requests.end()) { + return LINGLONG_ERR("duplicate source fetch request: " + sourceOutputName(*duplicate) + + " and " + *name); + } + + requests.push_back(std::move(normalized)); + } + + return LINGLONG_OK; +} + +auto ensureSourceDestination(SourceCacheLayout &cache, const std::filesystem::path &destination) + -> utils::error::Result +{ + LINGLONG_TRACE("ensure source destination"); + + auto ensured = cache.ensure(); + if (!ensured) { + return LINGLONG_ERR(ensured); + } + + std::error_code destinationError; + std::filesystem::create_directories(destination, destinationError); + if (destinationError) { + return LINGLONG_ERR("failed to create source destination: " + destinationError.message()); + } + + return LINGLONG_OK; +} + +} // namespace + +auto projectLockPath(const std::filesystem::path &manifest) -> std::filesystem::path +{ + auto result = manifest; + result.replace_extension(".lock"); + return result; +} + +auto lockedDependency(const BuilderProjectLock &lock, const std::string &type) + -> std::optional +{ + for (const auto &entry : lock.dependencies) { + if (entry.type == type) { + return entry; + } + } + return std::nullopt; +} + +auto readProjectLock(const std::filesystem::path &path) noexcept + -> utils::error::Result +{ + LINGLONG_TRACE("read project lock"); + + auto loaded = utils::serialize::LoadYAMLFile(path); + if (!loaded) { + return LINGLONG_ERR("failed to read " + path.string() + "; run ll-builder update-lockfile", + loaded); + } + + if (loaded->version != 1) { + return LINGLONG_ERR("unsupported project lock version; run ll-builder update-lockfile: " + + path.string()); + } + + if (!validDependencies(*loaded)) { + return LINGLONG_ERR( + "project lock dependencies are incomplete; run ll-builder update-lockfile"); + } + + return std::move(*loaded); +} + +auto completeProjectLock(std::vector sources, + LockedDependency base, + std::optional runtime) -> BuilderProjectLock +{ + BuilderProjectLock lock; + lock.version = 1; + lock.sources = std::move(sources); + lock.dependencies = { std::move(base) }; + if (runtime) { + lock.dependencies.push_back(std::move(*runtime)); + } + return lock; +} + +auto writeCompleteProjectLock(const std::filesystem::path &manifest, + const BuilderProjectLock &lock) noexcept -> utils::error::Result +{ + LINGLONG_TRACE("write complete project lock"); + + try { + if (!validDependencies(lock)) { + return LINGLONG_ERR("project lock dependencies are incomplete"); + } + + const auto output = ytj::to_yaml(lock); + const auto path = projectLockPath(manifest); + auto temporary = path; + temporary += ".tmp." + std::to_string(::getpid()); + + { + std::ofstream file(temporary, std::ios::binary | std::ios::trunc); + if (!file) { + return LINGLONG_ERR("failed to create " + temporary.string()); + } + file << output << '\n'; + file.flush(); + if (!file) { + return LINGLONG_ERR("failed to write " + temporary.string()); + } + } + + std::error_code ec; + std::filesystem::rename(temporary, path, ec); + if (ec) { + std::filesystem::remove(temporary); + return LINGLONG_ERR("failed to replace " + path.string() + ": " + ec.message()); + } + return LINGLONG_OK; + } catch (...) { + return LINGLONG_ERR(std::current_exception()); + } +} + +auto lockedDependencyReference(const LockedDependency &entry) noexcept + -> utils::error::Result +{ + LINGLONG_TRACE("resolve locked dependency architecture"); + + auto locked = package::FuzzyReference::parse(entry.resolved.reference); + if (!locked) { + return LINGLONG_ERR(locked); + } + if (!locked->channel || !locked->version) { + return LINGLONG_ERR("locked dependency must contain channel, ID and exact version; " + "run ll-builder update-lockfile"); + } + auto version = package::Version::parse(*locked->version); + if (!version) { + return LINGLONG_ERR(version); + } + + return package::Reference::create( + *locked->channel, + locked->id, + *version, + locked->arch.value_or(package::Architecture::currentCPUArchitecture())); +} + +auto projectLockDependencyState(const BuilderProjectLock &lock) noexcept + -> utils::error::Result +{ + LINGLONG_TRACE("get project lock dependency state"); + + try { + if (!validDependencies(lock)) { + return LINGLONG_ERR("project lock dependencies are incomplete"); + } + + auto state = nlohmann::json::object(); + for (const auto &entry : lock.dependencies) { + auto reference = lockedDependencyReference(entry); + if (!reference) { + return LINGLONG_ERR(reference); + } + + auto resolved = entry.resolved; + resolved.reference = reference->toString(); + std::sort(resolved.modules.begin(), resolved.modules.end()); + state[entry.type] = std::move(resolved); + } + + return state.dump(); + } catch (...) { + return LINGLONG_ERR(std::current_exception()); + } +} + +namespace { + +auto planDependency(const std::optional &requested, + const std::optional &entry, + const std::string &type, + bool locked) -> utils::error::Result> +{ + LINGLONG_TRACE("check locked dependency declaration"); + + if (!requested && !entry) { + return std::optional{}; + } + if (requested && entry && entry->requested == *requested) { + return std::optional{ LockedDependencyRequest{ *entry } }; + } + + if (locked) { + return LINGLONG_ERR(type + " differs from linglong.lock; run ll-builder update-lockfile"); + } + if (!requested) { + return std::optional{}; + } + return std::optional{ ResolveDependencyRequest{ *requested, type } }; +} + +} // namespace + +auto planProjectDependencies(const std::optional &base, + const std::optional &runtime, + const std::optional &existingLock, + bool locked) noexcept -> utils::error::Result +{ + LINGLONG_TRACE("plan project dependencies"); + + if (!base && !runtime) { + return LINGLONG_ERR("at least one of base or runtime must be specified"); + } + + if (existingLock && !validDependencies(*existingLock)) { + return LINGLONG_ERR("project lock dependencies are incomplete or inconsistent; run " + "ll-builder update-lockfile"); + } + + ProjectDependencyPlan plan; + plan.changed = !existingLock; + + const auto oldRuntime = + existingLock ? lockedDependency(*existingLock, "runtime") : std::nullopt; + auto runtimeRequest = planDependency(runtime, oldRuntime, "runtime", locked); + if (!runtimeRequest) { + return LINGLONG_ERR(runtimeRequest); + } + plan.runtime = std::move(*runtimeRequest); + plan.changed |= plan.runtime ? std::holds_alternative(*plan.runtime) + : oldRuntime.has_value(); + + if (!base) { + return plan; + } + + const auto oldBase = existingLock ? lockedDependency(*existingLock, "base") : std::nullopt; + auto baseRequest = planDependency(base, oldBase, "base", locked); + if (!baseRequest) { + return LINGLONG_ERR(baseRequest); + } + plan.base = std::move(*baseRequest); + plan.changed |= std::holds_alternative(*plan.base); + + return plan; +} + +auto resolveProjectLockSources(const std::optional> &sources, + const std::optional &existingLock) noexcept + -> utils::error::Result +{ + LINGLONG_TRACE("plan project lock sources"); + + ProjectSourcePlan plan; + if (!sources || sources->empty()) { + plan.changed = !existingLock || !existingLock->sources.empty(); + return plan; + } + + auto validated = validateSources(*sources); + if (!validated) { + return LINGLONG_ERR(validated); + } + + plan.changed = !existingLock; + plan.requests.reserve(sources->size()); + std::vector used(existingLock ? existingLock->sources.size() : 0, false); + for (const auto &source : *sources) { + const BuilderSourceLockEntry *matched = nullptr; + if (existingLock) { + const auto normalized = normalizedSource(source); + for (std::size_t old = 0; old < existingLock->sources.size(); ++old) { + if (used[old] + || !sourceFetchEqual(normalizedSource(existingLock->sources[old].source), + normalized)) { + continue; + } + if (source.kind == "git" + && !gitCommitFromHash(existingLock->sources[old].resolved.hash)) { + continue; + } + used[old] = true; + matched = &existingLock->sources[old]; + break; + } + } + + if (matched) { + plan.requests.emplace_back(LockedSourceRequest{ source, matched->resolved }); + } else { + plan.requests.emplace_back(GenerateSourceRequest{ source }); + plan.changed = true; + } + } + + if (std::any_of(used.begin(), used.end(), [](bool matched) { + return !matched; + })) { + plan.changed = true; + } + + return plan; +} + +auto verifyProjectLockSources(const std::optional> &sources, + const BuilderProjectLock &lock) noexcept + -> utils::error::Result +{ + LINGLONG_TRACE("verify project lock sources"); + + if (!sources || sources->empty()) { + if (!lock.sources.empty()) { + return LINGLONG_ERR( + "sources differ from linglong.lock; run ll-builder update-lockfile"); + } + return ProjectSourcePlan{}; + } + + auto validated = validateSources(*sources); + if (!validated) { + return LINGLONG_ERR(validated); + } + + auto matched = matchLockEntries(*sources, lock); + if (!matched) { + return LINGLONG_ERR(matched); + } + + ProjectSourcePlan plan; + plan.requests.reserve(sources->size()); + for (std::size_t index = 0; index < sources->size(); ++index) { + plan.requests.emplace_back(LockedSourceRequest{ (*sources)[index], (*matched)[index] }); + } + + return plan; +} + +auto fetchProjectSources(const ProjectSourcePlan &plan, + const std::filesystem::path &cache, + const std::filesystem::path &destination, + bool offline, + const SourceFetchProgress &progress) noexcept + -> utils::error::Result> +{ + LINGLONG_TRACE("fetch project sources"); + + SourceCacheLayout cacheLayout(cache); + auto preparedDestination = ensureSourceDestination(cacheLayout, destination); + if (!preparedDestination) { + return LINGLONG_ERR(preparedDestination); + } + + std::vector entries; + entries.reserve(plan.requests.size()); + for (std::size_t index = 0; index < plan.requests.size(); ++index) { + const auto &request = plan.requests[index]; + const auto &source = std::visit( + [](const auto &request) -> const BuilderProjectSource & { + return request.source; + }, + request); + const auto report = [&](SourceFetchStatus status) { + if (progress) { + progress(index, source, status); + } + }; + + report(SourceFetchStatus::Fetching); + SourceFetcher fetcher(request, cacheLayout); + auto fetched = fetcher.fetch(destination, offline); + if (!fetched) { + report(SourceFetchStatus::Failed); + return LINGLONG_ERR(fetched); + } + + entries.push_back(BuilderSourceLockEntry{ std::move(*fetched), normalizedSource(source) }); + report(SourceFetchStatus::Complete); + } + + return entries; +} + +} // namespace linglong::builder diff --git a/libs/linglong/src/linglong/builder/project_lock.h b/libs/linglong/src/linglong/builder/project_lock.h new file mode 100644 index 000000000..fd19bbea6 --- /dev/null +++ b/libs/linglong/src/linglong/builder/project_lock.h @@ -0,0 +1,103 @@ +// SPDX-FileCopyrightText: 2026 UnionTech Software Technology Co., Ltd. +// SPDX-License-Identifier: LGPL-3.0-or-later + +#pragma once + +#include "linglong/api/types/v1/BuilderProjectLock.hpp" +#include "linglong/api/types/v1/BuilderProjectSource.hpp" +#include "linglong/builder/source_fetcher.h" +#include "linglong/package/reference.h" +#include "linglong/utils/error/error.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace linglong::builder { + +enum class ProjectLockMode { Generate, Update }; + +using LockedDependency = api::types::v1::BuilderProjectLockDependency; + +struct ProjectSourcePlan +{ + std::vector requests; + bool changed{ false }; +}; + +enum class SourceFetchStatus { Fetching, Complete, Failed }; + +using SourceFetchProgress = + std::function; + +struct ResolveDependencyRequest +{ + std::string requested; + std::string type; +}; + +struct LockedDependencyRequest +{ + LockedDependency entry; +}; + +using DependencyFetchRequest = std::variant; + +struct ProjectDependencyPlan +{ + // An absent base request must be completed from the fetched runtime's package information. + std::optional base; + std::optional runtime; + bool changed{ false }; +}; + +auto projectLockPath(const std::filesystem::path &manifest) -> std::filesystem::path; + +auto readProjectLock(const std::filesystem::path &path) noexcept + -> utils::error::Result; + +auto resolveProjectLockSources( + const std::optional> &sources, + const std::optional &existingLock) noexcept + -> utils::error::Result; + +auto verifyProjectLockSources( + const std::optional> &sources, + const api::types::v1::BuilderProjectLock &lock) noexcept + -> utils::error::Result; + +auto fetchProjectSources(const ProjectSourcePlan &plan, + const std::filesystem::path &cache, + const std::filesystem::path &destination, + bool offline, + const SourceFetchProgress &progress = {}) noexcept + -> utils::error::Result>; + +auto completeProjectLock(std::vector sources, + LockedDependency base, + std::optional runtime = std::nullopt) + -> api::types::v1::BuilderProjectLock; + +auto writeCompleteProjectLock(const std::filesystem::path &manifest, + const api::types::v1::BuilderProjectLock &lock) noexcept + -> utils::error::Result; + +auto lockedDependencyReference(const LockedDependency &entry) noexcept + -> utils::error::Result; + +auto projectLockDependencyState(const api::types::v1::BuilderProjectLock &lock) noexcept + -> utils::error::Result; + +auto lockedDependency(const api::types::v1::BuilderProjectLock &lock, const std::string &type) + -> std::optional; + +auto planProjectDependencies(const std::optional &base, + const std::optional &runtime, + const std::optional &existingLock, + bool locked) noexcept -> utils::error::Result; + +} // namespace linglong::builder diff --git a/libs/linglong/src/linglong/builder/source_cache.cpp b/libs/linglong/src/linglong/builder/source_cache.cpp new file mode 100644 index 000000000..6869c18e5 --- /dev/null +++ b/libs/linglong/src/linglong/builder/source_cache.cpp @@ -0,0 +1,77 @@ +// SPDX-FileCopyrightText: 2026 UnionTech Software Technology Co., Ltd. +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include "source_cache.h" + +#include +#include +#include +#include + +namespace linglong::builder { + +SourceCacheLayout::SourceCacheLayout(std::filesystem::path cache) + : cacheRoot(std::move(cache) / "sources/v1") + , temporaryRoot(cacheRoot / "tmp") +{ +} + +auto SourceCacheLayout::object(std::string_view sourceKind, std::string_view hash) const + -> std::filesystem::path +{ + const auto separator = hash.rfind(':'); + if (separator == std::string_view::npos) { + return cacheRoot / std::string(sourceKind) / std::string(hash); + } + + auto algorithm = std::string(hash.substr(0, separator)); + std::replace(algorithm.begin(), algorithm.end(), ':', '-'); + return cacheRoot / std::string(sourceKind) / algorithm + / std::string(hash.substr(separator + 1)); +} + +auto SourceCacheLayout::gitObject(std::string_view commit, bool submodules) const + -> std::filesystem::path +{ + auto normalizedCommit = std::string(commit); + std::transform(normalizedCommit.begin(), + normalizedCommit.end(), + normalizedCommit.begin(), + [](unsigned char value) { + return static_cast(std::tolower(value)); + }); + + return cacheRoot / "git/commit" + / (normalizedCommit + (submodules ? "-submodules" : "-no-submodules")); +} + +auto SourceCacheLayout::lockFile(const std::filesystem::path &entry) const -> std::filesystem::path +{ + auto path = cacheRoot / "locks" / entry.lexically_relative(cacheRoot); + path += ".lock"; + return path; +} + +auto SourceCacheLayout::ensure() noexcept -> utils::error::Result +{ + LINGLONG_TRACE("ensure source cache directories"); + + std::error_code ec; + auto root = std::filesystem::absolute(cacheRoot, ec); + if (ec) { + return LINGLONG_ERR("failed to resolve source cache directory: " + ec.message()); + } + + auto temporary = root / "tmp"; + std::filesystem::create_directories(temporary, ec); + if (ec) { + return LINGLONG_ERR("failed to create source cache directories: " + ec.message()); + } + + cacheRoot = std::move(root); + temporaryRoot = std::move(temporary); + + return LINGLONG_OK; +} + +} // namespace linglong::builder diff --git a/libs/linglong/src/linglong/builder/source_cache.h b/libs/linglong/src/linglong/builder/source_cache.h new file mode 100644 index 000000000..aeec8690d --- /dev/null +++ b/libs/linglong/src/linglong/builder/source_cache.h @@ -0,0 +1,36 @@ +// SPDX-FileCopyrightText: 2026 UnionTech Software Technology Co., Ltd. +// SPDX-License-Identifier: LGPL-3.0-or-later + +#pragma once + +#include "linglong/utils/error/error.h" + +#include +#include + +namespace linglong::builder { + +class SourceCacheLayout +{ +public: + explicit SourceCacheLayout(std::filesystem::path cacheRoot); + + const std::filesystem::path &root() const noexcept { return cacheRoot; } + + const std::filesystem::path &temporary() const noexcept { return temporaryRoot; } + + auto object(std::string_view sourceKind, std::string_view hash) const -> std::filesystem::path; + + auto gitObject(std::string_view commit, bool submodules) const -> std::filesystem::path; + + // Lock files remain in place when their cache entries are replaced. + auto lockFile(const std::filesystem::path &entry) const -> std::filesystem::path; + + auto ensure() noexcept -> utils::error::Result; + +private: + std::filesystem::path cacheRoot; + std::filesystem::path temporaryRoot; +}; + +} // namespace linglong::builder diff --git a/libs/linglong/src/linglong/builder/source_fetcher.cpp b/libs/linglong/src/linglong/builder/source_fetcher.cpp index 09db4c186..0d3aa5b66 100644 --- a/libs/linglong/src/linglong/builder/source_fetcher.cpp +++ b/libs/linglong/src/linglong/builder/source_fetcher.cpp @@ -6,87 +6,977 @@ #include "source_fetcher.h" -#include "linglong/builder/builder_script.h" +#include "builder_script.h" #include "linglong/common/formatter.h" +#include "linglong/utils/cmd.h" #include "linglong/utils/error/error.h" +#include "linglong/utils/file.h" +#include "linglong/utils/filelock.h" #include "linglong/utils/log/log.h" +#include "linglong/utils/sha256.h" +#include "linglong/utils/temporary_directory.h" -#include +#include +#include +#include +#include +#include +#include +#include +#include namespace linglong::builder { -auto SourceFetcher::fetch(QDir destination) noexcept -> utils::error::Result +namespace { + +using api::types::v1::BuilderSourceLockResolved; + +std::string finishHash(digest::SHA256 &hash) { - LINGLONG_TRACE("fetch source"); + std::array value{}; + hash.final(value.data()); - if (!destination.mkpath(".")) { - return LINGLONG_ERR(destination.absolutePath().toStdString() - + "source directory failed to create."); + std::ostringstream output; + output << std::hex << std::setfill('0'); + for (auto byte : value) { + output << std::setw(2) << std::to_integer(byte); } - if (this->source.kind != "git" && this->source.kind != "dsc" && this->source.kind != "file" - && this->source.kind != "archive") { - return LINGLONG_ERR("unknown source kind"); + return output.str(); +} + +int fromHex(char character) +{ + if (character >= '0' && character <= '9') { + return character - '0'; + } + if (character >= 'a' && character <= 'f') { + return character - 'a' + 10; + } + if (character >= 'A' && character <= 'F') { + return character - 'A' + 10; + } + return -1; +} + +std::string percentDecode(std::string_view value) +{ + std::string decoded; + decoded.reserve(value.size()); + for (std::size_t index = 0; index < value.size(); ++index) { + if (value[index] == '%' && index + 2 < value.size()) { + const auto high = fromHex(value[index + 1]); + const auto low = fromHex(value[index + 2]); + if (high >= 0 && low >= 0) { + decoded.push_back(static_cast((high << 4) | low)); + index += 2; + continue; + } + } + + decoded.push_back(value[index]); + } + + return decoded; +} + +std::string trim(std::string value) +{ + const auto notSpace = [](unsigned char character) { + return !std::isspace(character); + }; + value.erase(value.begin(), std::find_if(value.begin(), value.end(), notSpace)); + value.erase(std::find_if(value.rbegin(), value.rend(), notSpace).base(), value.end()); + return value; +} + +std::string lastOutputLine(const std::string &output) +{ + auto trimmed = trim(output); + const auto separator = trimmed.rfind('\n'); + if (separator != std::string::npos) { + trimmed.erase(0, separator + 1); + } + return trim(std::move(trimmed)); +} + +auto normalizeSha256(std::optional hash) -> std::optional +{ + if (hash && hash->rfind("sha256:", 0) != 0) { + *hash = "sha256:" + *hash; + } + if (hash) { + std::transform(hash->begin(), hash->end(), hash->begin(), [](unsigned char character) { + return static_cast(std::tolower(character)); + }); + } + return hash; +} + +bool isSha256Hash(const std::string &hash, std::string_view prefix) +{ + if (hash.rfind(prefix, 0) != 0 || hash.size() != prefix.size() + 64) { + return false; + } + + return std::all_of(hash.begin() + static_cast(prefix.size()), + hash.end(), + [](unsigned char character) { + return std::isxdigit(character) != 0; + }); +} + +utils::error::Result copyTree(const std::filesystem::path &source, + const std::filesystem::path &destination) noexcept +{ + LINGLONG_TRACE("copy source tree"); + + std::error_code ec; + std::filesystem::remove_all(destination, ec); + if (ec) { + return LINGLONG_ERR("failed to remove source tree destination: " + ec.message()); + } + + auto copied = utils::copyDirectory(source, destination); + if (!copied) { + return LINGLONG_ERR(copied); + } + + return LINGLONG_OK; +} + +auto lockCacheEntry(const SourceCacheLayout &cache, const std::filesystem::path &entry) noexcept + -> utils::error::Result +{ + LINGLONG_TRACE("lock source cache entry"); + + const auto path = cache.lockFile(entry); + std::error_code ec; + std::filesystem::create_directories(path.parent_path(), ec); + if (ec) { + return LINGLONG_ERR("failed to create source cache lock directory: " + ec.message()); + } + + auto lock = utils::filelock::FileLock::create(path, utils::filelock::LockType::Write); + if (!lock) { + return LINGLONG_ERR(lock); + } + auto locked = lock->lock(utils::filelock::LockType::Write); + if (!locked) { + return LINGLONG_ERR(locked); + } + + return lock; +} + +template +auto publishCacheEntry(const std::filesystem::path &staging, + const std::filesystem::path &destination, + Validator valid) noexcept -> utils::error::Result +{ + LINGLONG_TRACE("publish source cache entry"); + + // The caller holds the entry lock through publication and materialization. + if (valid(destination)) { + return LINGLONG_OK; + } + + std::error_code ec; + std::filesystem::create_directories(destination.parent_path(), ec); + if (ec) { + return LINGLONG_ERR("failed to create source cache directory: " + ec.message()); + } + std::filesystem::remove_all(destination, ec); + if (ec) { + return LINGLONG_ERR("failed to remove invalid source cache entry: " + ec.message()); + } + + std::filesystem::rename(staging, destination, ec); + if (ec) { + return LINGLONG_ERR("failed to publish source cache entry: " + ec.message()); + } + return LINGLONG_OK; +} + +auto replaceStagedDirectory(const std::filesystem::path &staging, + const std::filesystem::path &destination) noexcept + -> utils::error::Result +{ + LINGLONG_TRACE("replace materialized source directory"); + + std::error_code ec; + std::filesystem::remove_all(destination, ec); + if (ec) { + return LINGLONG_ERR("failed to remove old source destination: " + ec.message()); + } + + std::filesystem::rename(staging, destination, ec); + if (ec) { + return LINGLONG_ERR("failed to publish source destination: " + ec.message()); + } + + return LINGLONG_OK; +} + +auto materializeFile(const std::filesystem::path &cached, + const std::filesystem::path &destination) noexcept + -> utils::error::Result +{ + LINGLONG_TRACE("materialize file source"); + + std::error_code ec; + std::filesystem::create_directories(destination.parent_path(), ec); + if (ec) { + return LINGLONG_ERR("failed to create source destination: " + ec.message()); + } + + auto temporary = utils::TemporaryDirectory::create(".materialize-", destination.parent_path()); + if (!temporary) { + return LINGLONG_ERR(temporary); + } + + const auto staging = temporary->path() / "source"; + std::filesystem::copy_file(cached, + staging, + std::filesystem::copy_options::overwrite_existing, + ec); + if (ec) { + return LINGLONG_ERR("failed to copy cached source: " + ec.message()); + } + + std::filesystem::rename(staging, destination, ec); + if (ec) { + return LINGLONG_ERR("failed to publish source destination: " + ec.message()); + } + + return LINGLONG_OK; +} + +} // namespace + +auto fileNameFromUrl(std::string_view url) -> std::string +{ + const auto end = url.find_first_of("?#"); + const auto path = url.substr(0, end); + const auto separator = path.rfind('/'); + return percentDecode(path.substr(separator == std::string_view::npos ? 0 : separator + 1)); +} + +auto sourceOutputName(const api::types::v1::BuilderProjectSource &source) -> std::string +{ + if (source.name) { + return *source.name; } if (!source.url) { - return LINGLONG_ERR("URL is missing"); + return {}; + } + + return fileNameFromUrl(*source.url); +} + +namespace { + +auto hashFileContents(digest::SHA256 &hash, const std::filesystem::path &path) noexcept + -> utils::error::Result +{ + LINGLONG_TRACE("hash directory source file contents"); + + std::ifstream input(path, std::ios::binary); + if (!input) { + return LINGLONG_ERR("failed to open " + path.string()); + } + + std::array buffer{}; + while (input) { + input.read(buffer.data(), buffer.size()); + hash.update(reinterpret_cast(buffer.data()), input.gcount()); + } + + if (!input.eof()) { + return LINGLONG_ERR("failed to read " + path.string()); + } + + return LINGLONG_OK; +} + +} // namespace + +auto fileHash(const std::filesystem::path &file) noexcept -> utils::error::Result +{ + LINGLONG_TRACE("calculate file hash"); + + digest::SHA256 hash; + auto result = hashFileContents(hash, file); + if (!result) { + return LINGLONG_ERR(result); } - if (this->source.kind == "git") { + + return "sha256:" + finishHash(hash); +} + +namespace { + +auto runFetchScript( + const std::string &scriptName, + std::vector arguments, + const std::vector> &environment = {}) noexcept + -> utils::error::Result +{ + LINGLONG_TRACE("run source fetch script"); + + auto scriptFile = findBuilderScript(scriptName); + if (!scriptFile) { + return LINGLONG_ERR(scriptFile); + } + + LogD("source fetch: running helper {} action={}", scriptName, arguments.front()); + arguments.insert(arguments.begin(), scriptFile->string()); + utils::Cmd command("sh"); + for (const auto &[name, value] : environment) { + command.setEnv(name, value); + } + + return command.exec(arguments); +} + +struct SourceFetchContext +{ + const SourceFetchRequest &request; + const SourceCacheLayout &cache; + const std::filesystem::path &destination; + bool offline; +}; + +auto requestSource(const SourceFetchRequest &request) + -> const api::types::v1::BuilderProjectSource & +{ + return std::visit( + [](const auto &value) -> const api::types::v1::BuilderProjectSource & { + return value.source; + }, + request); +} + +auto requestExpectedHash(const SourceFetchRequest &request) -> std::optional +{ + if (const auto *locked = std::get_if(&request)) { + return locked->resolved.hash; + } + + return std::get(request).source.digest; +} + +struct AcquiredSource +{ + std::filesystem::path artifact; + BuilderSourceLockResolved result; +}; + +bool isGitCommitId(std::string_view commit) +{ + return (commit.size() == 40 || commit.size() == 64) + && std::all_of(commit.begin(), commit.end(), [](unsigned char character) { + return std::isxdigit(character) != 0; + }); +} + +bool sameGitCommit(std::string_view left, std::string_view right) +{ + return left.size() == right.size() + && std::equal(left.begin(), + left.end(), + right.begin(), + [](unsigned char lhs, unsigned char rhs) { + return std::tolower(lhs) == std::tolower(rhs); + }); +} + +class SourceFetchHandler +{ +public: + virtual ~SourceFetchHandler() = default; + + auto fetch(const SourceFetchContext &context) const noexcept + -> utils::error::Result + { + LINGLONG_TRACE("fetch source with handler"); + + const auto &source = requestSource(context.request); + const auto name = sourceOutputName(source); + const auto *lockedRequest = std::get_if(&context.request); + auto expected = normalizeExpectedHash(requestExpectedHash(context.request)); + if (!expected) { + return LINGLONG_ERR(expected); + } + + BuilderSourceLockResolved resolved; + std::filesystem::path cached; + std::optional cacheLock; + bool cacheReady = false; + if (lockedRequest) { + assert(expected->has_value()); + resolved.hash = **expected; + cached = cachePath(context, resolved); + auto locked = lockCacheEntry(context.cache, cached); + if (!locked) { + return LINGLONG_ERR(locked); + } + cacheLock.emplace(std::move(*locked)); + + LogD("source fetch: validating cached source {} at {}", name, cached); + cacheReady = validate(context, cached, resolved); + if (cacheReady) { + LogD("source fetch: verified cache hit for {}", name); + } else { + cacheLock.reset(); + } + } + + if (!cacheReady) { + if (context.offline) { + return LINGLONG_ERR("cannot acquire a new or changed source in offline mode"); + } + + LogD("source fetch: acquiring {} ({})", name, source.kind); + auto temporary = utils::TemporaryDirectory::create("fetch-", context.cache.temporary()); + if (!temporary) { + return LINGLONG_ERR(temporary); + } + auto acquired = acquire(context, temporary->path()); + if (!acquired) { + return LINGLONG_ERR(acquired); + } + + if (*expected && **expected != acquired->result.hash) { + LogD("source fetch: hash mismatch for {} expected={} actual={}", + name, + **expected, + acquired->result.hash); + return LINGLONG_ERR(lockedRequest + ? "source hash does not match linglong.lock" + : "source hash does not match digest in linglong.yaml"); + } + resolved = std::move(acquired->result); + cached = cachePath(context, resolved); + auto locked = lockCacheEntry(context.cache, cached); + if (!locked) { + return LINGLONG_ERR(locked); + } + cacheLock.emplace(std::move(*locked)); + + LogD("source fetch: publishing {} to cache {}", name, cached); + auto published = publishCacheEntry(acquired->artifact, cached, [&](const auto &path) { + return validate(context, path, resolved); + }); + if (!published) { + return LINGLONG_ERR(published); + } + } + + LogD("source fetch: materializing {} from cache to {}", name, context.destination); + auto materialized = materialize(context, cached, resolved); + if (!materialized) { + return LINGLONG_ERR(materialized); + } + + return resolved; + } + +protected: + virtual auto acquire(const SourceFetchContext &context, + const std::filesystem::path &temporary) const noexcept + -> utils::error::Result = 0; + virtual bool validate(const SourceFetchContext &context, + const std::filesystem::path &cached, + const BuilderSourceLockResolved &resolved) const noexcept = 0; + virtual auto materialize(const SourceFetchContext &context, + const std::filesystem::path &cached, + const BuilderSourceLockResolved &resolved) const noexcept + -> utils::error::Result = 0; + virtual auto normalizeExpectedHash(std::optional expected) const noexcept + -> utils::error::Result> = 0; + + virtual auto cachePath(const SourceFetchContext &context, + const BuilderSourceLockResolved &resolved) const -> std::filesystem::path + { + return context.cache.object(requestSource(context.request).kind, resolved.hash); + } +}; + +class BlobSourceHandler final : public SourceFetchHandler +{ +public: + enum class Materialization { + File, + Archive, + }; + + explicit BlobSourceHandler(Materialization materialization) + : materialization(materialization) + { + } + +private: + auto normalizeExpectedHash(std::optional expected) const noexcept + -> utils::error::Result> override + { + LINGLONG_TRACE("validate expected file source hash"); + + expected = normalizeSha256(std::move(expected)); + if (expected && !isSha256Hash(*expected, "sha256:")) { + return LINGLONG_ERR("invalid source SHA-256 digest"); + } + return expected; + } + + auto acquire(const SourceFetchContext &context, + const std::filesystem::path &temporary) const noexcept + -> utils::error::Result override + { + LINGLONG_TRACE("acquire blob source"); + + const auto &source = requestSource(context.request); + const auto artifact = temporary / "source"; + auto downloaded = runFetchScript("fetch-" + source.kind + "-source", + { "download", *source.url, artifact.string() }); + if (!downloaded) { + return LINGLONG_ERR(downloaded); + } + + auto hash = fileHash(artifact); + if (!hash) { + return LINGLONG_ERR(hash); + } + + return AcquiredSource{ .artifact = artifact, + .result = BuilderSourceLockResolved{ .hash = std::move(*hash) } }; + } + + bool validate(const SourceFetchContext &, + const std::filesystem::path &cached, + const BuilderSourceLockResolved &resolved) const noexcept override + { + auto actual = fileHash(cached); + return actual && *actual == resolved.hash; + } + + auto materialize(const SourceFetchContext &context, + const std::filesystem::path &cached, + const BuilderSourceLockResolved &) const noexcept + -> utils::error::Result override + { + LINGLONG_TRACE("materialize blob source"); + + const auto output = context.destination / sourceOutputName(requestSource(context.request)); + if (materialization == Materialization::File) { + return materializeFile(cached, output); + } + + auto temporary = utils::TemporaryDirectory::create(".materialize-", context.destination); + if (!temporary) { + return LINGLONG_ERR(temporary); + } + + const auto staging = temporary->path() / "source"; + auto extracted = runFetchScript("fetch-archive-source", + { "materialize", cached.string(), staging.string() }); + if (!extracted) { + return LINGLONG_ERR(extracted); + } + + return replaceStagedDirectory(staging, output); + } + + Materialization materialization; +}; + +class DscSourceHandler final : public SourceFetchHandler +{ +private: + static constexpr char cachedFileName[] = "source.dsc"; + + auto normalizeExpectedHash(std::optional expected) const noexcept + -> utils::error::Result> override + { + LINGLONG_TRACE("validate expected dsc source hash"); + + expected = normalizeSha256(std::move(expected)); + if (expected && !isSha256Hash(*expected, "sha256:")) { + return LINGLONG_ERR("invalid source SHA-256 digest"); + } + return expected; + } + + auto acquire(const SourceFetchContext &context, + const std::filesystem::path &temporary) const noexcept + -> utils::error::Result override + { + LINGLONG_TRACE("acquire dsc source"); + + const auto &source = requestSource(context.request); + const auto name = fileNameFromUrl(*source.url); + if (name.empty()) { + return LINGLONG_ERR("dsc source URL does not contain a file name"); + } + + const auto artifact = temporary / "source"; + auto downloaded = + runFetchScript("fetch-dsc-source", { "download", *source.url, artifact.string() }); + if (!downloaded) { + return LINGLONG_ERR(downloaded); + } + + auto hash = fileHash(artifact / name); + if (!hash) { + return LINGLONG_ERR(hash); + } + + if (name != cachedFileName) { + std::error_code ec; + std::filesystem::rename(artifact / name, artifact / cachedFileName, ec); + if (ec) { + return LINGLONG_ERR("failed to prepare cached dsc source: " + ec.message()); + } + } + + const auto resolved = BuilderSourceLockResolved{ .hash = *hash }; + if (!validate(context, artifact, resolved)) { + return LINGLONG_ERR("downloaded dsc source failed validation"); + } + + return AcquiredSource{ .artifact = artifact, + .result = BuilderSourceLockResolved{ .hash = std::move(*hash) } }; + } + + bool validate(const SourceFetchContext &, + const std::filesystem::path &cached, + const BuilderSourceLockResolved &resolved) const noexcept override + { + auto actual = fileHash(cached / cachedFileName); + if (!actual || *actual != resolved.hash) { + return false; + } + + return static_cast( + runFetchScript("fetch-dsc-source", { "verify", cached.string(), cachedFileName })); + } + + auto materialize(const SourceFetchContext &context, + const std::filesystem::path &cached, + const BuilderSourceLockResolved &) const noexcept + -> utils::error::Result override + { + LINGLONG_TRACE("materialize dsc source"); + + auto temporary = utils::TemporaryDirectory::create(".materialize-", context.destination); + if (!temporary) { + return LINGLONG_ERR(temporary); + } + + const auto staging = temporary->path() / "source"; + auto extracted = + runFetchScript("fetch-dsc-source", + { "materialize", cached.string(), cachedFileName, staging.string() }); + if (!extracted) { + return LINGLONG_ERR(extracted); + } + + return replaceStagedDirectory(staging, + context.destination + / sourceOutputName(requestSource(context.request))); + } +}; + +class GitSourceHandler final : public SourceFetchHandler +{ +private: + auto normalizeExpectedHash(std::optional expected) const noexcept + -> utils::error::Result> override + { + LINGLONG_TRACE("validate expected git source hash"); + + if (expected && !gitCommitFromHash(*expected)) { + return LINGLONG_ERR("invalid git source hash"); + } + return expected; + } + + auto acquire(const SourceFetchContext &context, + const std::filesystem::path &temporary) const noexcept + -> utils::error::Result override + { + LINGLONG_TRACE("acquire git source"); + + const auto &source = requestSource(context.request); if (!source.commit) { - return LINGLONG_ERR("digest missing"); + return LINGLONG_ERR("git source requires commit"); } - } else { - if (!source.digest) { - return LINGLONG_ERR("digest missing"); + + const auto artifact = temporary / "checkout"; + auto ref = *source.commit; + if (const auto *locked = std::get_if(&context.request)) { + auto commit = gitCommitFromHash(locked->resolved.hash); + if (!commit) { + return LINGLONG_ERR("invalid git source hash in project lock"); + } + ref = std::move(*commit); + } + + auto checkedOut = + runFetchScript("fetch-git-source", + { "checkout", *source.url, ref, artifact.string() }, + { { "GIT_SUBMODULES", source.submodules.value_or(true) ? "true" : "" } }); + if (!checkedOut) { + return LINGLONG_ERR(checkedOut); } + + auto commit = lastOutputLine(*checkedOut); + if (!isGitCommitId(commit)) { + return LINGLONG_ERR("git source helper did not return a resolved commit"); + } + + LogD("source fetch: checked out git source {} at commit {}", + sourceOutputName(source), + commit); + auto hash = "git-commit:" + commit; + + return AcquiredSource{ .artifact = artifact, + .result = BuilderSourceLockResolved{ .hash = std::move(hash) } }; } - auto scriptFile = findBuilderScript("fetch-" + source.kind + "-source"); - if (!scriptFile) { - return LINGLONG_ERR(scriptFile); + bool validate(const SourceFetchContext &context, + const std::filesystem::path &cached, + const BuilderSourceLockResolved &resolved) const noexcept override + { + const auto commit = gitCommitFromHash(resolved.hash); + if (!commit) { + return false; + } + + std::error_code ec; + const auto metadata = std::filesystem::symlink_status(cached / ".git", ec); + if (ec || !std::filesystem::is_directory(metadata)) { + return false; + } + + utils::Cmd git("git"); + if (!git.exists()) { + return false; + } + auto head = git.exec({ "-C", cached.string(), "rev-parse", "--verify", "HEAD^{commit}" }); + if (!head || !sameGitCommit(lastOutputLine(*head), *commit)) { + return false; + } + + // The working tree is restored in a temporary copy. Validate the objects instead of + // relying on status, which can hide ignored files and assume-unchanged modifications. + if (!git.exec({ "-C", cached.string(), "fsck", "--full", "--no-dangling", *commit })) { + return false; + } + if (!git.exec({ "-C", cached.string(), "remote", "get-url", "origin" })) { + return false; + } + + if (!requestSource(context.request).submodules.value_or(true)) { + return true; + } + + auto submodules = git.exec({ "-C", cached.string(), "submodule", "status", "--recursive" }); + if (!submodules) { + return false; + } + + std::istringstream lines(*submodules); + std::string line; + while (std::getline(lines, line)) { + if (!line.empty() && line.front() != ' ') { + return false; + } + } + return static_cast(git.exec({ "-C", + cached.string(), + "submodule", + "foreach", + "--recursive", + "git fsck --full --no-dangling \"$sha1\"" })); } - if (source.kind == "git") { - m_cmd->setEnv("GIT_SUBMODULES", source.submodules.value_or(true) ? "true" : ""); + + auto materialize(const SourceFetchContext &context, + const std::filesystem::path &cached, + const BuilderSourceLockResolved &resolved) const noexcept + -> utils::error::Result override + { + LINGLONG_TRACE("materialize git checkout"); + + const auto &source = requestSource(context.request); + const auto destination = context.destination / sourceOutputName(source); + + std::error_code ec; + std::filesystem::create_directories(destination.parent_path(), ec); + if (ec) { + return LINGLONG_ERR("failed to create source destination: " + ec.message()); + } + + auto temporary = + utils::TemporaryDirectory::create(".materialize-", destination.parent_path()); + if (!temporary) { + return LINGLONG_ERR(temporary); + } + + const auto staging = temporary->path() / "source"; + auto copied = copyTree(cached, staging); + if (!copied) { + return LINGLONG_ERR(copied); + } + + utils::Cmd git("git"); + const auto commit = gitCommitFromHash(resolved.hash); + if (!commit) { + return LINGLONG_ERR("invalid git source hash"); + } + + std::vector resetArguments{ "-C", staging.string(), "reset", "--hard" }; + if (source.submodules.value_or(true)) { + resetArguments.emplace_back("--recurse-submodules"); + } + resetArguments.push_back(*commit); + auto reset = git.exec(resetArguments); + if (!reset) { + return LINGLONG_ERR("failed to reset materialized git source", reset); + } + + // A hard reset can leave skip-worktree files untouched. Force every tracked file to + // be restored from the reset index, regardless of its copied index flags or stat data. + auto restored = git.exec({ "-C", + staging.string(), + "checkout-index", + "--all", + "--force", + "--ignore-skip-worktree-bits" }); + if (!restored) { + return LINGLONG_ERR("failed to restore materialized git source files", restored); + } + + auto cleaned = git.exec({ "-C", staging.string(), "clean", "-ffdx" }); + if (!cleaned) { + return LINGLONG_ERR("failed to clean materialized git source", cleaned); + } + if (source.submodules.value_or(true)) { + auto cleanedSubmodules = git.exec({ "-C", + staging.string(), + "submodule", + "foreach", + "--recursive", + "git checkout-index --all --force " + "--ignore-skip-worktree-bits && git clean -ffdx" }); + if (!cleanedSubmodules) { + return LINGLONG_ERR("failed to clean materialized git submodules", + cleanedSubmodules); + } + } + + if (!validate(context, staging, resolved)) { + return LINGLONG_ERR("materialized git source failed validation"); + } + auto status = git.exec({ "-C", + staging.string(), + "status", + "--porcelain=v1", + "--untracked-files=all", + "--ignore-submodules=none", + "--ignored" }); + if (!status || !trim(*status).empty()) { + return LINGLONG_ERR("materialized git source working tree is not clean"); + } + + auto remote = + git.exec({ "-C", staging.string(), "remote", "set-url", "origin", *source.url }); + if (!remote) { + return LINGLONG_ERR("failed to update materialized git remote", remote); + } + auto synchronized = + git.exec({ "-C", staging.string(), "submodule", "sync", "--recursive" }); + if (!synchronized) { + return LINGLONG_ERR("failed to synchronize materialized git submodules", synchronized); + } + + return replaceStagedDirectory(staging, destination); } - auto output = m_cmd->exec( - std::vector{ scriptFile->string(), - destination.absoluteFilePath(getSourceName()).toStdString(), - *source.url, - source.kind == "git" ? *source.commit : *source.digest, - this->cacheDir.absolutePath().toStdString() }); - if (!output.has_value()) { - LogE("output error: {}", output.error()); - return LINGLONG_ERR("stderr:", output); + + auto cachePath(const SourceFetchContext &context, + const BuilderSourceLockResolved &resolved) const + -> std::filesystem::path override + { + const auto commit = gitCommitFromHash(resolved.hash); + return context.cache.gitObject(*commit, + requestSource(context.request).submodules.value_or(true)); } +}; - return LINGLONG_OK; +auto createSourceFetchHandler(std::string_view kind) noexcept + -> utils::error::Result> +{ + LINGLONG_TRACE("create source fetch handler"); + + std::unique_ptr handler; + if (kind == "file") { + handler = std::make_unique(BlobSourceHandler::Materialization::File); + } else if (kind == "archive") { + handler = std::make_unique(BlobSourceHandler::Materialization::Archive); + } else if (kind == "dsc") { + handler = std::make_unique(); + } else if (kind == "git") { + handler = std::make_unique(); + } else { + return LINGLONG_ERR("unknown source kind"); + } + return handler; } -// 如果source有name字段使用name字段，否则使用url的filename -QString SourceFetcher::getSourceName() +} // namespace + +auto gitCommitFromHash(std::string_view hash) -> std::optional { - if (source.name.has_value()) { - return QString::fromStdString(*source.name); + constexpr auto prefix = std::string_view("git-commit:"); + if (hash.rfind(prefix, 0) != 0) { + return std::nullopt; } - if (source.url.has_value()) { - QUrl url(source.url->c_str()); - return url.fileName(); + + const auto commit = hash.substr(prefix.size()); + if (!isGitCommitId(commit)) { + return std::nullopt; } - LogE("missing name and url field"); - Q_ASSERT(false); - return "unknown"; + return std::string(commit); } -SourceFetcher::SourceFetcher(api::types::v1::BuilderProjectSource source, const QDir &cacheDir) - : cacheDir(cacheDir) - , source(std::move(source)) +auto SourceFetcher::fetch(const std::filesystem::path &destination, bool offline) noexcept + -> utils::error::Result { - if (this->cacheDir.mkpath(".")) { - return; + LINGLONG_TRACE("fetch source"); + + const auto &source = requestSource(request); + if (!source.url) { + return LINGLONG_ERR("source URL is missing"); } - LogE("mkpath {} failed", this->cacheDir.absolutePath().toStdString()); - Q_ASSERT(false); + auto handler = createSourceFetchHandler(source.kind); + if (!handler) { + return LINGLONG_ERR(handler); + } + + SourceFetchContext context{ + .request = request, + .cache = cache, + .destination = destination, + .offline = offline, + }; + return (*handler)->fetch(context); +} + +SourceFetcher::SourceFetcher(SourceFetchRequest request, SourceCacheLayout cache) + : cache(std::move(cache)) + , request(std::move(request)) +{ } } // namespace linglong::builder diff --git a/libs/linglong/src/linglong/builder/source_fetcher.h b/libs/linglong/src/linglong/builder/source_fetcher.h index d7f41bf93..b88826098 100644 --- a/libs/linglong/src/linglong/builder/source_fetcher.h +++ b/libs/linglong/src/linglong/builder/source_fetcher.h @@ -6,31 +6,54 @@ #pragma once -#include "linglong/api/types/v1/BuilderConfig.hpp" #include "linglong/api/types/v1/BuilderProjectSource.hpp" -#include "linglong/utils/cmd.h" +#include "linglong/api/types/v1/BuilderSourceLockResolved.hpp" +#include "linglong/builder/source_cache.h" #include "linglong/utils/error/error.h" -#include -#include -#include +#include +#include +#include +#include +#include namespace linglong::builder { +struct GenerateSourceRequest +{ + // Acquire fresh content and validate the manifest digest before publishing it to the cache. + api::types::v1::BuilderProjectSource source; +}; + +struct LockedSourceRequest +{ + // Reuse verified cached content when available, otherwise acquire the locked content. + api::types::v1::BuilderProjectSource source; + api::types::v1::BuilderSourceLockResolved resolved; +}; + +using SourceFetchRequest = std::variant; + +auto fileHash(const std::filesystem::path &file) noexcept -> utils::error::Result; + +auto fileNameFromUrl(std::string_view url) -> std::string; + +auto sourceOutputName(const api::types::v1::BuilderProjectSource &source) -> std::string; + +auto gitCommitFromHash(std::string_view hash) -> std::optional; + class SourceFetcher { public: - explicit SourceFetcher(api::types::v1::BuilderProjectSource source, const QDir &cacheDir); - - auto fetch(QDir destination) noexcept -> utils::error::Result; + explicit SourceFetcher(SourceFetchRequest request, SourceCacheLayout cache); - void setCommand(std::shared_ptr cmd) { this->m_cmd = cmd; } + // The caller must ensure the source cache layout and destination directory exist. + auto fetch(const std::filesystem::path &destination, bool offline = false) noexcept + -> utils::error::Result; private: - QString getSourceName(); - QDir cacheDir; - api::types::v1::BuilderProjectSource source; - std::shared_ptr m_cmd = std::make_shared("sh"); + SourceCacheLayout cache; + SourceFetchRequest request; }; } // namespace linglong::builder diff --git a/libs/linglong/src/linglong/repo/ostree_repo.cpp b/libs/linglong/src/linglong/repo/ostree_repo.cpp index 9eb8a6f72..92270a2e0 100644 --- a/libs/linglong/src/linglong/repo/ostree_repo.cpp +++ b/libs/linglong/src/linglong/repo/ostree_repo.cpp @@ -2945,7 +2945,7 @@ utils::error::Result OSTreeRepo::getMergedModuleDir( return LINGLONG_ERR("merged doesn't exist"); } -// this method creates a temporary merged module directory for debugging purposes +// Select only the requested modules for a build or run context. // The returned TempLayerDir owns and removes the directory automatically. utils::error::Result OSTreeRepo::createTempMergedModuleDir( const package::Reference &ref, const std::vector &loadModules) const noexcept @@ -2956,40 +2956,39 @@ utils::error::Result OSTreeRepo::createTempMergedModuleDi return LINGLONG_ERR("failed to ensure merged directory", ret); } - auto layerItems = this->cache->queryExistingLayerItem(); QCryptographicHash hash(QCryptographicHash::Sha256); std::vector commits; - std::string findModules; + const auto layerItems = this->cache->queryExistingLayerItem(); - // ADD_FILES keeps files from the first checkout. Keep binary first so the merged layer uses - // its info.json. + // ADD_FILES keeps files from the first checkout. Keep the main module first so its + // info.json describes the merged layer. auto orderedModules = loadModules; auto binaryModule = std::find(orderedModules.begin(), orderedModules.end(), "binary"); + if (binaryModule == orderedModules.end()) { + binaryModule = std::find(orderedModules.begin(), orderedModules.end(), "runtime"); + } if (binaryModule != orderedModules.end()) { std::iter_swap(orderedModules.begin(), binaryModule); } for (const auto &module : orderedModules) { - auto layer = std::find_if(layerItems.cbegin(), layerItems.cend(), [&](const auto &item) { - const auto arch = item.info.arch.empty() ? std::string{} : item.info.arch.front(); - return item.info.id == ref.id && item.info.version == ref.version.toString() - && arch == ref.arch.toString() && item.info.packageInfoV2Module == module; - }); + const auto layer = + std::find_if(layerItems.cbegin(), layerItems.cend(), [&](const auto &item) { + const auto arch = item.info.arch.empty() ? std::string{} : item.info.arch.front(); + return item.info.id == ref.id && item.info.channel == ref.channel + && item.info.version == ref.version.toString() && arch == ref.arch.toString() + && item.info.packageInfoV2Module == module; + }); if (layer == layerItems.cend()) { - continue; + return LINGLONG_ERR("failed to find module " + module + " for " + ref.toString()); } commits.push_back(layer->commit); - findModules += layer->info.packageInfoV2Module + " "; hash.addData(QString::fromStdString(layer->commit).toUtf8()); } if (commits.empty()) { return LINGLONG_ERR("not found any layer"); } - // 模块未全部找到 - if (commits.size() < loadModules.size()) { - return LINGLONG_ERR(fmt::format("missing module, only found: {}", findModules)); - } // 合并layer，生成临时merged目录 const QString mergeID = hash.result().toHex(); auto temporaryDirectory = @@ -3043,10 +3042,9 @@ utils::error::Result OSTreeRepo::mergeModules() const noexcept if (!layer.info.arch.empty()) { arch = layer.info.arch.front(); } - // 将id、version和arch相同的item合并，不区分repo和channel - auto groupKey = QString("%1/%2/%3") - .arg(layer.info.id.c_str(), layer.info.version.c_str(), arch.c_str()) - .toStdString(); + // Group modules by channel, ID, version, and architecture. + const auto groupKey = + fmt::format("{}:{}/{}/{}", layer.info.channel, layer.info.id, layer.info.version, arch); layerGroup[groupKey].push_back(layer); } diff --git a/libs/linglong/src/linglong/runtime/run_context.cpp b/libs/linglong/src/linglong/runtime/run_context.cpp index 5d867d1d2..e40a5f599 100644 --- a/libs/linglong/src/linglong/runtime/run_context.cpp +++ b/libs/linglong/src/linglong/runtime/run_context.cpp @@ -4,6 +4,7 @@ #include "linglong/runtime/run_context.h" +#include "linglong/builder/project_lock.h" #include "linglong/cdi/cdi.h" #include "linglong/cli/cli.h" #include "linglong/common/display.h" @@ -319,7 +320,8 @@ utils::error::Result RunContext::resolve(const linglong::package::Referenc } utils::error::Result RunContext::resolve(const api::types::v1::BuilderProject &target, - const std::filesystem::path &buildOutput) + const std::filesystem::path &buildOutput, + const api::types::v1::BuilderProjectLock &lock) { LINGLONG_TRACE("resolve RunContext from builder project " + target.package.id); @@ -339,41 +341,38 @@ utils::error::Result RunContext::resolve(const api::types::v1::BuilderProj return LINGLONG_ERR("can't resolve run context from package kind " + target.package.kind); } - auto base = target.base; - if (target.runtime) { - auto runtimeFuzzyRef = package::FuzzyReference::parse(*target.runtime); - if (!runtimeFuzzyRef) { - return LINGLONG_ERR(runtimeFuzzyRef); - } + const auto base = + std::find_if(lock.dependencies.begin(), lock.dependencies.end(), [](const auto &entry) { + return entry.type == "base"; + }); + if (base == lock.dependencies.end()) { + return LINGLONG_ERR("project lock base dependency is missing"); + } - auto ref = repo.clearReferenceLocal(*runtimeFuzzyRef, true); + const auto runtime = + std::find_if(lock.dependencies.begin(), lock.dependencies.end(), [](const auto &entry) { + return entry.type == "runtime"; + }); + if (runtime != lock.dependencies.end()) { + auto ref = builder::lockedDependencyReference(*runtime); if (!ref) { - return LINGLONG_ERR("ref doesn't exist " + runtimeFuzzyRef->toString()); + return LINGLONG_ERR("invalid locked runtime reference", ref); } + auto res = RuntimeLayer::create(std::move(ref).value(), *this); if (!res) { return LINGLONG_ERR(res); } runtimeLayer = std::move(res).value(); - - if (!base) { - base = runtimeLayer->getCachedItem().info.base; - } - } - - if (!base) { - return LINGLONG_ERR("at least one of base or runtime must be specified"); - } - - auto baseFuzzyRef = package::FuzzyReference::parse(*base); - if (!baseFuzzyRef) { - return LINGLONG_ERR(baseFuzzyRef); + } else { + runtimeLayer.reset(); } - auto ref = repo.clearReferenceLocal(*baseFuzzyRef, true); + auto ref = builder::lockedDependencyReference(*base); if (!ref) { - return LINGLONG_ERR(ref); + return LINGLONG_ERR("invalid locked base reference", ref); } + auto res = RuntimeLayer::create(std::move(ref).value(), *this); if (!res) { return LINGLONG_ERR(res); @@ -392,7 +391,12 @@ utils::error::Result RunContext::resolve(const api::types::v1::BuilderProj resolveHostDynamic(); - return resolveLayer(false, {}); + return resolveLayer(false, + {}, + base->resolved.modules, + runtime != lock.dependencies.end() + ? std::make_optional(runtime->resolved.modules) + : std::nullopt); } utils::error::Result RunContext::resolve(const api::types::v1::RunContextConfig &config) @@ -606,8 +610,11 @@ utils::error::Result RunContext::setupCDIDevices(generator::ContainerCfgBu return LINGLONG_OK; } -utils::error::Result RunContext::resolveLayer(bool depsExcludeDev, - const std::vector &appModules) +utils::error::Result +RunContext::resolveLayer(bool depsExcludeDev, + const std::vector &appModules, + const std::optional> &baseModules, + const std::optional> &runtimeModules) { LINGLONG_TRACE("resolve layers"); @@ -615,7 +622,7 @@ utils::error::Result RunContext::resolveLayer(bool depsExcludeDev, if (depsExcludeDev) { depsExcludeModules = std::vector{ "develop" }; } - auto ref = baseLayer->resolveLayer(std::nullopt, depsExcludeModules); + auto ref = baseLayer->resolveLayer(baseModules, depsExcludeModules); if (!ref.has_value()) { return LINGLONG_ERR("failed to resolve base layer", ref); } @@ -632,7 +639,7 @@ utils::error::Result RunContext::resolveLayer(bool depsExcludeDev, } if (runtimeLayer) { - auto ref = runtimeLayer->resolveLayer(std::nullopt, depsExcludeModules); + auto ref = runtimeLayer->resolveLayer(runtimeModules, depsExcludeModules); if (!ref.has_value()) { return LINGLONG_ERR("failed to resolve runtime layer", ref); } diff --git a/libs/linglong/src/linglong/runtime/run_context.h b/libs/linglong/src/linglong/runtime/run_context.h index 192f9bf9f..f4e7996e4 100644 --- a/libs/linglong/src/linglong/runtime/run_context.h +++ b/libs/linglong/src/linglong/runtime/run_context.h @@ -7,6 +7,7 @@ #pragma once #include "linglong/api/types/v1/BuilderProject.hpp" +#include "linglong/api/types/v1/BuilderProjectLock.hpp" #include "linglong/api/types/v1/ContainerProcessStateInfo.hpp" #include "linglong/api/types/v1/ExtensionDefine.hpp" #include "linglong/api/types/v1/Mount.hpp" @@ -66,7 +67,8 @@ class RunContext const ResolveOptions &opts = ResolveOptions{}); utils::error::Result resolve(const api::types::v1::BuilderProject &target, - const std::filesystem::path &buildOutput); + const std::filesystem::path &buildOutput, + const api::types::v1::BuilderProjectLock &lock); utils::error::Result resolve(const api::types::v1::RunContextConfig &config); @@ -119,8 +121,11 @@ class RunContext -> utils::error::Result; private: - utils::error::Result resolveLayer(bool depsExcludeDev, - const std::vector &appModules); + utils::error::Result + resolveLayer(bool depsExcludeDev, + const std::vector &appModules, + const std::optional> &baseModules = std::nullopt, + const std::optional> &runtimeModules = std::nullopt); utils::error::Result resolveLayerExtensions( RuntimeLayer &layer, const std::vector &externalExtensionDefs); diff --git a/libs/linglong/tests/ll-tests/CMakeLists.txt b/libs/linglong/tests/ll-tests/CMakeLists.txt index 58e11583b..abe892fe5 100644 --- a/libs/linglong/tests/ll-tests/CMakeLists.txt +++ b/libs/linglong/tests/ll-tests/CMakeLists.txt @@ -19,6 +19,7 @@ pfl_add_executable( src/linglong/builder/linglong_builder_test.cpp src/linglong/builder/pull_dependency_test.cpp src/linglong/builder/source_fetcher_test.cpp + src/linglong/builder/project_lock_test.cpp src/linglong/cli/cli_printer_test.cpp src/linglong/cli/cli_test.cpp src/linglong/common/gkeyfile_wrapper_test.cpp diff --git a/libs/linglong/tests/ll-tests/src/linglong/builder/linglong_builder_test.cpp b/libs/linglong/tests/ll-tests/src/linglong/builder/linglong_builder_test.cpp index 7217405b9..3ebf5d077 100644 --- a/libs/linglong/tests/ll-tests/src/linglong/builder/linglong_builder_test.cpp +++ b/libs/linglong/tests/ll-tests/src/linglong/builder/linglong_builder_test.cpp @@ -5,17 +5,629 @@ #include "../../common/tempdir.h" #include "../mocks/linglong_builder_mock.h" +#include "linglong/builder/builder_directory.h" #include "linglong/builder/linglong_builder.h" +#include "linglong/builder/source_cache.h" +#include "linglong/utils/env.h" #include "linglong/utils/error/error.h" +#include "linglong/utils/file.h" +#include #include #include +#include #include #include +#include #include +#include #include +static void expectProjectProcessSuccess(pid_t pid) +{ + ASSERT_GT(pid, 0); + int status = 0; + pid_t waited; + do { + waited = ::waitpid(pid, &status, 0); + } while (waited < 0 && errno == EINTR); + + ASSERT_EQ(waited, pid); + ASSERT_TRUE(WIFEXITED(status)); + EXPECT_EQ(WEXITSTATUS(status), 0); +} + +TEST(LinglongBuilder, YamlPathPreservesFileSymlinksForExplicitAndDefaultLookup) +{ + TempDir workingDir; + TempDir sharedConfig; + const auto target = sharedConfig.path() / "app.yaml"; + const auto manifest = workingDir.path() / "linglong.yaml"; + std::ofstream(target) << "shared configuration"; + std::filesystem::create_symlink(target, manifest); + + const auto implicit = linglong::builder::getProjectYamlPath(workingDir.path(), {}); + const auto explicitPath = linglong::builder::getProjectYamlPath(workingDir.path(), manifest); + const auto relativePath = linglong::builder::getProjectYamlPath( + workingDir.path(), + std::filesystem::relative(workingDir.path()) / "linglong.yaml"); + ASSERT_TRUE(implicit); + ASSERT_TRUE(explicitPath); + ASSERT_TRUE(relativePath); + EXPECT_EQ(*implicit, manifest); + EXPECT_EQ(*explicitPath, manifest); + EXPECT_EQ(*relativePath, manifest); + EXPECT_EQ(linglong::builder::projectLockPath(*implicit), workingDir.path() / "linglong.lock"); + EXPECT_EQ(linglong::builder::projectLockPath(*explicitPath), + workingDir.path() / "linglong.lock"); + + std::ifstream input(*explicitPath); + std::string content; + std::getline(input, content); + EXPECT_EQ(content, "shared configuration"); + + const auto alias = sharedConfig.path() / "project-alias"; + std::filesystem::create_directory_symlink(workingDir.path(), alias); + const auto aliased = linglong::builder::getProjectYamlPath(alias, alias / "linglong.yaml"); + ASSERT_TRUE(aliased); + EXPECT_EQ(*aliased, manifest); +} + +TEST(LinglongBuilder, YamlPathPrefersArchitectureEntryAndRejectsMissingFiles) +{ + TempDir workingDir; + TempDir sharedConfig; + const auto target = sharedConfig.path() / "app.yaml"; + const auto manifest = workingDir.path() + / ("linglong." + linglong::package::Architecture::currentCPUArchitecture().toString() + + ".yaml"); + std::ofstream(target) << "shared configuration"; + std::ofstream(workingDir.path() / "linglong.yaml") << "fallback configuration"; + std::filesystem::create_symlink(target, manifest); + + const auto implicit = linglong::builder::getProjectYamlPath(workingDir.path(), {}); + const auto explicitPath = linglong::builder::getProjectYamlPath(workingDir.path(), manifest); + ASSERT_TRUE(implicit); + ASSERT_TRUE(explicitPath); + EXPECT_EQ(*implicit, manifest); + EXPECT_EQ(*explicitPath, manifest); + + EXPECT_FALSE( + linglong::builder::getProjectYamlPath(workingDir.path(), workingDir.path() / "missing.yaml")); + EXPECT_FALSE(linglong::builder::getProjectYamlPath(workingDir.path(), + workingDir.path() / "missing-directory" + / "linglong.yaml")); + const auto brokenLink = workingDir.path() / "broken.yaml"; + std::filesystem::create_symlink(sharedConfig.path() / "missing.yaml", brokenLink); + EXPECT_FALSE(linglong::builder::getProjectYamlPath(workingDir.path(), brokenLink)); +} + +TEST(LinglongBuilder, BuilderLocksYamlEntryDirectoryAcrossWorkingDirectories) +{ + TempDir workingDir; + TempDir sharedConfig; + const auto yamlDirectory = workingDir.path() / "config"; + const auto manifest = yamlDirectory / "linglong.yaml"; + const auto target = sharedConfig.path() / "app.yaml"; + std::filesystem::create_directory(yamlDirectory); + std::ofstream(target) << "shared configuration"; + std::filesystem::create_symlink(target, manifest); + + { + linglong::builder::BuilderMock builder(workingDir.path()); + builder.projectYamlFile = manifest.string(); + ASSERT_TRUE(builder.cleanBuildArtifacts()); + ASSERT_TRUE(builder.cleanBuildArtifacts()); + + for (const auto &directory : { workingDir.path(), yamlDirectory }) { + const auto pid = ::fork(); + if (pid == 0) { + ::alarm(10); + linglong::builder::BuilderMock competing(directory); + competing.projectYamlFile = manifest.string(); + auto result = competing.cleanBuildArtifacts(); + ::_exit(!result + && result.error().message().find("already in use") != std::string::npos + ? 0 + : 1); + } + expectProjectProcessSuccess(pid); + } + + // The target's directory has its own resource lock and remains independent. + auto independent = + linglong::builder::BuilderDirectoryLayout(sharedConfig.path()).lockProject(); + ASSERT_TRUE(independent) << independent.error().message(); + } + + auto projectLock = linglong::builder::BuilderDirectoryLayout(workingDir.path()).lockProject(); + auto yamlLock = linglong::builder::BuilderDirectoryLayout(yamlDirectory).lockProject(); + ASSERT_TRUE(projectLock) << projectLock.error().message(); + ASSERT_TRUE(yamlLock) << yamlLock.error().message(); +} + +TEST(LinglongBuilder, BuilderDeduplicatesWorkingAndYamlDirectoryAliases) +{ + TempDir workingDir; + TempDir aliases; + const auto alias = aliases.path() / "project"; + std::filesystem::create_directory_symlink(workingDir.path(), alias); + linglong::builder::BuilderMock builder(workingDir.path()); + builder.projectYamlFile = (alias / "linglong.yaml").string(); + + ASSERT_TRUE(builder.cleanBuildArtifacts()); + ASSERT_TRUE(builder.cleanBuildArtifacts()); +} + +TEST(LinglongBuilder, BuilderReleasesWorkingDirectoryLockWhenYamlDirectoryIsBusy) +{ + TempDir workingDir; + const auto yamlDirectory = workingDir.path() / "config"; + std::filesystem::create_directory(yamlDirectory); + auto held = linglong::builder::BuilderDirectoryLayout(yamlDirectory).lockProject(); + ASSERT_TRUE(held); + // Initialize shared mock resources before forking. + linglong::builder::BuilderMock builder(workingDir.path()); + builder.projectYamlFile = (yamlDirectory / "linglong.yaml").string(); + + const auto pid = ::fork(); + if (pid == 0) { + ::alarm(10); + auto result = builder.cleanBuildArtifacts(); + if (result || result.error().message().find("already in use") == std::string::npos) { + ::_exit(1); + } + + // The failed Builder is still alive; it must not retain the first lock. + auto released = linglong::builder::BuilderDirectoryLayout(workingDir.path()).lockProject(); + ::_exit(released && released->isLocked() ? 0 : 2); + } + expectProjectProcessSuccess(pid); +} + +TEST(LinglongBuilder, ProjectLockRejectsConcurrentProcessesAndDirectoryAliases) +{ + TempDir workingDir; + TempDir otherProject; + const auto project = workingDir.path() / "project"; + const auto alias = workingDir.path() / "alias"; + std::filesystem::create_directory(project); + std::filesystem::create_directory_symlink(project, alias); + + auto lock = linglong::builder::BuilderDirectoryLayout(project).lockProject(); + ASSERT_TRUE(lock) << lock.error().message(); + ASSERT_TRUE(lock->isLocked()); + + for (const auto &path : { project, project / ".", alias }) { + const auto pid = ::fork(); + if (pid == 0) { + ::alarm(10); + auto conflicting = linglong::builder::BuilderDirectoryLayout(path).lockProject(); + if (conflicting + || conflicting.error().message().find("already in use") == std::string::npos) { + ::_exit(1); + } + + auto independent = + linglong::builder::BuilderDirectoryLayout(otherProject.path()).lockProject(); + ::_exit(independent && independent->isLocked() ? 0 : 2); + } + expectProjectProcessSuccess(pid); + } +} + +TEST(LinglongBuilder, ProjectLockSurvivesCleanAndCanBeAcquiredAfterRelease) +{ + TempDir workingDir; + linglong::builder::BuilderDirectoryLayout directories(workingDir.path()); + ASSERT_TRUE(directories.ensureRoot()); + std::ofstream(directories.root() / "state") << "build state"; + + std::optional held; + auto lock = directories.lockProject(); + ASSERT_TRUE(lock) << lock.error().message(); + held.emplace(std::move(*lock)); + + ASSERT_TRUE(directories.clean()); + EXPECT_FALSE(std::filesystem::exists(directories.root())); + EXPECT_TRUE(std::filesystem::is_regular_file(workingDir.path() / ".ll-builder.lock")); + + const auto pid = ::fork(); + if (pid == 0) { + ::alarm(10); + auto conflicting = directories.lockProject(); + ::_exit(!conflicting + && conflicting.error().message().find("already in use") != std::string::npos + ? 0 + : 1); + } + expectProjectProcessSuccess(pid); + + held.reset(); + auto acquired = directories.lockProject(); + ASSERT_TRUE(acquired) << acquired.error().message(); + EXPECT_TRUE(acquired->isLocked()); +} + +TEST(LinglongBuilder, ProjectLockIsReleasedWhenProcessExits) +{ + TempDir workingDir; + linglong::builder::BuilderDirectoryLayout directories(workingDir.path()); + const auto pid = ::fork(); + if (pid == 0) { + auto lock = directories.lockProject(); + // Exit without destructors to verify that a leftover file does not keep the project busy. + ::_exit(lock && lock->isLocked() ? 0 : 1); + } + expectProjectProcessSuccess(pid); + + ASSERT_TRUE(std::filesystem::is_regular_file(workingDir.path() / ".ll-builder.lock")); + auto acquired = directories.lockProject(); + ASSERT_TRUE(acquired) << acquired.error().message(); + EXPECT_TRUE(acquired->isLocked()); +} + +TEST(LinglongBuilder, ProjectOperationsRejectBusyProjectBeforeDoingWork) +{ + TempDir workingDir; + linglong::builder::BuilderMock builder(linglong::api::types::v1::BuilderProject{}, + workingDir.path()); + auto lock = linglong::builder::BuilderDirectoryLayout(workingDir.path()).lockProject(); + ASSERT_TRUE(lock); + + for (int operation = 0; operation < 8; ++operation) { + SCOPED_TRACE(operation); + const auto pid = ::fork(); + if (pid == 0) { + ::alarm(10); + linglong::utils::error::Result result; + switch (operation) { + case 0: + result = builder.build(); + break; + case 1: + result = builder.cleanBuildArtifacts(); + break; + case 2: + result = builder.run({}, {}); + break; + case 3: + result = builder.exportUAB({}); + break; + case 4: + result = builder.exportLayer({}); + break; + case 5: + result = builder.push("binary"); + break; + case 6: + result = builder.processProjectLock(linglong::builder::ProjectLockMode::Generate); + break; + case 7: + result = builder.processProjectLock(linglong::builder::ProjectLockMode::Update); + break; + } + + ::_exit(!result && result.error().message().find("already in use") != std::string::npos + ? 0 + : 1); + } + expectProjectProcessSuccess(pid); + } + EXPECT_FALSE(std::filesystem::exists(workingDir.path() / "linglong")); + EXPECT_FALSE(std::filesystem::exists(workingDir.path() / ".uabBuild")); +} + +TEST(LinglongBuilder, BuilderReusesProjectLockUntilItsResourcesAreDestroyed) +{ + TempDir workingDir; + linglong::builder::BuilderDirectoryLayout directories(workingDir.path()); + { + linglong::builder::BuilderMock builder(linglong::api::types::v1::BuilderProject{}, + workingDir.path()); + // A failed operation still leaves the instance holding its project lock. + EXPECT_FALSE(builder.build()); + ASSERT_TRUE(builder.cleanBuildArtifacts()); + ASSERT_TRUE(builder.cleanBuildArtifacts()); + + const auto pid = ::fork(); + if (pid == 0) { + ::alarm(10); + auto conflicting = directories.lockProject(); + ::_exit(!conflicting + && conflicting.error().message().find("already in use") != std::string::npos + ? 0 + : 1); + } + expectProjectProcessSuccess(pid); + } + + auto acquired = directories.lockProject(); + ASSERT_TRUE(acquired) << acquired.error().message(); + EXPECT_TRUE(acquired->isLocked()); +} + +TEST(LinglongBuilder, LockedBuildRejectsMissingLockBeforeCreatingState) +{ + // Include the preparation error in build()'s stage error message. + linglong::utils::EnvironmentVariableGuard backtrace("LINYAPS_BACKTRACE", "1"); + TempDir workingDir; + linglong::api::types::v1::BuilderProject project; + linglong::builder::BuilderMock builder(std::move(project), workingDir.path()); + builder.projectYamlFile = (workingDir.path() / "linglong.yaml").string(); + linglong::builder::BuilderBuildOptions options; + options.locked = true; + builder.setBuildOptions(options); + + auto result = builder.build(); + ASSERT_FALSE(result.has_value()); + EXPECT_NE(result.error().message().find("does not exist"), std::string::npos); + EXPECT_FALSE(std::filesystem::exists(workingDir.path() / "linglong")); +} + +TEST(LinglongBuilder, LockedBuildRejectsSkipFetchBeforeCreatingState) +{ + linglong::utils::EnvironmentVariableGuard backtrace("LINYAPS_BACKTRACE", "1"); + TempDir workingDir; + linglong::builder::BuilderMock builder(linglong::api::types::v1::BuilderProject{}, + workingDir.path()); + builder.projectYamlFile = (workingDir.path() / "linglong.yaml").string(); + linglong::builder::BuilderBuildOptions options; + options.locked = true; + options.skipFetchSource = true; + builder.setBuildOptions(options); + + auto result = builder.build(); + ASSERT_FALSE(result); + EXPECT_NE(result.error().message().find("--locked cannot be combined with --skip-fetch-source"), + std::string::npos); + EXPECT_FALSE(std::filesystem::exists(workingDir.path() / "linglong")); +} + +TEST(LinglongBuilder, NormalBuildWithoutLockContinuesProjectValidation) +{ + linglong::utils::EnvironmentVariableGuard backtrace("LINYAPS_BACKTRACE", "1"); + TempDir workingDir; + linglong::api::types::v1::BuilderProject project; + linglong::builder::BuilderMock builder(std::move(project), workingDir.path()); + builder.projectYamlFile = (workingDir.path() / "linglong.yaml").string(); + + auto result = builder.build(); + ASSERT_FALSE(result); + EXPECT_NE(result.error().message().find("invalid project info"), std::string::npos); + EXPECT_FALSE(std::filesystem::exists(workingDir.path() / "linglong")); +} + +TEST(LinglongBuilder, BuildsRejectInvalidExistingLockBeforeCreatingState) +{ + linglong::utils::EnvironmentVariableGuard backtrace("LINYAPS_BACKTRACE", "1"); + for (const int mode : { 0, 1, 2 }) { + SCOPED_TRACE(mode); + TempDir workingDir; + const auto manifest = workingDir.path() / "linglong.yaml"; + std::ofstream(linglong::builder::projectLockPath(manifest)) << "invalid lock"; + linglong::api::types::v1::BuilderProject project; + linglong::builder::BuilderMock builder(std::move(project), workingDir.path()); + builder.projectYamlFile = manifest.string(); + linglong::builder::BuilderBuildOptions options; + options.locked = mode == 1; + options.skipFetchSource = mode == 2; + builder.setBuildOptions(options); + + auto result = builder.build(); + ASSERT_FALSE(result); + EXPECT_NE(result.error().message().find("failed to read"), std::string::npos); + EXPECT_FALSE(std::filesystem::exists(workingDir.path() / "linglong")); + } +} + +TEST(LinglongBuilder, GenerateLockRejectsExistingFileBeforeCreatingState) +{ + TempDir workingDir; + const auto manifest = workingDir.path() / "linglong.yaml"; + const auto lockPath = linglong::builder::projectLockPath(manifest); + std::ofstream(lockPath) << "invalid lock"; + linglong::api::types::v1::BuilderProject project; + linglong::builder::BuilderMock builder(std::move(project), workingDir.path()); + builder.projectYamlFile = manifest.string(); + + auto result = builder.processProjectLock(linglong::builder::ProjectLockMode::Generate); + ASSERT_FALSE(result); + EXPECT_NE(result.error().message().find("already exists"), std::string::npos); + EXPECT_FALSE(std::filesystem::exists(workingDir.path() / "linglong")); + std::ifstream lockFile(lockPath); + std::string content; + std::getline(lockFile, content); + EXPECT_EQ(content, "invalid lock"); +} + +TEST(LinglongBuilder, UpdateLockRequiresExistingFileBeforeCreatingState) +{ + TempDir workingDir; + linglong::api::types::v1::BuilderProject project; + linglong::builder::BuilderMock builder(std::move(project), workingDir.path()); + builder.projectYamlFile = (workingDir.path() / "linglong.yaml").string(); + + auto result = builder.processProjectLock(linglong::builder::ProjectLockMode::Update); + ASSERT_FALSE(result); + EXPECT_NE(result.error().message().find("does not exist"), std::string::npos); + EXPECT_FALSE(std::filesystem::exists(workingDir.path() / "linglong")); +} + +TEST(LinglongBuilder, UpdateLockIgnoresOldContentAndPreservesItOnFailure) +{ + TempDir workingDir; + const auto manifest = workingDir.path() / "linglong.yaml"; + const auto lockPath = linglong::builder::projectLockPath(manifest); + std::ofstream(lockPath) << "invalid lock"; + linglong::api::types::v1::BuilderProject project; + linglong::api::types::v1::BuilderProjectSource source; + source.kind = "unsupported"; + project.sources = std::vector{ source }; + linglong::builder::BuilderMock builder(std::move(project), workingDir.path()); + builder.projectYamlFile = manifest.string(); + linglong::utils::EnvironmentVariableGuard cache("LINGLONG_FETCH_CACHE", + (workingDir.path() / "cache").string()); + + auto result = builder.processProjectLock(linglong::builder::ProjectLockMode::Update); + ASSERT_FALSE(result); + EXPECT_NE(result.error().message().find("unsupported source kind"), std::string::npos); + std::ifstream lockFile(lockPath); + std::string content; + std::getline(lockFile, content); + EXPECT_EQ(content, "invalid lock"); +} + +TEST(LinglongBuilder, SkipFetchPreservesManualSourcesDespiteExistingLock) +{ + TempDir workingDir; + const auto manifest = workingDir.path() / "linglong.yaml"; + std::ofstream(manifest) << "version: 1\n"; + const auto sourceDirectory = workingDir.path() / "linglong/sources"; + std::filesystem::create_directories(sourceDirectory); + const auto manualSource = sourceDirectory / "manual-source"; + std::ofstream(manualSource) << "local changes"; + const auto lockPath = linglong::builder::projectLockPath(manifest); + std::ofstream(lockPath) << "invalid lock"; + + linglong::api::types::v1::BuilderProject project; + linglong::api::types::v1::BuilderProjectSource source; + source.kind = "git"; + source.url = "https://example.com/source.git"; + source.commit = "0123456789012345678901234567890123456789"; + project.sources = std::vector{ source }; + linglong::builder::BuilderMock builder(std::move(project), workingDir.path()); + builder.projectYamlFile = manifest.string(); + linglong::builder::BuilderBuildOptions options; + options.skipFetchSource = true; + builder.setBuildOptions(options); + + auto skipped = builder.buildStageFetchSource(); + ASSERT_TRUE(skipped.has_value()) << skipped.error().message(); + EXPECT_TRUE(std::filesystem::exists(manualSource)); + std::ifstream lockFile(lockPath); + std::string lockContent; + std::getline(lockFile, lockContent); + EXPECT_EQ("invalid lock", lockContent); +} + +TEST(LinglongBuilder, BuildOverlaysPreserveHistoryUntilDependenciesChange) +{ + TempDir workingDir; + linglong::builder::BuilderDirectoryLayout directories(workingDir.path()); + const auto baseUpper = directories.buildBaseOverlay() / "upperdir"; + const auto runtimeUpper = directories.buildRuntimeOverlay() / "upperdir"; + auto prepared = linglong::builder::detail::prepareBuildOverlays(directories, "first", true); + ASSERT_TRUE(prepared) << prepared.error().message(); + EXPECT_FALSE(*prepared); + ASSERT_TRUE(std::filesystem::is_directory(baseUpper)); + ASSERT_TRUE(std::filesystem::is_directory(runtimeUpper)); + std::ofstream(baseUpper / "installed-tool") << "APT installation"; + std::ofstream(runtimeUpper / "build-history") << "build changes"; + std::filesystem::create_directories(directories.sources()); + std::ofstream(directories.sources() / "source") << "source changes"; + + auto reused = linglong::builder::detail::prepareBuildOverlays(directories, "first", true); + ASSERT_TRUE(reused) << reused.error().message(); + EXPECT_TRUE(*reused); + EXPECT_TRUE(std::filesystem::exists(baseUpper / "installed-tool")); + EXPECT_TRUE(std::filesystem::exists(runtimeUpper / "build-history")); + + auto changed = linglong::builder::detail::prepareBuildOverlays(directories, "second", true); + ASSERT_TRUE(changed) << changed.error().message(); + EXPECT_FALSE(*changed); + EXPECT_TRUE(std::filesystem::is_directory(baseUpper)); + EXPECT_TRUE(std::filesystem::is_directory(runtimeUpper)); + EXPECT_FALSE(std::filesystem::exists(baseUpper / "installed-tool")); + EXPECT_FALSE(std::filesystem::exists(runtimeUpper / "build-history")); + EXPECT_TRUE(std::filesystem::exists(directories.sources() / "source")); + auto state = linglong::utils::readFile(directories.buildOverlayState()); + ASSERT_TRUE(state); + EXPECT_EQ(*state, "second"); + + std::ofstream(baseUpper / "new-build-history") << "build changes"; + ASSERT_TRUE(directories.clean()); + EXPECT_FALSE(std::filesystem::exists(directories.overlays())); + auto cleaned = linglong::builder::detail::prepareBuildOverlays(directories, "second", true); + ASSERT_TRUE(cleaned); + EXPECT_FALSE(*cleaned); + EXPECT_FALSE(std::filesystem::exists(baseUpper / "new-build-history")); +} + +TEST(LinglongBuilder, BuildOverlaysResetWhenRuntimeIsAddedOrRemoved) +{ + TempDir workingDir; + linglong::builder::BuilderDirectoryLayout directories(workingDir.path()); + const auto baseUpper = directories.buildBaseOverlay() / "upperdir"; + const auto runtimeOverlay = directories.buildRuntimeOverlay(); + ASSERT_TRUE(linglong::builder::detail::prepareBuildOverlays(directories, "base", false)); + std::ofstream(baseUpper / "old-history") << "base changes"; + EXPECT_FALSE(std::filesystem::exists(runtimeOverlay)); + + auto added = linglong::builder::detail::prepareBuildOverlays(directories, "base-runtime", true); + ASSERT_TRUE(added); + EXPECT_FALSE(*added); + EXPECT_FALSE(std::filesystem::exists(baseUpper / "old-history")); + ASSERT_TRUE(std::filesystem::is_directory(runtimeOverlay / "upperdir")); + std::ofstream(runtimeOverlay / "upperdir/history") << "runtime changes"; + + auto removed = linglong::builder::detail::prepareBuildOverlays(directories, "base", false); + ASSERT_TRUE(removed); + EXPECT_FALSE(*removed); + EXPECT_FALSE(std::filesystem::exists(runtimeOverlay)); +} + +TEST(LinglongBuilder, BuildOverlaysResetWhenStateOrRequiredDirectoriesAreMissing) +{ + TempDir workingDir; + linglong::builder::BuilderDirectoryLayout directories(workingDir.path()); + const auto baseUpper = directories.buildBaseOverlay() / "upperdir"; + const auto runtimeUpper = directories.buildRuntimeOverlay() / "upperdir"; + ASSERT_TRUE(linglong::builder::detail::prepareBuildOverlays(directories, "dependencies", true)); + std::ofstream(baseUpper / "history") << "build changes"; + ASSERT_TRUE(std::filesystem::remove(directories.buildOverlayState())); + auto unknown = + linglong::builder::detail::prepareBuildOverlays(directories, "dependencies", true); + ASSERT_TRUE(unknown); + EXPECT_FALSE(*unknown); + EXPECT_FALSE(std::filesystem::exists(baseUpper / "history")); + + std::ofstream(baseUpper / "history") << "build changes"; + ASSERT_TRUE(std::filesystem::remove(runtimeUpper)); + auto incomplete = + linglong::builder::detail::prepareBuildOverlays(directories, "dependencies", true); + ASSERT_TRUE(incomplete); + EXPECT_FALSE(*incomplete); + EXPECT_FALSE(std::filesystem::exists(baseUpper / "history")); + EXPECT_TRUE(std::filesystem::is_directory(runtimeUpper)); +} + +TEST(LinglongBuilder, ResetsOverlaysWhenDependencyArchitectureChanges) +{ + TempDir temporary; + const linglong::builder::BuilderDirectoryLayout directories(temporary.path()); + const linglong::builder::LockedDependency base{ "org.test.Base/1.0.0", + { { "binary" }, "main:org.test.Base/1.0.0.0" }, + "base" }; + auto lock = linglong::builder::completeProjectLock({}, base); + for (const auto &name : { "x86_64", "arm64" }) { + lock.dependencies[0].resolved.reference = base.resolved.reference + "/" + name; + auto state = linglong::builder::projectLockDependencyState(lock); + ASSERT_TRUE(state) << state.error().message(); + auto prepared = linglong::builder::detail::prepareBuildOverlays(directories, *state, false); + ASSERT_TRUE(prepared) << prepared.error().message(); + EXPECT_FALSE(*prepared); + const auto history = directories.buildBaseOverlay() / "upperdir/history"; + EXPECT_FALSE(std::filesystem::exists(history)); + std::ofstream(history) << name; + auto reused = linglong::builder::detail::prepareBuildOverlays(directories, *state, false); + ASSERT_TRUE(reused); + EXPECT_TRUE(*reused); + EXPECT_TRUE(std::filesystem::exists(history)); + } +} + TEST(LinglongBuilder, installModule) { TempDir buildOutput; @@ -127,10 +739,11 @@ TEST(LinglongBuilder, mergeOutput) // Define merge targets std::vector targets = { "bin/", "share/data1" }; - linglong::builder::detail::mergeOutput({ srcPath1, srcPath2 }, - destDir.path(), - targets, - { "lib/systemd" }); + auto result = linglong::builder::detail::mergeOutput({ srcPath1, srcPath2 }, + destDir.path(), + targets, + { "lib/systemd" }); + ASSERT_TRUE(result) << result.error().message(); const auto &destPath = destDir.path(); // Check files are merged @@ -146,6 +759,39 @@ TEST(LinglongBuilder, mergeOutput) EXPECT_FALSE(std::filesystem::exists(destPath / "lib/systemd/dbus.service")); } +TEST(LinglongBuilder, mergeOutputReportsMissingSource) +{ + TempDir temporary; + const auto source = temporary.path() / "upper"; + std::filesystem::create_directories(source / "bin"); + std::ofstream(source / "bin/tool") << "tool"; + const auto destination = temporary.path() / "output"; + + auto result = linglong::builder::detail::mergeOutput({ temporary.path() / "missing", source }, + destination, + { "bin/" }, + {}); + ASSERT_FALSE(result); + EXPECT_NE(result.error().message().find((temporary.path() / "missing").string()), + std::string::npos); + EXPECT_FALSE(std::filesystem::exists(destination)); +} + +TEST(LinglongBuilder, mergeOutputReportsCopyFailures) +{ + TempDir temporary; + const auto source = temporary.path() / "upper"; + const auto destination = temporary.path() / "output"; + std::filesystem::create_directories(source / "bin"); + std::filesystem::create_directories(destination); + std::ofstream(source / "bin/tool") << "tool"; + std::ofstream(destination / "bin") << "existing file"; + + auto result = linglong::builder::detail::mergeOutput({ source }, destination, { "bin/" }, {}); + ASSERT_FALSE(result); + EXPECT_TRUE(std::filesystem::is_regular_file(destination / "bin")); +} + TEST(LinglongBuilder, UabExportFilename) { linglong::builder::BuilderMock builder; @@ -168,6 +814,47 @@ TEST(LinglongBuilder, LayerExportFilename) EXPECT_EQ(filename, "org.deepin.demo_1.0.0.0_arm64_binary.layer"); }; +TEST(BuilderDirectoryLayout, ResetsStageDirectoriesAndPreservesCache) +{ + TempDir workingDir; + linglong::builder::BuilderDirectoryLayout directories(workingDir.path()); + + EXPECT_EQ(directories.root(), workingDir.path() / "linglong"); + EXPECT_EQ(directories.cache(), directories.root() / "cache"); + EXPECT_EQ(directories.sources(), directories.root() / "sources"); + EXPECT_EQ(directories.buildOutput(), directories.root() / "output/_build"); + EXPECT_EQ(directories.buildBaseOverlay(), directories.overlays() / "build_base"); + EXPECT_EQ(directories.buildRuntimeOverlay(), directories.overlays() / "build_runtime"); + EXPECT_EQ(directories.prepareBaseOverlay(), directories.overlays() / "prepare_base"); + EXPECT_EQ(directories.prepareRuntimeOverlay(), directories.overlays() / "prepare_runtime"); + EXPECT_EQ(directories.buildOverlayState(), directories.overlays() / "build_overlay.state"); + EXPECT_EQ(directories.module("binary"), directories.root() / "output/binary"); + + ASSERT_TRUE(directories.ensureRoot()); + EXPECT_TRUE(std::filesystem::exists(directories.root())); + EXPECT_FALSE(std::filesystem::exists(directories.cache())); + + linglong::builder::SourceCacheLayout sourceCache(directories.cache()); + ASSERT_TRUE(sourceCache.ensure()); + EXPECT_TRUE(std::filesystem::exists(directories.cache())); + std::ofstream(directories.cache() / "persistent") << "cache"; + std::filesystem::create_directories(directories.sources()); + std::ofstream(directories.sources() / "stale") << "source"; + ASSERT_TRUE(directories.resetSources()); + EXPECT_TRUE(std::filesystem::exists(directories.sources())); + EXPECT_FALSE(std::filesystem::exists(directories.sources() / "stale")); + EXPECT_TRUE(std::filesystem::exists(directories.cache() / "persistent")); + + std::filesystem::create_directories(directories.output()); + std::ofstream(directories.output() / "stale") << "output"; + ASSERT_TRUE(directories.resetOutput()); + EXPECT_TRUE(std::filesystem::exists(directories.buildOutput())); + EXPECT_FALSE(std::filesystem::exists(directories.output() / "stale")); + + ASSERT_TRUE(directories.clean()); + EXPECT_FALSE(std::filesystem::exists(directories.root())); +} + TEST(LinglongBuilder, CleanNoBuildDir) { TempDir workingDir; diff --git a/libs/linglong/tests/ll-tests/src/linglong/builder/project_lock_test.cpp b/libs/linglong/tests/ll-tests/src/linglong/builder/project_lock_test.cpp new file mode 100644 index 000000000..8c45d3ffa --- /dev/null +++ b/libs/linglong/tests/ll-tests/src/linglong/builder/project_lock_test.cpp @@ -0,0 +1,1875 @@ +// SPDX-FileCopyrightText: 2026 UnionTech Software Technology Co., Ltd. +// SPDX-License-Identifier: LGPL-3.0-or-later + +#include + +#include "common/tempdir.h" +#include "linglong/api/types/v1/BuilderProjectLock.hpp" +#include "linglong/api/types/v1/Generators.hpp" +#include "linglong/builder/project_lock.h" +#include "linglong/builder/source_cache.h" +#include "linglong/builder/source_fetcher.h" +#include "linglong/package/architecture.h" +#include "linglong/utils/cmd.h" +#include "linglong/utils/env.h" +#include "linglong/utils/serialize/yaml.h" +#include "linglong/utils/temporary_directory.h" + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace linglong::builder { + +enum class SourcePreparationMode { UseOrGenerate, Locked, Generate, Update }; + +struct FetchedProjectSources +{ + std::vector entries; + bool changed; +}; + +static auto +resolveAndFetchProjectSources(const std::vector &sources, + const std::filesystem::path &cache, + const std::filesystem::path &destination, + bool offline, + const std::optional &existingLock) + -> utils::error::Result +{ + LINGLONG_TRACE("plan and fetch test project sources"); + + auto plan = resolveProjectLockSources(sources, existingLock); + if (!plan) { + return LINGLONG_ERR(plan); + } + + auto entries = fetchProjectSources(*plan, cache, destination, offline); + if (!entries) { + return LINGLONG_ERR(entries); + } + + return FetchedProjectSources{ plan->changed ? std::move(*entries) : existingLock->sources, + plan->changed }; +} + +static auto verifyAndFetchProjectSources( + const std::vector &sources, + const std::filesystem::path &cache, + const std::filesystem::path &destination, + bool offline, + const api::types::v1::BuilderProjectLock &lock) -> utils::error::Result +{ + LINGLONG_TRACE("verify and fetch test project sources"); + + auto plan = verifyProjectLockSources(sources, lock); + if (!plan) { + return LINGLONG_ERR(plan); + } + + auto entries = fetchProjectSources(*plan, cache, destination, offline); + if (!entries) { + return LINGLONG_ERR(entries); + } + + return LINGLONG_OK; +} + +static auto prepareSourcesFromManifest( + const std::vector &sources, + const std::filesystem::path &manifest, + const std::filesystem::path &cache, + SourcePreparationMode mode) -> utils::error::Result +{ + LINGLONG_TRACE("prepare test sources from manifest"); + + // Command preconditions are covered by Builder tests; select the source resolution input here. + std::optional loaded; + const auto path = projectLockPath(manifest); + if (mode == SourcePreparationMode::Locked + || (mode == SourcePreparationMode::UseOrGenerate && std::filesystem::exists(path))) { + auto lock = readProjectLock(path); + if (!lock) { + return LINGLONG_ERR(lock); + } + loaded = std::move(*lock); + } + + auto temporaryDestination = utils::TemporaryDirectory::create("fetch-"); + if (!temporaryDestination) { + return LINGLONG_ERR(temporaryDestination); + } + if (mode == SourcePreparationMode::Locked) { + auto verified = verifyAndFetchProjectSources(sources, + cache, + temporaryDestination->path(), + false, + *loaded); + if (!verified) { + return LINGLONG_ERR(verified); + } + return FetchedProjectSources{ loaded->sources, false }; + } + + return resolveAndFetchProjectSources(sources, + cache, + temporaryDestination->path(), + false, + loaded); +} + +class ProjectLockSourceTest : public ::testing::Test +{ +protected: + void SetUp() override + { + std::ofstream(manifest) << "version: 1\n"; + const auto commands = temporary.path() / "bin"; + std::filesystem::create_directories(commands); + const auto downloader = commands / "wget"; + // Use local files as remote artifacts and record every download request. + std::ofstream(downloader) << R"(#!/bin/sh +set -eu +test "$#" -eq 3 +test "$2" = "-O" +printf '%s\n' "$1" >> "$LINGLONG_TEST_SOURCE_DOWNLOAD_LOG" +/bin/cp -- "$1" "$3" +)"; + std::error_code ec; + std::filesystem::permissions(downloader, std::filesystem::perms::owner_all, ec); + ASSERT_FALSE(ec) << ec.message(); + downloadLog = + std::make_unique("LINGLONG_TEST_SOURCE_DOWNLOAD_LOG", + downloads.string()); + const auto *originalPath = std::getenv("PATH"); + commandPath = std::make_unique( + "PATH", + commands.string() + ":" + (originalPath ? originalPath : "")); + } + + auto fileSource(const std::string &name, const std::string &contents) + -> api::types::v1::BuilderProjectSource + { + const auto artifact = temporary.path() / name; + std::ofstream(artifact) << contents; + auto digest = fileHash(artifact); + EXPECT_TRUE(digest.has_value()); + api::types::v1::BuilderProjectSource source; + source.kind = "file"; + source.name = name; + source.url = artifact.string(); + if (digest) { + source.digest = *digest; + } + return source; + } + + auto generateLock(const std::vector &sources) + -> utils::error::Result + { + LINGLONG_TRACE("generate test project lock"); + auto generated = + prepareSourcesFromManifest(sources, manifest, cache, SourcePreparationMode::Generate); + if (!generated) { + return LINGLONG_ERR(generated); + } + const LockedDependency base{ "org.test.Base", + { { "binary" }, "main:org.test.Base/1.0.0" }, + "base" }; + auto lock = completeProjectLock(generated->entries, base); + auto written = writeCompleteProjectLock(manifest, lock); + if (!written) { + return LINGLONG_ERR(written); + } + return lock; + } + + auto downloadRequests() const -> std::vector + { + std::ifstream log(downloads); + std::vector requests; + std::string line; + while (std::getline(log, line)) { + requests.push_back(line); + } + return requests; + } + + TempDir temporary; + const std::filesystem::path manifest = temporary.path() / "linglong.yaml"; + const std::filesystem::path cache = temporary.path() / "cache"; + const std::filesystem::path downloads = temporary.path() / "downloads"; + std::unique_ptr downloadLog; + std::unique_ptr commandPath; +}; + +class ProjectLockGitTest : public ProjectLockSourceTest +{ +protected: + void SetUp() override + { + ProjectLockSourceTest::SetUp(); + if (HasFatalFailure()) { + return; + } + if (!git.exists()) { + GTEST_SKIP() << "git is not available"; + } + ASSERT_TRUE(git.exec({ "init", "--initial-branch=main", repository.string() })); + ASSERT_TRUE(git.exec({ "-C", repository.string(), "config", "user.name", "Test" })); + ASSERT_TRUE( + git.exec({ "-C", repository.string(), "config", "user.email", "test@example.com" })); + auto committed = commitSource("initial content"); + ASSERT_TRUE(committed) << committed.error().message(); + initialCommit = *committed; + source.kind = "git"; + source.url = repository.string(); + source.commit = "main"; + source.name = "source"; + source.submodules = false; + } + + auto gitHead(const std::filesystem::path &checkout) -> utils::error::Result + { + LINGLONG_TRACE("read test git HEAD"); + auto head = git.exec({ "-C", checkout.string(), "rev-parse", "HEAD" }); + if (!head) { + return LINGLONG_ERR(head); + } + return head->substr(0, head->find_first_of("\r\n")); + } + + auto commitSource(const std::string &contents) -> utils::error::Result + { + LINGLONG_TRACE("commit test git source"); + std::ofstream(repository / "source.txt") << contents; + auto added = git.exec({ "-C", repository.string(), "add", "source.txt" }); + if (!added) { + return LINGLONG_ERR(added); + } + auto committed = git.exec({ "-C", repository.string(), "commit", "-m", contents }); + if (!committed) { + return LINGLONG_ERR(committed); + } + return gitHead(repository); + } + + utils::Cmd git{ "git" }; + const std::filesystem::path repository = temporary.path() / "repository"; + api::types::v1::BuilderProjectSource source; + std::string initialCommit; +}; + +TEST_F(ProjectLockGitTest, BranchAdvanceKeepsLockedHashUntilExplicitUpdate) +{ + auto lock = generateLock({ source }); + ASSERT_TRUE(lock) << lock.error().message(); + ASSERT_EQ(lock->sources.front().resolved.hash, "git-commit:" + initialCommit); + const auto serialized = nlohmann::json(*lock); + EXPECT_EQ(serialized["sources"][0]["source"]["commit"], "main"); + EXPECT_FALSE(serialized["sources"][0]["resolved"].contains("commit")); + + auto advanced = commitSource("new content"); + ASSERT_TRUE(advanced) << advanced.error().message(); + ASSERT_NE(*advanced, initialCommit); + const std::vector caches{ cache, temporary.path() / "empty-cache" }; + for (std::size_t index = 0; index < caches.size(); ++index) { + SCOPED_TRACE(index); + const auto destination = temporary.path() / ("normal-" + std::to_string(index)); + auto resolved = + resolveAndFetchProjectSources({ source }, caches[index], destination, false, *lock); + ASSERT_TRUE(resolved) << resolved.error().message(); + EXPECT_FALSE(resolved->changed); + EXPECT_EQ(resolved->entries.front().resolved.hash, "git-commit:" + initialCommit); + auto head = gitHead(destination / *source.name); + ASSERT_TRUE(head) << head.error().message(); + EXPECT_EQ(*head, initialCommit); + } + const auto lockedDestination = temporary.path() / "locked"; + auto verified = verifyAndFetchProjectSources({ source }, cache, lockedDestination, true, *lock); + ASSERT_TRUE(verified) << verified.error().message(); + auto lockedHead = gitHead(lockedDestination / *source.name); + ASSERT_TRUE(lockedHead) << lockedHead.error().message(); + EXPECT_EQ(*lockedHead, initialCommit); + + auto updated = + prepareSourcesFromManifest({ source }, manifest, cache, SourcePreparationMode::Update); + ASSERT_TRUE(updated) << updated.error().message(); + EXPECT_TRUE(updated->changed); + EXPECT_EQ(updated->entries.front().resolved.hash, "git-commit:" + *advanced); + ASSERT_TRUE( + writeCompleteProjectLock(manifest, + completeProjectLock(updated->entries, lock->dependencies.front()))); + auto reused = + prepareSourcesFromManifest({ source }, manifest, cache, SourcePreparationMode::UseOrGenerate); + ASSERT_TRUE(reused) << reused.error().message(); + EXPECT_FALSE(reused->changed); + EXPECT_EQ(reused->entries.front().resolved.hash, "git-commit:" + *advanced); +} + +TEST_F(ProjectLockGitTest, RejectsDuplicateRequestsAfterBranchAdvances) +{ + source.name = "A"; + auto lock = generateLock({ source }); + ASSERT_TRUE(lock) << lock.error().message(); + const auto previous = nlohmann::json(*lock); + + auto advanced = commitSource("new content"); + ASSERT_TRUE(advanced) << advanced.error().message(); + ASSERT_NE(*advanced, initialCommit); + auto duplicate = source; + duplicate.name = "B"; + duplicate.version = "different description"; + + for (const auto mode : { SourcePreparationMode::UseOrGenerate, + SourcePreparationMode::Locked, + SourcePreparationMode::Generate, + SourcePreparationMode::Update }) { + SCOPED_TRACE(static_cast(mode)); + auto result = prepareSourcesFromManifest({ source, duplicate }, manifest, cache, mode); + ASSERT_FALSE(result); + EXPECT_NE(result.error().message().find("duplicate source fetch request: A and B"), + std::string::npos) + << result.error().message(); + } + + SourceCacheLayout layout(cache); + EXPECT_FALSE(std::filesystem::exists(layout.gitObject(*advanced, false))); + auto preserved = readProjectLock(projectLockPath(manifest)); + ASSERT_TRUE(preserved) << preserved.error().message(); + EXPECT_EQ(nlohmann::json(*preserved), previous); +} + +TEST_F(ProjectLockGitTest, ChangedBranchNameResolvesNewCommitInNormalBuild) +{ + auto lock = generateLock({ source }); + ASSERT_TRUE(lock) << lock.error().message(); + auto advanced = commitSource("new content"); + ASSERT_TRUE(advanced) << advanced.error().message(); + ASSERT_TRUE(git.exec({ "-C", repository.string(), "branch", "other" })); + auto changed = source; + changed.commit = "other"; + auto verified = + verifyAndFetchProjectSources({ changed }, cache, temporary.path() / "locked", false, *lock); + ASSERT_FALSE(verified); + EXPECT_NE(verified.error().message().find("sources differ from linglong.lock"), + std::string::npos); + + const auto destination = temporary.path() / "changed"; + auto resolved = resolveAndFetchProjectSources({ changed }, cache, destination, false, *lock); + ASSERT_TRUE(resolved) << resolved.error().message(); + EXPECT_TRUE(resolved->changed); + EXPECT_EQ(resolved->entries.front().source.commit, changed.commit); + EXPECT_EQ(resolved->entries.front().resolved.hash, "git-commit:" + *advanced); + auto head = gitHead(destination / *changed.name); + ASSERT_TRUE(head) << head.error().message(); + EXPECT_EQ(*head, *advanced); +} + +TEST_F(ProjectLockGitTest, ReadsLegacyResolvedCommitAndWritesOnlyHash) +{ + auto lock = generateLock({ source }); + ASSERT_TRUE(lock) << lock.error().message(); + auto legacy = nlohmann::json(*lock); + legacy["sources"][0]["resolved"]["commit"] = initialCommit; + // JSON is valid YAML, so this exercises the project lock file reader as well. + std::ofstream(projectLockPath(manifest)) << legacy.dump(2) << '\n'; + auto loaded = readProjectLock(projectLockPath(manifest)); + ASSERT_TRUE(loaded) << loaded.error().message(); + EXPECT_EQ(loaded->sources.front().resolved.hash, "git-commit:" + initialCommit); + auto verified = + verifyAndFetchProjectSources({ source }, cache, temporary.path() / "locked", true, *loaded); + ASSERT_TRUE(verified) << verified.error().message(); + ASSERT_TRUE(writeCompleteProjectLock(manifest, *loaded)); + const auto written = YAML::LoadFile(projectLockPath(manifest).string()); + EXPECT_EQ(written["sources"][0]["source"]["commit"].as(), "main"); + EXPECT_FALSE(written["sources"][0]["resolved"]["commit"].IsDefined()); +} + +TEST_F(ProjectLockGitTest, InvalidHashDoesNotFallBackToDeclaredBranch) +{ + auto lock = generateLock({ source }); + ASSERT_TRUE(lock) << lock.error().message(); + lock->sources.front().resolved.hash = "git-commit:main"; + auto verified = + verifyAndFetchProjectSources({ source }, cache, temporary.path() / "locked", false, *lock); + ASSERT_FALSE(verified); + EXPECT_NE(verified.error().message().find("git source has an invalid hash"), std::string::npos); + + SourceFetcher fetcher(LockedSourceRequest{ source, lock->sources.front().resolved }, + SourceCacheLayout(cache)); + const auto destination = temporary.path() / "fetched"; + std::filesystem::create_directories(destination); + auto fetched = fetcher.fetch(destination); + ASSERT_FALSE(fetched); + EXPECT_NE(fetched.error().message().find("invalid git source hash"), std::string::npos); +} + +TEST(ProjectLock, DerivesLockPathFromManifest) +{ + EXPECT_EQ(projectLockPath("/project/linglong.yaml"), "/project/linglong.lock"); + EXPECT_EQ(projectLockPath("/project/linglong.x86_64.yaml"), "/project/linglong.x86_64.lock"); +} + +TEST(SourceCacheLayout, DefinesObjectRootsAndCleansTemporaryDirectories) +{ + TempDir temporary; + SourceCacheLayout cache(temporary.path() / "cache"); + + EXPECT_EQ(cache.root(), temporary.path() / "cache/sources/v1"); + EXPECT_EQ(cache.object("file", "sha256:abc"), cache.root() / "file/sha256/abc"); + EXPECT_EQ(cache.object("archive", "sha256:abc"), cache.root() / "archive/sha256/abc"); + EXPECT_EQ(cache.object("dsc", "sha256:abc"), cache.root() / "dsc/sha256/abc"); + EXPECT_EQ(cache.gitObject("ABCDEF", true), cache.root() / "git/commit/abcdef-submodules"); + EXPECT_EQ(cache.gitObject("ABCDEF", false), cache.root() / "git/commit/abcdef-no-submodules"); + EXPECT_EQ(cache.temporary(), cache.root() / "tmp"); + EXPECT_EQ(cache.lockFile(cache.object("file", "sha256:abc")), + cache.root() / "locks/file/sha256/abc.lock"); + EXPECT_EQ(cache.lockFile(cache.gitObject("ABCDEF", true)), + cache.root() / "locks/git/commit/abcdef-submodules.lock"); + EXPECT_NE(cache.lockFile(cache.gitObject("ABCDEF", true)), + cache.lockFile(cache.gitObject("ABCDEF", false))); + + std::filesystem::path temporaryPath; + { + ASSERT_TRUE(cache.ensure()); + auto directory = utils::TemporaryDirectory::create("test-", cache.temporary()); + ASSERT_TRUE(directory.has_value()) << directory.error().message(); + temporaryPath = directory->path(); + EXPECT_EQ(temporaryPath.parent_path(), cache.temporary()); + std::ofstream(temporaryPath / "file") << "temporary"; + } + EXPECT_FALSE(std::filesystem::exists(temporaryPath)); + EXPECT_TRUE(std::filesystem::exists(cache.temporary())); +} + +TEST(ProjectLock, ReadRequiresExistingFile) +{ + TempDir temporary; + const auto manifest = temporary.path() / "linglong.yaml"; + std::ofstream(manifest) << "version: 1\n"; + + auto result = readProjectLock(projectLockPath(manifest)); + EXPECT_FALSE(result.has_value()); +} + +TEST(ProjectLock, ReadRejectsUnsupportedFormatVersion) +{ + TempDir temporary; + const auto path = temporary.path() / "linglong.lock"; + std::ofstream(path) << "version: 2\nsources: []\ndependencies: []\n"; + + auto result = readProjectLock(path); + ASSERT_FALSE(result); + EXPECT_NE(result.error().message().find("unsupported project lock version"), std::string::npos); +} + +TEST(ProjectLock, UnspecifiedArchitectureUsesBuildHost) +{ + TempDir temporary; + const auto manifest = temporary.path() / "linglong.yaml"; + const LockedDependency base{ "org.test.Base/1.0", + { { "binary" }, "main:org.test.Base/1.0.0/unknown" }, + "base" }; + const auto lock = completeProjectLock({}, base); + ASSERT_TRUE(writeCompleteProjectLock(manifest, lock)); + auto loaded = readProjectLock(projectLockPath(manifest)); + ASSERT_TRUE(loaded) << loaded.error().message(); + const auto serialized = nlohmann::json(*loaded); + EXPECT_FALSE(serialized.contains("build_architecture")); + EXPECT_EQ(loaded->dependencies.front().resolved.reference, base.resolved.reference); + + auto reference = lockedDependencyReference(loaded->dependencies.front()); + ASSERT_TRUE(reference) << reference.error().message(); + EXPECT_EQ(reference->arch, package::Architecture::currentCPUArchitecture()); + EXPECT_EQ(reference->version.toString(), "1.0.0"); + EXPECT_EQ(nlohmann::json(*loaded), serialized); +} + +TEST(ProjectLock, LockedDependencyPreservesExplicitArchitecture) +{ + TempDir temporary; + const auto manifest = temporary.path() / "linglong.yaml"; + const LockedDependency base{ "org.test.Base/1.0/arm64", + { { "binary" }, "main:org.test.Base/1.0.0/arm64" }, + "base" }; + ASSERT_TRUE(writeCompleteProjectLock(manifest, completeProjectLock({}, base))); + auto loaded = readProjectLock(projectLockPath(manifest)); + ASSERT_TRUE(loaded) << loaded.error().message(); + auto reference = lockedDependencyReference(loaded->dependencies.front()); + ASSERT_TRUE(reference) << reference.error().message(); + EXPECT_EQ(reference->toString(), base.resolved.reference); +} + +TEST(ProjectLock, LockedDependencyRequiresAnExactReference) +{ + for (const auto &reference : { "org.test.Base/1.0.0", + "main:org.test.Base", + "main:org.test.Base/unknown", + "main:org.test.Base/1.0.0/unkown" }) { + SCOPED_TRACE(reference); + const LockedDependency base{ "org.test.Base", { { "binary" }, reference }, "base" }; + EXPECT_FALSE(lockedDependencyReference(base)); + } +} + +TEST(ProjectLock, FetchEmptySourcePlanPreparesDirectories) +{ + for (const bool offline : { false, true }) { + SCOPED_TRACE(offline); + TempDir temporary; + const auto cache = temporary.path() / "cache"; + const auto destination = temporary.path() / "sources"; + int progressCalls = 0; + const auto progress = + [&](std::size_t, const api::types::v1::BuilderProjectSource &, SourceFetchStatus) { + ++progressCalls; + }; + auto entries = fetchProjectSources({}, cache, destination, offline, progress); + ASSERT_TRUE(entries) << entries.error().message(); + EXPECT_TRUE(entries->empty()); + EXPECT_EQ(progressCalls, 0); + EXPECT_TRUE(std::filesystem::is_directory(SourceCacheLayout(cache).temporary())); + EXPECT_TRUE(std::filesystem::is_directory(destination)); + } +} + +TEST(ProjectLock, FetchInvalidSourceRequestReportsError) +{ + TempDir temporary; + const auto cache = temporary.path() / "cache"; + const auto destination = temporary.path() / "sources"; + api::types::v1::BuilderProjectSource source; + source.kind = "file"; + source.name = "source"; + ProjectSourcePlan plan; + plan.requests.emplace_back(GenerateSourceRequest{ source }); + auto entries = fetchProjectSources(plan, cache, destination, false); + ASSERT_FALSE(entries); + EXPECT_NE(entries.error().message().find("source URL is missing"), std::string::npos); + EXPECT_TRUE(std::filesystem::is_directory(SourceCacheLayout(cache).temporary())); + EXPECT_TRUE(std::filesystem::is_directory(destination)); +} + +TEST(ProjectLock, GeneratesAndUpdatesEmptyLock) +{ + TempDir temporary; + const auto manifest = temporary.path() / "linglong.yaml"; + std::ofstream(manifest) << "version: 1\n"; + + auto generated = prepareSourcesFromManifest({}, + manifest, + temporary.path() / "cache", + SourcePreparationMode::Generate); + ASSERT_TRUE(generated.has_value()) << generated.error().message(); + EXPECT_TRUE(generated->changed); + EXPECT_FALSE(std::filesystem::exists(projectLockPath(manifest))); + const LockedDependency base{ "org.test.Base", + { { "binary", "develop" }, "main:org.test.Base/1.0.0" }, + "base" }; + ASSERT_TRUE(writeCompleteProjectLock(manifest, completeProjectLock(generated->entries, base))); + std::error_code ec; + EXPECT_TRUE(std::filesystem::exists(projectLockPath(manifest), ec)); + EXPECT_FALSE(ec); + auto lock = + utils::serialize::LoadYAMLFile(projectLockPath(manifest)); + ASSERT_TRUE(lock.has_value()) << lock.error().message(); + EXPECT_EQ(lock->version, 1); + EXPECT_TRUE(lock->sources.empty()); + + auto updated = prepareSourcesFromManifest({}, + manifest, + temporary.path() / "cache", + SourcePreparationMode::Update); + EXPECT_TRUE(updated.has_value()) << updated.error().message(); + EXPECT_TRUE(updated->changed); + api::types::v1::BuilderProjectLock incomplete; + incomplete.version = 1; + incomplete.sources = updated->entries; + EXPECT_FALSE(writeCompleteProjectLock(manifest, incomplete).has_value()); + EXPECT_TRUE(std::filesystem::exists(projectLockPath(manifest))); + + EXPECT_FALSE(std::filesystem::exists(temporary.path() / "cache/sources/v1/.lock", ec)); + EXPECT_FALSE(ec); +} + +TEST(ProjectLock, RecordsCompleteDependenciesAndKeepsExistingLockReadOnly) +{ + TempDir temporary; + const auto manifest = temporary.path() / "linglong.yaml"; + std::ofstream(manifest) << "version: 1\n"; + auto generated = prepareSourcesFromManifest({}, + manifest, + temporary.path() / "cache", + SourcePreparationMode::Generate); + ASSERT_TRUE(generated.has_value()) << generated.error().message(); + const LockedDependency base{ "org.test.Base", + { { "binary", "develop", "locale" }, "main:org.test.Base/1.0.0" }, + "base" }; + const LockedDependency runtime{ "org.test.Runtime", + { { "binary", "develop" }, "main:org.test.Runtime/2.0.0" }, + "runtime" }; + auto complete = completeProjectLock(generated->entries, base, runtime); + ASSERT_TRUE(writeCompleteProjectLock(manifest, complete)); + + auto lock = + utils::serialize::LoadYAMLFile(projectLockPath(manifest)); + ASSERT_TRUE(lock.has_value()) << lock.error().message(); + ASSERT_EQ(lock->dependencies.size(), 2); + EXPECT_EQ(lock->dependencies[0].type, "base"); + EXPECT_EQ(lock->dependencies[0].resolved.modules, + (std::vector{ "binary", "develop", "locale" })); + EXPECT_EQ(lock->dependencies[1].type, "runtime"); + auto state = projectLockDependencyState(*lock); + ASSERT_TRUE(state.has_value()) << state.error().message(); + auto preparedState = projectLockDependencyState(complete); + ASSERT_TRUE(preparedState.has_value()) << preparedState.error().message(); + EXPECT_EQ(*state, *preparedState); + const auto stateJson = nlohmann::json::parse(*state); + ASSERT_TRUE(stateJson.contains("base")); + ASSERT_TRUE(stateJson.contains("runtime")); + EXPECT_EQ(stateJson.at("base").at("reference"), + "main:org.test.Base/1.0.0/" + + package::Architecture::currentCPUArchitecture().toString()); + EXPECT_EQ(stateJson.at("base").at("modules").at(2), "locale"); + EXPECT_FALSE(stateJson.at("base").contains("requested")); + EXPECT_FALSE(stateJson.at("base").contains("commit")); + + auto locked = prepareSourcesFromManifest({}, + manifest, + temporary.path() / "cache", + SourcePreparationMode::Locked); + ASSERT_TRUE(locked.has_value()) << locked.error().message(); + EXPECT_FALSE(locked->changed); + EXPECT_TRUE(locked->entries.empty()); + + auto updated = prepareSourcesFromManifest({}, + manifest, + temporary.path() / "cache", + SourcePreparationMode::Update); + ASSERT_TRUE(updated.has_value()) << updated.error().message(); + EXPECT_TRUE(updated->changed); + auto loaded = readProjectLock(projectLockPath(manifest)); + ASSERT_TRUE(loaded.has_value()) << loaded.error().message(); + auto unchangedState = projectLockDependencyState(*loaded); + ASSERT_TRUE(unchangedState); + EXPECT_EQ(*state, *unchangedState); + + ASSERT_TRUE(std::filesystem::remove(projectLockPath(manifest))); + auto reused = verifyAndFetchProjectSources({}, + temporary.path() / "cache", + temporary.path() / "sources", + false, + *loaded); + ASSERT_TRUE(reused.has_value()) << reused.error().message(); +} + +TEST(ProjectLock, MissingAndEmptySourcesHaveTheSamePlanningSemantics) +{ + const std::vector>> + declarations{ + std::nullopt, + std::vector{}, + }; + api::types::v1::BuilderProjectLock emptyLock{}; + api::types::v1::BuilderProjectLock populatedLock{}; + api::types::v1::BuilderProjectSource source; + source.kind = "file"; + source.url = "https://example.com/source"; + populatedLock.sources = { { { "sha256:" + std::string(64, 'a') }, source } }; + + for (const auto &sources : declarations) { + SCOPED_TRACE(sources.has_value()); + auto fresh = resolveProjectLockSources(sources, std::nullopt); + ASSERT_TRUE(fresh) << fresh.error().message(); + EXPECT_TRUE(fresh->changed); + EXPECT_TRUE(fresh->requests.empty()); + + auto unchanged = resolveProjectLockSources(sources, emptyLock); + ASSERT_TRUE(unchanged) << unchanged.error().message(); + EXPECT_FALSE(unchanged->changed); + EXPECT_TRUE(unchanged->requests.empty()); + + auto removed = resolveProjectLockSources(sources, populatedLock); + ASSERT_TRUE(removed) << removed.error().message(); + EXPECT_TRUE(removed->changed); + EXPECT_TRUE(removed->requests.empty()); + + auto verified = verifyProjectLockSources(sources, emptyLock); + ASSERT_TRUE(verified) << verified.error().message(); + EXPECT_FALSE(verified->changed); + EXPECT_TRUE(verified->requests.empty()); + + auto mismatch = verifyProjectLockSources(sources, populatedLock); + ASSERT_FALSE(mismatch); + EXPECT_NE(mismatch.error().message().find("sources differ from linglong.lock"), + std::string::npos); + EXPECT_EQ(populatedLock.sources.size(), 1); + } +} + +TEST_F(ProjectLockSourceTest, PlansLockedAndFreshSourcesWithoutFetching) +{ + auto first = fileSource("first", "locked content"); + auto second = fileSource("second", "new content"); + api::types::v1::BuilderProjectLock lock{}; + lock.sources = { { { *first.digest }, first } }; + ASSERT_TRUE(std::filesystem::remove(*first.url)); + ASSERT_TRUE(std::filesystem::remove(*second.url)); + + auto plan = resolveProjectLockSources(std::vector{ first, second }, lock); + ASSERT_TRUE(plan) << plan.error().message(); + EXPECT_TRUE(plan->changed); + ASSERT_EQ(plan->requests.size(), 2); + ASSERT_TRUE(std::holds_alternative(plan->requests[0])); + EXPECT_EQ(std::get(plan->requests[0]).resolved.hash, *first.digest); + ASSERT_TRUE(std::holds_alternative(plan->requests[1])); + EXPECT_EQ(std::get(plan->requests[1]).source.url, second.url); + + auto verified = verifyProjectLockSources(std::vector{ first }, lock); + ASSERT_TRUE(verified) << verified.error().message(); + EXPECT_FALSE(verified->changed); + ASSERT_EQ(verified->requests.size(), 1); + EXPECT_TRUE(std::holds_alternative(verified->requests.front())); + EXPECT_TRUE(downloadRequests().empty()); + EXPECT_FALSE(std::filesystem::exists(cache)); +} + +TEST_F(ProjectLockSourceTest, FetchStartsOnlyAfterExplicitCall) +{ + auto source = fileSource("source", "source content"); + const auto destination = temporary.path() / "sources"; + auto plan = resolveProjectLockSources(std::vector{ source }, std::nullopt); + ASSERT_TRUE(plan) << plan.error().message(); + EXPECT_TRUE(downloadRequests().empty()); + EXPECT_FALSE(std::filesystem::exists(cache)); + EXPECT_FALSE(std::filesystem::exists(destination)); + + auto entries = fetchProjectSources(*plan, cache, destination, false); + ASSERT_TRUE(entries) << entries.error().message(); + ASSERT_EQ(entries->size(), 1); + EXPECT_EQ(entries->front().resolved.hash, *source.digest); + EXPECT_EQ(downloadRequests(), (std::vector{ *source.url })); + auto outputHash = fileHash(destination / *source.name); + ASSERT_TRUE(outputHash) << outputHash.error().message(); + EXPECT_EQ(*outputHash, *source.digest); + EXPECT_TRUE(std::filesystem::is_directory(SourceCacheLayout(cache).temporary())); +} + +TEST_F(ProjectLockSourceTest, ReportsProgressForFreshAndCachedSourcesInYamlOrder) +{ + const auto first = fileSource("first", "first content"); + const auto second = fileSource("second", "second content"); + const std::vector sources{ second, first }; + auto plan = resolveProjectLockSources(sources, std::nullopt); + ASSERT_TRUE(plan) << plan.error().message(); + using Event = std::pair; + std::vector events; + const auto progress = [&](std::size_t index, + const api::types::v1::BuilderProjectSource &source, + SourceFetchStatus status) { + ASSERT_LT(index, sources.size()); + EXPECT_EQ(source.url, sources[index].url); + EXPECT_EQ(source.name, sources[index].name); + events.emplace_back(index, status); + }; + + auto entries = fetchProjectSources(*plan, cache, temporary.path() / "fresh", false, progress); + ASSERT_TRUE(entries) << entries.error().message(); + const std::vector expected{ { 0, SourceFetchStatus::Fetching }, + { 0, SourceFetchStatus::Complete }, + { 1, SourceFetchStatus::Fetching }, + { 1, SourceFetchStatus::Complete } }; + EXPECT_EQ(events, expected); + EXPECT_EQ(downloadRequests(), (std::vector{ *second.url, *first.url })); + + api::types::v1::BuilderProjectLock lock{}; + lock.sources = *entries; + auto lockedPlan = verifyProjectLockSources(sources, lock); + ASSERT_TRUE(lockedPlan) << lockedPlan.error().message(); + ASSERT_TRUE(std::filesystem::remove(*first.url)); + ASSERT_TRUE(std::filesystem::remove(*second.url)); + events.clear(); + auto cached = + fetchProjectSources(*lockedPlan, cache, temporary.path() / "cached", true, progress); + ASSERT_TRUE(cached) << cached.error().message(); + EXPECT_EQ(events, expected); + EXPECT_EQ(downloadRequests().size(), 2); + EXPECT_TRUE(std::filesystem::exists(temporary.path() / "cached" / *first.name)); + EXPECT_TRUE(std::filesystem::exists(temporary.path() / "cached" / *second.name)); +} + +TEST_F(ProjectLockSourceTest, ReportsFailureAndStopsBeforeLaterSources) +{ + const auto first = fileSource("first", "first content"); + auto second = fileSource("second", "second content"); + second.digest = "sha256:" + std::string(64, '0'); + const auto third = fileSource("third", "third content"); + auto plan = resolveProjectLockSources(std::vector{ first, second, third }, std::nullopt); + ASSERT_TRUE(plan) << plan.error().message(); + using Event = std::pair; + std::vector events; + auto fetched = fetchProjectSources(*plan, + cache, + temporary.path() / "sources", + false, + [&](std::size_t index, + const api::types::v1::BuilderProjectSource &, + SourceFetchStatus status) { + events.emplace_back(index, status); + }); + + ASSERT_FALSE(fetched); + EXPECT_EQ(events, + (std::vector{ { 0, SourceFetchStatus::Fetching }, + { 0, SourceFetchStatus::Complete }, + { 1, SourceFetchStatus::Fetching }, + { 1, SourceFetchStatus::Failed } })); + EXPECT_EQ(downloadRequests(), (std::vector{ *first.url, *second.url })); + EXPECT_TRUE(std::filesystem::exists(temporary.path() / "sources" / *first.name)); + EXPECT_FALSE(std::filesystem::exists(temporary.path() / "sources" / *second.name)); + EXPECT_FALSE(std::filesystem::exists(temporary.path() / "sources" / *third.name)); +} + +TEST_F(ProjectLockSourceTest, PlanUsesYamlOrderAndNamesWithoutChangingLock) +{ + auto first = fileSource("first", "first content"); + auto second = fileSource("second", "second content"); + api::types::v1::BuilderProjectLock lock{}; + lock.version = 1; + lock.sources = { { { *second.digest }, second }, { { *first.digest }, first } }; + const auto previous = nlohmann::json(lock); + first.name = "renamed"; + first.version = "new version"; + + auto plan = resolveProjectLockSources(std::vector{ first, second }, lock); + ASSERT_TRUE(plan) << plan.error().message(); + EXPECT_FALSE(plan->changed); + ASSERT_EQ(plan->requests.size(), 2); + ASSERT_TRUE(std::holds_alternative(plan->requests[0])); + const auto &request = std::get(plan->requests[0]); + EXPECT_EQ(request.source.name, first.name); + EXPECT_EQ(request.source.version, first.version); + EXPECT_EQ(request.resolved.hash, *first.digest); + auto verified = verifyProjectLockSources(std::vector{ first, second }, lock); + ASSERT_TRUE(verified) << verified.error().message(); + EXPECT_FALSE(verified->changed); + EXPECT_EQ(nlohmann::json(lock), previous); + EXPECT_TRUE(downloadRequests().empty()); + EXPECT_FALSE(std::filesystem::exists(cache)); +} + +TEST_F(ProjectLockSourceTest, RemovedSourcesMarkPlanChanged) +{ + auto first = fileSource("first", "first content"); + auto second = fileSource("second", "second content"); + api::types::v1::BuilderProjectLock lock{}; + lock.sources = { { { *first.digest }, first }, { { *second.digest }, second } }; + + auto plan = resolveProjectLockSources(std::vector{ first }, lock); + ASSERT_TRUE(plan) << plan.error().message(); + EXPECT_TRUE(plan->changed); + ASSERT_EQ(plan->requests.size(), 1); + EXPECT_TRUE(std::holds_alternative(plan->requests.front())); + auto removedAll = resolveProjectLockSources(std::nullopt, lock); + ASSERT_TRUE(removedAll) << removedAll.error().message(); + EXPECT_TRUE(removedAll->changed); + EXPECT_TRUE(removedAll->requests.empty()); + auto verified = verifyProjectLockSources(std::vector{ first }, lock); + ASSERT_FALSE(verified); + EXPECT_NE(verified.error().message().find("sources differ from linglong.lock"), + std::string::npos); + EXPECT_TRUE(downloadRequests().empty()); + EXPECT_FALSE(std::filesystem::exists(cache)); +} + +TEST_F(ProjectLockSourceTest, NormalBuildUpdatesOnlyChangedSource) +{ + auto first = fileSource("first", "first version"); + auto second = fileSource("second", "second version"); + auto generated = generateLock({ first, second }); + ASSERT_TRUE(generated) << generated.error().message(); + const auto originalSecond = generated->sources[1].resolved.hash; + EXPECT_EQ(downloadRequests(), (std::vector{ *first.url, *second.url })); + + first = fileSource("first", "new first version"); + auto changed = prepareSourcesFromManifest({ first, second }, + manifest, + cache, + SourcePreparationMode::UseOrGenerate); + ASSERT_TRUE(changed) << changed.error().message(); + EXPECT_TRUE(changed->changed); + EXPECT_EQ(changed->entries[0].resolved.hash, *first.digest); + EXPECT_EQ(changed->entries[1].resolved.hash, originalSecond); + ASSERT_TRUE(writeCompleteProjectLock( + manifest, + completeProjectLock(changed->entries, generated->dependencies.front()))); + EXPECT_EQ(downloadRequests(), + (std::vector{ *first.url, *second.url, *first.url })); + + auto unchanged = prepareSourcesFromManifest({ first, second }, + manifest, + cache, + SourcePreparationMode::UseOrGenerate); + ASSERT_TRUE(unchanged) << unchanged.error().message(); + EXPECT_FALSE(unchanged->changed); + + auto locked = readProjectLock(projectLockPath(manifest)); + ASSERT_TRUE(locked) << locked.error().message(); + auto verified = verifyAndFetchProjectSources({ first, second }, + cache, + temporary.path() / "verified", + false, + *locked); + ASSERT_TRUE(verified) << verified.error().message(); + EXPECT_EQ(downloadRequests().size(), 3); + + auto explicitUpdate = + prepareSourcesFromManifest({ first, second }, manifest, cache, SourcePreparationMode::Update); + ASSERT_TRUE(explicitUpdate) << explicitUpdate.error().message(); + EXPECT_TRUE(explicitUpdate->changed); + EXPECT_EQ( + downloadRequests(), + (std::vector{ *first.url, *second.url, *first.url, *first.url, *second.url })); +} + +TEST_F(ProjectLockSourceTest, ChangedUrlDownloadsEvenWhenDigestIsCached) +{ + auto source = fileSource("original", "same content"); + auto lock = generateLock({ source }); + ASSERT_TRUE(lock) << lock.error().message(); + + auto changed = fileSource("replacement", "same content"); + changed.name = source.name; + ASSERT_EQ(changed.digest, source.digest); + auto resolved = prepareSourcesFromManifest({ changed }, + manifest, + cache, + SourcePreparationMode::UseOrGenerate); + ASSERT_TRUE(resolved) << resolved.error().message(); + EXPECT_TRUE(resolved->changed); + ASSERT_EQ(resolved->entries.size(), 1); + EXPECT_EQ(resolved->entries.front().source.url, changed.url); + EXPECT_EQ(resolved->entries.front().resolved.hash, *source.digest); + EXPECT_EQ(downloadRequests(), (std::vector{ *source.url, *changed.url })); + + ASSERT_TRUE( + writeCompleteProjectLock(manifest, + completeProjectLock(resolved->entries, lock->dependencies.front()))); + auto updated = readProjectLock(projectLockPath(manifest)); + ASSERT_TRUE(updated) << updated.error().message(); + EXPECT_EQ(updated->sources.front().source.url, changed.url); +} + +TEST_F(ProjectLockSourceTest, ChangedUrlWithStaleDigestFailsBeforePublishing) +{ + auto source = fileSource("original", "original content"); + auto lock = generateLock({ source }); + ASSERT_TRUE(lock) << lock.error().message(); + const auto previousLock = nlohmann::json(*lock).dump(); + + auto changed = fileSource("replacement", "different content"); + ASSERT_NE(changed.digest, source.digest); + const auto replacementDigest = *changed.digest; + changed.digest = source.digest; + changed.name = source.name; + const auto destination = temporary.path() / "changed"; + auto resolved = resolveAndFetchProjectSources({ changed }, cache, destination, false, *lock); + ASSERT_FALSE(resolved); + EXPECT_NE(resolved.error().message().find("source hash does not match digest in linglong.yaml"), + std::string::npos) + << resolved.error().message(); + EXPECT_EQ(downloadRequests(), (std::vector{ *source.url, *changed.url })); + EXPECT_FALSE(std::filesystem::exists(destination / *changed.name)); + SourceCacheLayout layout(cache); + EXPECT_FALSE(std::filesystem::exists(layout.object("file", replacementDigest))); + auto cachedHash = fileHash(layout.object("file", *source.digest)); + ASSERT_TRUE(cachedHash) << cachedHash.error().message(); + EXPECT_EQ(*cachedHash, *source.digest); + auto preserved = readProjectLock(projectLockPath(manifest)); + ASSERT_TRUE(preserved) << preserved.error().message(); + EXPECT_EQ(nlohmann::json(*preserved).dump(), previousLock); +} + +TEST_F(ProjectLockSourceTest, ChangedUrlWithoutDigestResolvesNewContent) +{ + auto source = fileSource("original", "original content"); + auto lock = generateLock({ source }); + ASSERT_TRUE(lock) << lock.error().message(); + + auto changed = fileSource("replacement", "different content"); + const auto replacementDigest = *changed.digest; + changed.digest.reset(); + changed.name = source.name; + const auto destination = temporary.path() / "changed"; + auto resolved = resolveAndFetchProjectSources({ changed }, cache, destination, false, *lock); + ASSERT_TRUE(resolved) << resolved.error().message(); + EXPECT_TRUE(resolved->changed); + ASSERT_EQ(resolved->entries.size(), 1); + EXPECT_EQ(resolved->entries.front().source.url, changed.url); + EXPECT_EQ(resolved->entries.front().resolved.hash, replacementDigest); + auto outputHash = fileHash(destination / *changed.name); + ASSERT_TRUE(outputHash) << outputHash.error().message(); + EXPECT_EQ(*outputHash, replacementDigest); + SourceCacheLayout layout(cache); + EXPECT_TRUE(std::filesystem::exists(layout.object("file", replacementDigest))); + EXPECT_EQ(downloadRequests(), (std::vector{ *source.url, *changed.url })); +} + +class ProjectLockDigestTest : public ProjectLockSourceTest, + public ::testing::WithParamInterface +{ +}; + +TEST_P(ProjectLockDigestTest, DigestChangesDownloadWithoutChangingUrl) +{ + auto source = fileSource("source", "original content"); + if (GetParam() == "Add") { + source.digest.reset(); + } + auto lock = generateLock({ source }); + ASSERT_TRUE(lock) << lock.error().message(); + + auto changed = source; + if (GetParam() == "Add") { + changed.digest = lock->sources.front().resolved.hash; + } else { + changed = fileSource("source", "updated content"); + if (GetParam() == "Remove") { + changed.digest.reset(); + } + } + ASSERT_EQ(changed.url, source.url); + ASSERT_NE(changed.digest, source.digest); + auto expectedHash = fileHash(*changed.url); + ASSERT_TRUE(expectedHash) << expectedHash.error().message(); + SourceCacheLayout layout(cache); + const auto cached = layout.object("file", *expectedHash); + if (!std::filesystem::exists(cached)) { + std::filesystem::copy_file(*changed.url, cached); + } + + auto verified = + verifyAndFetchProjectSources({ changed }, cache, temporary.path() / "locked", false, *lock); + ASSERT_FALSE(verified); + EXPECT_NE(verified.error().message().find("sources differ from linglong.lock"), + std::string::npos); + EXPECT_EQ(downloadRequests(), (std::vector{ *source.url })); + + const auto destination = temporary.path() / "changed"; + auto resolved = resolveAndFetchProjectSources({ changed }, cache, destination, false, *lock); + ASSERT_TRUE(resolved) << resolved.error().message(); + EXPECT_TRUE(resolved->changed); + ASSERT_EQ(resolved->entries.size(), 1); + EXPECT_EQ(resolved->entries.front().source.digest, changed.digest); + EXPECT_EQ(resolved->entries.front().resolved.hash, *expectedHash); + auto outputHash = fileHash(destination / *changed.name); + ASSERT_TRUE(outputHash) << outputHash.error().message(); + EXPECT_EQ(*outputHash, *expectedHash); + EXPECT_EQ(downloadRequests(), (std::vector{ *source.url, *source.url })); + + ASSERT_TRUE( + writeCompleteProjectLock(manifest, + completeProjectLock(resolved->entries, lock->dependencies.front()))); + auto reused = prepareSourcesFromManifest({ changed }, + manifest, + cache, + SourcePreparationMode::UseOrGenerate); + ASSERT_TRUE(reused) << reused.error().message(); + EXPECT_FALSE(reused->changed); + EXPECT_EQ(downloadRequests().size(), 2); +} + +INSTANTIATE_TEST_SUITE_P(DigestChanges, + ProjectLockDigestTest, + ::testing::Values("Modify", "Add", "Remove"), + [](const ::testing::TestParamInfo &info) { + return info.param; + }); + +TEST_F(ProjectLockSourceTest, ChangedDigestMustMatchDownloadedContent) +{ + auto source = fileSource("original", "original content"); + auto lock = generateLock({ source }); + ASSERT_TRUE(lock) << lock.error().message(); + + auto other = fileSource("other", "different content"); + SourceCacheLayout layout(cache); + std::filesystem::copy_file(*other.url, layout.object("file", *other.digest)); + auto changed = source; + changed.digest = other.digest; + ASSERT_NE(changed.digest, source.digest); + const auto destination = temporary.path() / "changed"; + auto resolved = resolveAndFetchProjectSources({ changed }, cache, destination, false, *lock); + ASSERT_FALSE(resolved); + EXPECT_NE(resolved.error().message().find("source hash does not match digest in linglong.yaml"), + std::string::npos) + << resolved.error().message(); + EXPECT_EQ(downloadRequests(), (std::vector{ *source.url, *source.url })); + EXPECT_FALSE(std::filesystem::exists(destination / *changed.name)); + auto preserved = readProjectLock(projectLockPath(manifest)); + ASSERT_TRUE(preserved) << preserved.error().message(); + EXPECT_EQ(nlohmann::json(*preserved), nlohmann::json(*lock)); +} + +TEST_F(ProjectLockSourceTest, OfflineRequiresMatchingLockEntryEvenWhenDigestIsCached) +{ + auto source = fileSource("original", "same content"); + auto lock = generateLock({ source }); + ASSERT_TRUE(lock) << lock.error().message(); + + auto changed = source; + changed.url = (temporary.path() / "replacement").string(); + auto changedResult = + resolveAndFetchProjectSources({ changed }, cache, temporary.path() / "changed", true, *lock); + ASSERT_FALSE(changedResult); + EXPECT_NE(changedResult.error().message().find("new or changed source in offline mode"), + std::string::npos); + + auto generated = resolveAndFetchProjectSources({ source }, + cache, + temporary.path() / "generated", + true, + std::nullopt); + EXPECT_FALSE(generated); + auto matched = + resolveAndFetchProjectSources({ source }, cache, temporary.path() / "matched", true, *lock); + ASSERT_TRUE(matched) << matched.error().message(); + EXPECT_FALSE(matched->changed); + EXPECT_EQ(downloadRequests(), (std::vector{ *source.url })); +} + +TEST(ProjectLock, NonGitSubmodulesDoNotChangeSourceMatching) +{ + const std::vector> values{ std::nullopt, false, true }; + for (const auto &kind : { "file", "archive", "dsc" }) { + for (const auto &previous : values) { + api::types::v1::BuilderProjectSource source; + source.kind = kind; + source.url = "https://example.com/source"; + source.submodules = previous; + api::types::v1::BuilderProjectLock lock{}; + lock.sources = { { { "sha256:" + std::string(64, '0') }, source } }; + const auto originalLock = nlohmann::json(lock); + + for (const auto ¤t : values) { + source.submodules = current; + SCOPED_TRACE(originalLock.dump()); + SCOPED_TRACE(nlohmann::json(source).dump()); + auto plan = resolveProjectLockSources(std::vector{ source }, lock); + ASSERT_TRUE(plan) << plan.error().message(); + EXPECT_FALSE(plan->changed); + ASSERT_EQ(plan->requests.size(), 1); + EXPECT_TRUE(std::holds_alternative(plan->requests.front())); + + auto verified = verifyProjectLockSources(std::vector{ source }, lock); + ASSERT_TRUE(verified) << verified.error().message(); + EXPECT_FALSE(verified->changed); + ASSERT_EQ(verified->requests.size(), 1); + EXPECT_TRUE( + std::holds_alternative(verified->requests.front())); + EXPECT_EQ(nlohmann::json(lock), originalLock); + } + } + } +} + +TEST(ProjectLock, NonGitCommitDoesNotChangeSourceMatching) +{ + const std::vector> values{ std::nullopt, "main", "release" }; + for (const auto &kind : { "file", "archive", "dsc" }) { + for (const auto &previous : values) { + api::types::v1::BuilderProjectSource source; + source.kind = kind; + source.url = "https://example.com/source"; + source.commit = previous; + api::types::v1::BuilderProjectLock lock{}; + lock.sources = { { { "sha256:" + std::string(64, '0') }, source } }; + const auto originalLock = nlohmann::json(lock); + + for (const auto ¤t : values) { + source.commit = current; + SCOPED_TRACE(originalLock.dump()); + SCOPED_TRACE(nlohmann::json(source).dump()); + auto plan = resolveProjectLockSources(std::vector{ source }, lock); + ASSERT_TRUE(plan) << plan.error().message(); + EXPECT_FALSE(plan->changed); + ASSERT_EQ(plan->requests.size(), 1); + EXPECT_TRUE(std::holds_alternative(plan->requests.front())); + + auto verified = verifyProjectLockSources(std::vector{ source }, lock); + ASSERT_TRUE(verified) << verified.error().message(); + EXPECT_FALSE(verified->changed); + EXPECT_EQ(nlohmann::json(lock), originalLock); + } + } + } +} + +TEST(ProjectLock, GitCommitRemainsPartOfSourceMatching) +{ + api::types::v1::BuilderProjectSource source; + source.kind = "git"; + source.url = "https://example.com/source.git"; + source.commit = "main"; + api::types::v1::BuilderProjectLock lock{}; + lock.sources = { { { "git-commit:" + std::string(40, 'a') }, source } }; + source.commit = "release"; + + auto plan = resolveProjectLockSources(std::vector{ source }, lock); + ASSERT_TRUE(plan) << plan.error().message(); + EXPECT_TRUE(plan->changed); + ASSERT_EQ(plan->requests.size(), 1); + EXPECT_TRUE(std::holds_alternative(plan->requests.front())); + EXPECT_FALSE(verifyProjectLockSources(std::vector{ source }, lock)); +} + +TEST(ProjectLock, GitSubmodulesRemainPartOfSourceMatching) +{ + const std::vector> values{ std::nullopt, false, true }; + for (const auto &previous : values) { + api::types::v1::BuilderProjectSource source; + source.kind = "git"; + source.url = "https://example.com/source.git"; + source.commit = "main"; + source.submodules = previous; + api::types::v1::BuilderProjectLock lock{}; + lock.sources = { { { "git-commit:" + std::string(40, 'a') }, source } }; + + for (const auto ¤t : values) { + source.submodules = current; + const bool matches = previous.value_or(true) == current.value_or(true); + auto plan = resolveProjectLockSources(std::vector{ source }, lock); + ASSERT_TRUE(plan) << plan.error().message(); + EXPECT_EQ(plan->changed, !matches); + ASSERT_EQ(plan->requests.size(), 1); + EXPECT_EQ(std::holds_alternative(plan->requests.front()), matches); + + auto verified = verifyProjectLockSources(std::vector{ source }, lock); + EXPECT_EQ(verified.has_value(), matches); + } + } +} + +TEST_F(ProjectLockSourceTest, NonGitCommitAndSubmodulesChangesReuseCachedContentOffline) +{ + auto source = fileSource("source", "locked content"); + source.commit = "main"; + source.submodules = true; + auto lock = generateLock({ source }); + ASSERT_TRUE(lock) << lock.error().message(); + ASSERT_EQ(lock->sources.size(), 1); + EXPECT_FALSE(lock->sources.front().source.commit.has_value()); + EXPECT_FALSE(lock->sources.front().source.submodules.has_value()); + ASSERT_TRUE(std::filesystem::remove(*source.url)); + source.commit = "release"; + + for (const auto &value : std::vector>{ std::nullopt, false, true }) { + source.submodules = value; + auto fetched = resolveAndFetchProjectSources({ source }, + cache, + temporary.path() / "normal", + true, + *lock); + ASSERT_TRUE(fetched) << fetched.error().message(); + EXPECT_FALSE(fetched->changed); + auto verified = + verifyAndFetchProjectSources({ source }, cache, temporary.path() / "locked", true, *lock); + ASSERT_TRUE(verified) << verified.error().message(); + } + + EXPECT_EQ(downloadRequests(), (std::vector{ *source.url })); +} + +TEST_F(ProjectLockSourceTest, NameAndVersionChangesReuseLockedContent) +{ + auto source = fileSource("original", "locked content"); + source.version = "1"; + auto lock = generateLock({ source }); + ASSERT_TRUE(lock) << lock.error().message(); + ASSERT_TRUE(std::filesystem::remove(*source.url)); + const auto previousLock = nlohmann::json(*lock).dump(); + + auto renamed = source; + renamed.name = "renamed"; + auto nameRemoved = source; + nameRemoved.name.reset(); + auto reversioned = source; + reversioned.version = "2"; + auto bothChanged = renamed; + bothChanged.version.reset(); + std::vector changes{ renamed, + nameRemoved, + reversioned, + bothChanged }; + for (std::size_t index = 0; index < changes.size(); ++index) { + const auto &changed = changes[index]; + SCOPED_TRACE(index); + const auto destination = temporary.path() / ("normal-" + std::to_string(index)); + auto resolved = resolveAndFetchProjectSources({ changed }, cache, destination, true, *lock); + ASSERT_TRUE(resolved) << resolved.error().message(); + EXPECT_FALSE(resolved->changed); + auto outputHash = fileHash(destination / sourceOutputName(changed)); + ASSERT_TRUE(outputHash) << outputHash.error().message(); + EXPECT_EQ(*outputHash, *source.digest); + + const auto lockedDestination = temporary.path() / ("locked-" + std::to_string(index)); + auto verified = + verifyAndFetchProjectSources({ changed }, cache, lockedDestination, true, *lock); + ASSERT_TRUE(verified) << verified.error().message(); + auto lockedOutputHash = fileHash(lockedDestination / sourceOutputName(changed)); + ASSERT_TRUE(lockedOutputHash) << lockedOutputHash.error().message(); + EXPECT_EQ(*lockedOutputHash, *source.digest); + } + EXPECT_EQ(nlohmann::json(*lock).dump(), previousLock); + EXPECT_EQ(downloadRequests(), (std::vector{ *source.url })); +} + +TEST(ProjectLock, DependencyPlansFollowDeclarationsAndMode) +{ + const LockedDependency base{ "org.test.Base", + { { "binary" }, "main:org.test.Base/1.0.0" }, + "base" }; + const auto lock = completeProjectLock({}, base); + auto unchanged = planProjectDependencies(base.requested, std::nullopt, lock, false); + ASSERT_TRUE(unchanged); + EXPECT_FALSE(unchanged->changed); + ASSERT_TRUE(unchanged->base); + EXPECT_TRUE(std::holds_alternative(*unchanged->base)); + EXPECT_FALSE(unchanged->runtime); + auto verified = planProjectDependencies(base.requested, std::nullopt, lock, true); + ASSERT_TRUE(verified); + EXPECT_FALSE(verified->changed); + + auto changed = + planProjectDependencies(std::string{ "org.test.Base/2.0" }, std::nullopt, lock, false); + ASSERT_TRUE(changed); + EXPECT_TRUE(changed->changed); + ASSERT_TRUE(changed->base); + ASSERT_TRUE(std::holds_alternative(*changed->base)); + EXPECT_EQ(std::get(*changed->base).requested, "org.test.Base/2.0"); + + auto rejected = + planProjectDependencies(std::string{ "org.test.Base/2.0" }, std::nullopt, lock, true); + EXPECT_FALSE(rejected); + + auto forced = planProjectDependencies(base.requested, std::nullopt, std::nullopt, false); + ASSERT_TRUE(forced); + EXPECT_TRUE(forced->changed); + ASSERT_TRUE(forced->base); + EXPECT_TRUE(std::holds_alternative(*forced->base)); + + const LockedDependency runtime{ "org.test.Runtime", + { { "binary" }, "main:org.test.Runtime/1.0.0" }, + "runtime" }; + const auto runtimeLock = completeProjectLock({}, base, runtime); + auto removed = planProjectDependencies(base.requested, std::nullopt, runtimeLock, false); + ASSERT_TRUE(removed); + EXPECT_TRUE(removed->changed); + EXPECT_FALSE(removed->runtime); + EXPECT_FALSE(planProjectDependencies(base.requested, std::nullopt, runtimeLock, true)); + auto added = planProjectDependencies(base.requested, runtime.requested, lock, false); + ASSERT_TRUE(added); + EXPECT_TRUE(added->changed); + ASSERT_TRUE(added->runtime); + EXPECT_TRUE(std::holds_alternative(*added->runtime)); + EXPECT_FALSE(planProjectDependencies(base.requested, runtime.requested, lock, true)); +} + +TEST(ProjectLock, DependencyPlansDeferImplicitBaseUntilRuntimeMetadataIsAvailable) +{ + const LockedDependency base{ "org.test.Base/1.0", + { { "binary" }, "main:org.test.Base/1.0.0" }, + "base" }; + const LockedDependency runtime{ "org.test.Runtime/1.0", + { { "binary" }, "main:org.test.Runtime/1.0.0" }, + "runtime" }; + const auto lock = completeProjectLock({}, base, runtime); + for (const bool locked : { false, true }) { + auto plan = planProjectDependencies(std::nullopt, runtime.requested, lock, locked); + ASSERT_TRUE(plan); + EXPECT_FALSE(plan->base); + ASSERT_TRUE(plan->runtime); + EXPECT_TRUE(std::holds_alternative(*plan->runtime)); + EXPECT_FALSE(plan->changed); + } + + auto fresh = planProjectDependencies(std::nullopt, runtime.requested, std::nullopt, false); + ASSERT_TRUE(fresh); + EXPECT_TRUE(fresh->changed); + EXPECT_FALSE(fresh->base); + ASSERT_TRUE(fresh->runtime); + EXPECT_TRUE(std::holds_alternative(*fresh->runtime)); + + const std::string newRuntime = "org.test.Runtime/2.0"; + auto newPlan = planProjectDependencies(std::nullopt, newRuntime, lock, false); + ASSERT_TRUE(newPlan); + EXPECT_TRUE(newPlan->changed); + EXPECT_FALSE(newPlan->base); + ASSERT_TRUE(newPlan->runtime); + EXPECT_TRUE(std::holds_alternative(*newPlan->runtime)); + EXPECT_FALSE(planProjectDependencies(std::nullopt, newRuntime, lock, true)); + + auto added = planProjectDependencies(std::nullopt, + runtime.requested, + completeProjectLock({}, base), + false); + ASSERT_TRUE(added); + EXPECT_TRUE(added->changed); + EXPECT_FALSE(added->base); + + auto complete = planProjectDependencies(base.requested, runtime.requested, lock, true); + ASSERT_TRUE(complete); + EXPECT_FALSE(complete->changed); + ASSERT_TRUE(complete->base); + EXPECT_TRUE(std::holds_alternative(*complete->base)); + + auto changed = + planProjectDependencies(std::string{ "org.test.Base/2.0" }, runtime.requested, lock, false); + ASSERT_TRUE(changed); + EXPECT_TRUE(changed->changed); + ASSERT_TRUE(changed->base); + EXPECT_TRUE(std::holds_alternative(*changed->base)); + EXPECT_FALSE( + planProjectDependencies(std::string{ "org.test.Base/2.0" }, runtime.requested, lock, true)); + EXPECT_FALSE(planProjectDependencies(std::nullopt, std::nullopt, std::nullopt, false)); + EXPECT_FALSE(planProjectDependencies(std::nullopt, std::nullopt, lock, true)); +} + +TEST(ProjectLock, RejectsIncompleteDependencyLock) +{ + TempDir temporary; + const auto manifest = temporary.path() / "linglong.yaml"; + std::ofstream(manifest) << "version: 1\n"; + std::ofstream(projectLockPath(manifest)) << "version: 1\n" + "sources: []\n" + "dependencies:\n" + " - type: base\n" + " requested: org.test.Base\n" + " resolved:\n" + " reference: main:org.test.Base/1.0.0\n"; + + auto locked = prepareSourcesFromManifest({}, + manifest, + temporary.path() / "cache", + SourcePreparationMode::Locked); + ASSERT_FALSE(locked.has_value()); +} + +TEST(ProjectLock, RecordsOnlyAvailableModules) +{ + TempDir temporary; + const auto manifest = temporary.path() / "linglong.yaml"; + std::ofstream(manifest) << "version: 1\n"; + auto generated = prepareSourcesFromManifest({}, + manifest, + temporary.path() / "cache", + SourcePreparationMode::Generate); + ASSERT_TRUE(generated); + const LockedDependency base{ "org.test.Base", + { { "binary" }, "main:org.test.Base/1.0.0" }, + "base" }; + ASSERT_TRUE(writeCompleteProjectLock(manifest, completeProjectLock(generated->entries, base))); + auto loaded = prepareSourcesFromManifest({}, + manifest, + temporary.path() / "cache", + SourcePreparationMode::Locked); + ASSERT_TRUE(loaded); + auto lock = readProjectLock(projectLockPath(manifest)); + ASSERT_TRUE(lock); + auto lockedBase = lockedDependency(*lock, "base"); + ASSERT_TRUE(lockedBase); + EXPECT_EQ(lockedBase->resolved.modules, (std::vector{ "binary" })); +} + +TEST(ProjectLock, DependencyStateTracksOnlyResolvedLayers) +{ + const LockedDependency base{ "org.test.Base/1.0", + { { "binary", "develop" }, "main:org.test.Base/1.0.0" }, + "base" }; + const LockedDependency runtime{ "org.test.Runtime/2.0", + { { "binary" }, "main:org.test.Runtime/2.0.0" }, + "runtime" }; + const auto original = completeProjectLock({}, base, runtime); + auto previous = projectLockDependencyState(original); + ASSERT_TRUE(previous) << previous.error().message(); + + auto declarationsChanged = original; + declarationsChanged.dependencies[0].requested = + "main:org.test.Base/1.0.0/" + package::Architecture::currentCPUArchitecture().toString(); + declarationsChanged.dependencies[1].requested = + "main:org.test.Runtime/2.0.0/" + package::Architecture::currentCPUArchitecture().toString(); + api::types::v1::BuilderProjectSource source; + source.kind = "file"; + source.url = "https://example.com/changed-source"; + declarationsChanged.sources = { { { "sha256:" + std::string(64, 'a') }, source } }; + std::reverse(declarationsChanged.dependencies.begin(), declarationsChanged.dependencies.end()); + std::reverse(declarationsChanged.dependencies[1].resolved.modules.begin(), + declarationsChanged.dependencies[1].resolved.modules.end()); + auto unchanged = projectLockDependencyState(declarationsChanged); + ASSERT_TRUE(unchanged) << unchanged.error().message(); + EXPECT_EQ(*unchanged, *previous); + + for (const auto &change : { "base", "runtime", "modules", "remove-runtime" }) { + SCOPED_TRACE(change); + auto changed = original; + if (std::string(change) == "base") { + changed.dependencies[0].resolved.reference = "main:org.test.Base/1.0.1"; + } else if (std::string(change) == "runtime") { + changed.dependencies[1].resolved.reference = "main:org.test.Runtime/2.0.1"; + } else if (std::string(change) == "modules") { + changed.dependencies[0].resolved.modules = { "binary" }; + } else { + changed.dependencies.pop_back(); + } + auto state = projectLockDependencyState(changed); + ASSERT_TRUE(state) << state.error().message(); + EXPECT_NE(*state, *previous); + } +} + +TEST(ProjectLock, DependencyStateIncludesActualArchitecture) +{ + const LockedDependency base{ "org.test.Base/1.0", + { { "binary" }, "main:org.test.Base/1.0.0/unknown" }, + "base" }; + const LockedDependency runtime{ "org.test.Runtime/2.0", + { { "binary" }, "main:org.test.Runtime/2.0.0/unknown" }, + "runtime" }; + const auto sharedLock = completeProjectLock({}, base, runtime); + const auto sharedState = projectLockDependencyState(sharedLock); + ASSERT_TRUE(sharedState) << sharedState.error().message(); + const auto host = package::Architecture::currentCPUArchitecture().toString(); + const auto parsed = nlohmann::json::parse(*sharedState); + EXPECT_EQ(parsed["base"]["reference"], "main:org.test.Base/1.0.0/" + host); + EXPECT_EQ(parsed["runtime"]["reference"], "main:org.test.Runtime/2.0.0/" + host); + + std::vector states; + for (const std::string architecture : { "x86_64", "arm64" }) { + auto lock = sharedLock; + lock.dependencies[0].resolved.reference = "main:org.test.Base/1.0.0/" + architecture; + lock.dependencies[1].resolved.reference = "main:org.test.Runtime/2.0.0/" + architecture; + auto state = projectLockDependencyState(lock); + ASSERT_TRUE(state) << state.error().message(); + EXPECT_EQ(*state == *sharedState, architecture == host); + states.push_back(*state); + } + EXPECT_NE(states[0], states[1]); +} + +TEST(ProjectLock, FailedUpdatePreservesPreviousLock) +{ + TempDir temporary; + const auto manifest = temporary.path() / "linglong.yaml"; + std::ofstream(manifest) << "version: 1\n"; + auto generated = prepareSourcesFromManifest({}, + manifest, + temporary.path() / "cache", + SourcePreparationMode::Generate); + ASSERT_TRUE(generated); + const LockedDependency base{ "org.test.Base", + { { "binary", "develop" }, "main:org.test.Base/1.0.0" }, + "base" }; + ASSERT_TRUE(writeCompleteProjectLock(manifest, completeProjectLock(generated->entries, base))); + const auto before = readProjectLock(projectLockPath(manifest)); + ASSERT_TRUE(before) << before.error().message(); + const auto previous = projectLockDependencyState(*before); + ASSERT_TRUE(previous); + + api::types::v1::BuilderProjectSource invalid; + invalid.kind = "file"; + invalid.url = "https://example.com/source"; + invalid.name = "invalid/name"; + auto updated = prepareSourcesFromManifest({ invalid }, + manifest, + temporary.path() / "cache", + SourcePreparationMode::Update); + EXPECT_FALSE(updated.has_value()); + const auto after = readProjectLock(projectLockPath(manifest)); + ASSERT_TRUE(after) << after.error().message(); + const auto current = projectLockDependencyState(*after); + ASSERT_TRUE(current); + EXPECT_EQ(*previous, *current); +} + +TEST(ProjectLock, RejectsInvalidSources) +{ + TempDir temporary; + const auto manifest = temporary.path() / "linglong.yaml"; + std::ofstream(manifest) << "version: 1\n"; + + std::vector> cases; + + api::types::v1::BuilderProjectSource unsupported; + unsupported.kind = "unknown"; + unsupported.url = "https://example.com/source"; + cases.emplace_back(std::move(unsupported), "unsupported source kind"); + + api::types::v1::BuilderProjectSource missingUrl; + missingUrl.kind = "file"; + missingUrl.name = "source"; + cases.emplace_back(std::move(missingUrl), "source URL is missing"); + + api::types::v1::BuilderProjectSource missingCommit; + missingCommit.kind = "git"; + missingCommit.url = "https://example.com/source.git"; + cases.emplace_back(std::move(missingCommit), "git source commit is missing"); + + api::types::v1::BuilderProjectSource gitDigest; + gitDigest.kind = "git"; + gitDigest.url = "https://example.com/source.git"; + gitDigest.commit = "main"; + gitDigest.digest = std::string(64, '0'); + cases.emplace_back(std::move(gitDigest), "git source does not support digest"); + + api::types::v1::BuilderProjectSource invalidDigest; + invalidDigest.kind = "file"; + invalidDigest.url = "https://example.com/source"; + invalidDigest.digest = "invalid"; + cases.emplace_back(std::move(invalidDigest), "source digest must be a SHA-256 value"); + + api::types::v1::BuilderProjectSource invalidName; + invalidName.kind = "file"; + invalidName.url = "https://example.com/source"; + invalidName.name = "directory/source"; + cases.emplace_back(std::move(invalidName), "source name must be a single path component"); + + api::types::v1::BuilderProjectSource emptyUrlName; + emptyUrlName.kind = "file"; + emptyUrlName.url = "https://example.com/directory/"; + cases.emplace_back(std::move(emptyUrlName), "source name must be a single path component"); + + api::types::v1::BuilderProjectSource emptyGitName; + emptyGitName.kind = "git"; + emptyGitName.url = "https://example.com/.git"; + emptyGitName.commit = "main"; + cases.emplace_back(std::move(emptyGitName), "source name must be a single path component"); + + for (const auto &[source, expectedError] : cases) { + SCOPED_TRACE(expectedError); + auto result = prepareSourcesFromManifest({ source }, + manifest, + temporary.path() / "cache", + SourcePreparationMode::Generate); + ASSERT_FALSE(result.has_value()); + EXPECT_NE(result.error().message().find(expectedError), std::string::npos) + << result.error().message(); + } +} + +TEST(ProjectLock, DuplicateFetchRequestsUseNormalizedSourceFields) +{ + for (const auto &kind : { "git", "file", "archive", "dsc" }) { + SCOPED_TRACE(kind); + api::types::v1::BuilderProjectSource first; + first.kind = kind; + first.url = "https://example.com/source"; + first.name = "first"; + first.version = "old description"; + first.commit = "main"; + if (first.kind != "git") { + first.digest = std::string(64, 'A'); + first.submodules = false; + } + + auto second = first; + second.name = "second"; + second.version = "new description"; + second.submodules = true; + if (second.kind != "git") { + second.digest = "sha256:" + std::string(64, 'a'); + second.commit = "ignored for non-Git sources"; + } + + api::types::v1::BuilderProjectLock lock{}; + const auto hash = first.kind == "git" ? "git-commit:" + std::string(40, 'a') + : "sha256:" + std::string(64, 'a'); + lock.sources = { { { hash }, first }, { { hash }, second } }; + + for (const auto &sources : { std::vector{ first, second }, std::vector{ second, first } }) { + auto generated = resolveProjectLockSources(sources, std::nullopt); + ASSERT_FALSE(generated); + EXPECT_NE(generated.error().message().find("duplicate source fetch request"), + std::string::npos) + << generated.error().message(); + + auto reused = resolveProjectLockSources(sources, lock); + ASSERT_FALSE(reused); + EXPECT_NE(reused.error().message().find("duplicate source fetch request"), + std::string::npos) + << reused.error().message(); + + auto verified = verifyProjectLockSources(sources, lock); + ASSERT_FALSE(verified); + EXPECT_NE(verified.error().message().find("duplicate source fetch request"), + std::string::npos) + << verified.error().message(); + } + } +} + +TEST(ProjectLock, DistinctFetchRequestsRemainValid) +{ + for (const auto &kind : { "git", "file", "archive", "dsc" }) { + SCOPED_TRACE(kind); + api::types::v1::BuilderProjectSource first; + first.kind = kind; + first.url = "https://example.com/source"; + first.name = "first"; + if (first.kind == "git") { + first.commit = "main"; + first.submodules = true; + } else { + first.digest = "sha256:" + std::string(64, 'a'); + } + + std::vector changes; + auto changed = first; + changed.url = "https://example.com/other-source"; + changes.push_back(changed); + changed = first; + changed.kind = first.kind == "file" ? "archive" : "file"; + changes.push_back(changed); + changed = first; + if (first.kind == "git") { + changed.commit = "other"; + changes.push_back(changed); + changed = first; + changed.submodules = false; + } else { + changed.digest = "sha256:" + std::string(64, 'b'); + } + changes.push_back(changed); + + for (auto second : changes) { + second.name = "second"; + SCOPED_TRACE(nlohmann::json(second).dump()); + auto plan = resolveProjectLockSources(std::vector{ first, second }, std::nullopt); + ASSERT_TRUE(plan) << plan.error().message(); + EXPECT_EQ(plan->requests.size(), 2); + + const auto hash = [](const auto &source) { + return source.kind == "git" ? "git-commit:" + std::string(40, 'a') + : "sha256:" + std::string(64, 'a'); + }; + api::types::v1::BuilderProjectLock lock{}; + lock.sources = { { { hash(first) }, first }, { { hash(second) }, second } }; + auto verified = verifyProjectLockSources(std::vector{ second, first }, lock); + ASSERT_TRUE(verified) << verified.error().message(); + EXPECT_EQ(verified->requests.size(), 2); + } + } +} + +TEST(ProjectLock, ReportsDifferentSourceFieldsConsistently) +{ + TempDir temporary; + const auto manifest = temporary.path() / "linglong.yaml"; + std::ofstream(manifest) << "version: 1\n"; + std::ofstream(projectLockPath(manifest)) + << "version: 1\n" + "dependencies:\n" + " - type: base\n" + " requested: org.test.Base\n" + " resolved:\n" + " reference: main:org.test.Base/1.0.0\n" + " modules: [binary, develop]\n" + "sources:\n" + " - source:\n" + " kind: file\n" + " url: https://example.com/old-source\n" + " resolved:\n" + " hash: sha256:0000000000000000000000000000000000000000000000000000000000000000\n"; + + api::types::v1::BuilderProjectSource source; + source.kind = "file"; + source.url = "https://example.com/new-source"; + auto result = prepareSourcesFromManifest({ source }, + manifest, + temporary.path() / "cache", + SourcePreparationMode::Locked); + + ASSERT_FALSE(result.has_value()); + EXPECT_NE(result.error().message().find( + "sources differ from linglong.lock; run ll-builder update-lockfile"), + std::string::npos) + << result.error().message(); + std::error_code ec; + EXPECT_FALSE(std::filesystem::exists(temporary.path() / "cache", ec)); + EXPECT_FALSE(ec); +} + +TEST_F(ProjectLockSourceTest, UppercaseManifestDigestMatchesDownloadedContents) +{ + auto source = fileSource("source", "contents"); + std::transform(source.digest->begin() + 7, + source.digest->end(), + source.digest->begin() + 7, + [](unsigned char character) { + return static_cast(std::toupper(character)); + }); + auto lock = generateLock({ source }); + ASSERT_TRUE(lock) << lock.error().message(); + EXPECT_TRUE( + verifyAndFetchProjectSources({ source }, cache, temporary.path() / "locked", true, *lock)); +} + +TEST_F(ProjectLockSourceTest, DscCacheSupportsDifferentUrlBasenamesAndRelativeRoots) +{ + const auto commands = temporary.path() / "bin"; + std::ofstream(commands / "dget") << R"shell(#!/bin/sh +set -eu +/bin/cp -- "$3" "$(basename "$3")" +)shell"; + std::ofstream(commands / "dscverify") << "#!/bin/sh\ntest -f \"$2\"\n"; + std::ofstream(commands / "dpkg-source") << "#!/bin/sh\nmkdir \"$4\"\n"; + for (const auto &command : { "dget", "dscverify", "dpkg-source" }) { + std::filesystem::permissions(commands / command, std::filesystem::perms::owner_all); + } + + auto first = fileSource("first.dsc", "identical dsc contents"); + first.kind = "dsc"; + first.name = "first"; + auto second = first; + second.name = "second"; + second.url = (temporary.path() / "second.dsc").string(); + std::filesystem::copy_file(*first.url, *second.url); + + for (const bool relative : { false, true }) { + SCOPED_TRACE(relative); + const auto root = temporary.path() / (relative ? "relative-cache" : "absolute-cache"); + const auto cachePath = relative ? std::filesystem::relative(root) : root; + SourceCacheLayout layout(cachePath); + ASSERT_TRUE(layout.ensure()); + auto destination = utils::TemporaryDirectory::create("fetch-", layout.temporary()); + ASSERT_TRUE(destination) << destination.error().message(); + + ProjectSourcePlan generation; + generation.requests = { GenerateSourceRequest{ first } }; + auto fetched = fetchProjectSources(generation, cachePath, destination->path(), false); + ASSERT_TRUE(fetched) << fetched.error().message(); + const auto cached = layout.object("dsc", fetched->front().resolved.hash); + ASSERT_TRUE(std::filesystem::exists(cached / "source.dsc")); + EXPECT_TRUE(std::filesystem::is_directory(destination->path() / *first.name)); + + ProjectSourcePlan locked; + locked.requests = { LockedSourceRequest{ first, fetched->front().resolved } }; + ASSERT_TRUE(fetchProjectSources(locked, cachePath, destination->path(), true)); + + generation.requests = { GenerateSourceRequest{ second } }; + ASSERT_TRUE(fetchProjectSources(generation, cachePath, destination->path(), false)); + EXPECT_TRUE(std::filesystem::is_directory(destination->path() / *second.name)); + locked.requests.emplace_back(LockedSourceRequest{ second, fetched->front().resolved }); + EXPECT_TRUE(fetchProjectSources(locked, cachePath, destination->path(), true)); + } +} + +} // namespace linglong::builder diff --git a/libs/linglong/tests/ll-tests/src/linglong/builder/pull_dependency_test.cpp b/libs/linglong/tests/ll-tests/src/linglong/builder/pull_dependency_test.cpp index 3041e0081..de18c05af 100644 --- a/libs/linglong/tests/ll-tests/src/linglong/builder/pull_dependency_test.cpp +++ b/libs/linglong/tests/ll-tests/src/linglong/builder/pull_dependency_test.cpp @@ -9,17 +9,24 @@ #define private public #include "linglong/builder/linglong_builder.h" #undef private +#include "linglong/api/types/v1/Generators.hpp" +#include "linglong/builder/source_cache.h" +#include "linglong/builder/source_fetcher.h" #include "linglong/package/fuzzy_reference.h" #include "linglong/package/reference.h" #include "linglong/package_manager/package_task.h" #include "linglong/repo/ostree_repo.h" +#include "linglong/utils/env.h" #include "linglong/utils/error/error.h" +#include "linglong/utils/file.h" #include "ocppi/cli/crun/Crun.hpp" #include +#include #include #include #include +#include using namespace linglong; using ::testing::_; @@ -28,6 +35,8 @@ using ::testing::Return; namespace { +const auto currentArchitecture = package::Architecture::currentCPUArchitecture().toString(); + class MockRepo : public repo::OSTreeRepo { public: @@ -54,6 +63,12 @@ class MockRepo : public repo::OSTreeRepo (const package::FuzzyReference &fuzzyRef), (override, const, noexcept)); + MOCK_METHOD(utils::error::Result, + matchRemoteByPriority, + (const package::FuzzyReference &fuzzy, + const std::optional &remoteRepo), + (override, const, noexcept)); + MOCK_METHOD(utils::error::Result, pull, (service::Task & taskContext, @@ -63,6 +78,11 @@ class MockRepo : public repo::OSTreeRepo MOCK_METHOD(utils::error::Result, mergeModules, (), (override, const, noexcept)); + MOCK_METHOD(std::vector, + getModuleList, + (const package::Reference &ref), + (override, const, noexcept)); + private: static api::types::v1::RepoConfigV2 makeConfig() { @@ -78,6 +98,18 @@ class MockRepo : public repo::OSTreeRepo } }; +std::string lockedReference(const package::Reference &reference, bool explicitArchitecture = true) +{ + return reference.channel + ":" + reference.id + "/" + reference.version.toString() + "/" + + (explicitArchitecture ? reference.arch.toString() : "unknown"); +} + +std::string lockedReference(const utils::error::Result &reference, + bool explicitArchitecture = true) +{ + return lockedReference(*reference, explicitArchitecture); +} + auto layerCached() { return package::LayerDir("/fake/layer/dir"); @@ -96,10 +128,27 @@ auto refWithRepo(package::Reference ref, std::string repoName = "stable") }; } +auto remotePackages(const std::vector &references) + -> utils::error::Result +{ + std::vector infos; + for (const auto &reference : references) { + api::types::v1::PackageInfoV2 info; + info.arch = { reference.arch.toString() }; + info.channel = reference.channel; + info.id = reference.id; + info.version = reference.version.toString(); + infos.push_back(std::move(info)); + } + repo::RemotePackages packages; + packages.addPackages(refWithRepo(references.front(), "custom").repo, std::move(infos)); + return utils::error::Result{ std::move(packages) }; +} + auto builderProject(std::optional runtime = std::nullopt) { api::types::v1::BuilderProject project; - project.base = "org.deepin.base/23.0.0.0/x86_64"; + project.base = "org.deepin.base/23.0.0.0/" + currentArchitecture; project.build = ""; project.command = std::vector{ "true" }; project.package = api::types::v1::BuilderProjectPackage{ @@ -127,6 +176,13 @@ runtime::ContainerBuilder &containerBuilder() return builder; } +class SourceTestBuilder : public builder::Builder +{ +public: + using Builder::Builder; + using Builder::buildStageFetchSource; +}; + class PullDependencyTest : public ::testing::Test { protected: @@ -134,23 +190,1724 @@ class PullDependencyTest : public ::testing::Test void TearDown() override { m_repo.reset(); } + void expectCachedModule(const package::Reference &reference, const std::string &module) + { + EXPECT_CALL(*m_repo, getLayerDir(reference, module)) + .WillRepeatedly([](const package::Reference &, + const std::string &) -> utils::error::Result { + return layerCached(); + }); + } + + void expectFreshBase(const package::Reference &reference) + { + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(reference)); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, false)).Times(0); + EXPECT_CALL(*m_repo, latestRemoteReference(_)).Times(0); + EXPECT_CALL(*m_repo, getLayerDir(reference, "binary")).WillOnce(Return(layerCached())); + expectCachedModule(reference, "develop"); + EXPECT_CALL(*m_repo, getLayerItem(reference, _)).Times(0); + EXPECT_CALL(*m_repo, getModuleList(reference)) + .WillOnce(Return(std::vector{ "binary" })); + EXPECT_CALL(*m_repo, mergeModules()).WillOnce(Return(utils::error::Result{})); + } + TempDir m_tmpDir; std::unique_ptr m_repo; }; +TEST_F(PullDependencyTest, MissingLockKeepsNulloptDuringPrepare) +{ + builder::Builder builder(api::types::v1::BuilderProject{}, + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (m_tmpDir.path() / "linglong.yaml").string(); + builder.projectLock = api::types::v1::BuilderProjectLock{}; + builder.pendingSources.emplace(); + + auto result = builder.buildStagePrepare(); + ASSERT_FALSE(result); + EXPECT_THAT(result.error().message(), testing::HasSubstr("invalid project info")); + EXPECT_FALSE(builder.projectLock); + EXPECT_FALSE(builder.pendingSources); +} + +TEST_F(PullDependencyTest, SkipFetchLoadsLockBeforeProjectValidation) +{ + builder::Builder builder(api::types::v1::BuilderProject{}, + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (m_tmpDir.path() / "linglong.yaml").string(); + const builder::LockedDependency base{ "org.deepin.base/23.0.0", + { { "binary" }, "main:org.deepin.base/23.0.0.0" }, + "base" }; + ASSERT_TRUE(builder::writeCompleteProjectLock(builder.projectYamlFile, + builder::completeProjectLock({}, base))); + builder.setBuildOptions(builder::BuilderBuildOptions{ .skipFetchSource = true }); + + auto prepared = builder.buildStagePrepare(); + ASSERT_FALSE(prepared); + EXPECT_THAT(prepared.error().message(), testing::HasSubstr("invalid project info")); + ASSERT_TRUE(builder.projectLock); + ASSERT_EQ(builder.projectLock->dependencies.size(), 1); + EXPECT_EQ(builder.projectLock->dependencies.front().resolved.reference, + base.resolved.reference); + EXPECT_FALSE(builder.pendingSources); +} + +TEST_F(PullDependencyTest, SkipFetchReusesDependenciesAndPreservesExistingSourceEntries) +{ + auto reference = + package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + ASSERT_TRUE(reference); + auto project = builderProject(); + project.base = "org.deepin.base/23.0.0"; + api::types::v1::BuilderProjectSource source; + source.kind = "file"; + source.url = "file://" + (m_tmpDir.path() / "unfetched-source").string(); + project.sources = std::vector{ source }; + SourceTestBuilder builder(project, + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (m_tmpDir.path() / "linglong.yaml").string(); + const builder::LockedDependency base{ *project.base, + { { "binary" }, lockedReference(reference) }, + "base" }; + auto oldSource = source; + oldSource.url = "file://" + (m_tmpDir.path() / "old-source").string(); + const auto hash = "sha256:" + std::string(64, 'a'); + builder.projectLock = builder::completeProjectLock({ { { hash }, oldSource } }, base); + ASSERT_TRUE(builder::writeCompleteProjectLock(builder.projectYamlFile, *builder.projectLock)); + auto before = utils::readFile(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(before); + builder.setBuildOptions( + builder::BuilderBuildOptions{ .skipFetchSource = true, .skipPullDepend = true }); + auto skipped = builder.buildStageFetchSource(); + ASSERT_TRUE(skipped); + EXPECT_FALSE(builder.pendingSources); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).Times(0); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, false)).Times(0); + EXPECT_CALL(*m_repo, latestRemoteReference(_)).Times(0); + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)).Times(0); + EXPECT_CALL(*m_repo, getLayerDir(*reference, "binary")).WillOnce(Return(layerCached())); + EXPECT_CALL(*m_repo, getModuleList(_)).Times(0); + EXPECT_CALL(*m_repo, mergeModules()).WillOnce(Return(utils::error::Result{})); + + auto fetched = builder.buildStagePullDependency(); + ASSERT_TRUE(fetched) << fetched.error().message(); + EXPECT_EQ(builder.project->base, project.base); + auto after = utils::readFile(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(after); + EXPECT_EQ(*after, *before); + ASSERT_EQ(builder.projectLock->sources.size(), 1); + EXPECT_EQ(builder.projectLock->sources.front().resolved.hash, hash); + EXPECT_EQ(builder.projectLock->sources.front().source.url, oldSource.url); +} + +TEST_F(PullDependencyTest, SkipFetchUpdatesDependenciesAndKeepsSourceEntriesAfterFailure) +{ + LINGLONG_TRACE("update dependencies after skipping source fetch"); + auto oldReference = + package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + auto newReference = + package::Reference::parse("main:org.deepin.base/24.0.0.0/" + currentArchitecture); + ASSERT_TRUE(oldReference); + ASSERT_TRUE(newReference); + auto project = builderProject(); + project.base = "org.deepin.base/24.0.0.0/" + currentArchitecture; + api::types::v1::BuilderProjectSource source; + source.kind = "file"; + source.url = "file://" + (m_tmpDir.path() / "unfetched-source").string(); + project.sources = std::vector{ source }; + SourceTestBuilder builder(project, + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (m_tmpDir.path() / "linglong.yaml").string(); + const builder::LockedDependency base{ "org.deepin.base/23.0.0.0/" + currentArchitecture, + { { "binary" }, lockedReference(oldReference) }, + "base" }; + const auto hash = "sha256:" + std::string(64, 'a'); + builder.projectLock = builder::completeProjectLock({ { { hash }, source } }, base); + ASSERT_TRUE(builder::writeCompleteProjectLock(builder.projectYamlFile, *builder.projectLock)); + auto before = utils::readFile(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(before); + builder.setBuildOptions( + builder::BuilderBuildOptions{ .skipFetchSource = true, .skipPullDepend = true }); + ASSERT_TRUE(builder.buildStageFetchSource()); + EXPECT_FALSE(builder.pendingSources); + + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(*newReference)); + expectCachedModule(*newReference, "binary"); + expectCachedModule(*newReference, "develop"); + EXPECT_CALL(*m_repo, mergeModules()).WillOnce(Return(LINGLONG_ERR("merge failed"))); + auto failed = builder.buildStagePullDependency(); + ASSERT_FALSE(failed); + EXPECT_EQ(builder.project->base, project.base); + EXPECT_EQ(builder.projectLock->dependencies.front().resolved.reference, + lockedReference(oldReference)); + auto unchanged = utils::readFile(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(unchanged); + EXPECT_EQ(*unchanged, *before); + testing::Mock::VerifyAndClearExpectations(m_repo.get()); + + expectFreshBase(*newReference); + auto updated = builder.buildStagePullDependency(); + ASSERT_TRUE(updated) << updated.error().message(); + EXPECT_FALSE(builder.pendingSources); + auto written = builder::readProjectLock(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(written); + ASSERT_EQ(written->dependencies.size(), 1); + EXPECT_EQ(written->dependencies.front().requested, *project.base); + EXPECT_EQ(written->dependencies.front().resolved.reference, lockedReference(newReference)); + ASSERT_EQ(written->sources.size(), 1); + EXPECT_EQ(written->sources.front().resolved.hash, hash); + EXPECT_EQ(written->sources.front().source.url, source.url); +} + +TEST_F(PullDependencyTest, SkipFetchWithoutLockRecordsDependenciesAndLeavesSourcesForNormalBuild) +{ + auto reference = + package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + ASSERT_TRUE(reference); + auto project = builderProject(); + api::types::v1::BuilderProjectSource source; + source.kind = "file"; + source.url = "file://" + (m_tmpDir.path() / "unfetched-source").string(); + project.sources = std::vector{ source }; + SourceTestBuilder builder(project, + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (m_tmpDir.path() / "linglong.yaml").string(); + const auto sourceDirectory = m_tmpDir.path() / "linglong/sources"; + std::filesystem::create_directories(sourceDirectory); + const auto manualSource = sourceDirectory / "manual-source"; + std::ofstream(manualSource) << "local changes"; + builder.setBuildOptions( + builder::BuilderBuildOptions{ .skipFetchSource = true, .skipPullDepend = true }); + ASSERT_TRUE(builder.buildStageFetchSource()); + EXPECT_FALSE(builder.projectLock); + EXPECT_FALSE(builder.pendingSources); + expectFreshBase(*reference); + + auto fetched = builder.buildStagePullDependency(); + ASSERT_TRUE(fetched) << fetched.error().message(); + auto written = builder::readProjectLock(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(written); + EXPECT_TRUE(written->sources.empty()); + ASSERT_EQ(written->dependencies.size(), 1); + EXPECT_EQ(written->dependencies.front().resolved.reference, lockedReference(reference)); + EXPECT_TRUE(std::filesystem::exists(manualSource)); + auto next = builder::resolveProjectLockSources(project.sources, *written); + ASSERT_TRUE(next) << next.error().message(); + EXPECT_TRUE(next->changed); + ASSERT_EQ(next->requests.size(), 1); + EXPECT_TRUE(std::holds_alternative(next->requests.front())); +} + +TEST_F(PullDependencyTest, FreshSourcesWaitForDependenciesBeforeCreatingLock) +{ + auto baseRef = + package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + ASSERT_TRUE(baseRef) << baseRef.error().message(); + SourceTestBuilder builder(builderProject(), + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (m_tmpDir.path() / "linglong.yaml").string(); + builder.setBuildOptions(builder::BuilderBuildOptions{ .skipPullDepend = true }); + utils::EnvironmentVariableGuard cache("LINGLONG_FETCH_CACHE", + (m_tmpDir.path() / "cache").string()); + + auto fetched = builder.buildStageFetchSource(); + ASSERT_TRUE(fetched) << fetched.error().message(); + EXPECT_FALSE(builder.projectLock); + ASSERT_TRUE(builder.pendingSources); + EXPECT_TRUE(builder.pendingSources->empty()); + EXPECT_FALSE(std::filesystem::exists(builder::projectLockPath(builder.projectYamlFile))); + + expectFreshBase(*baseRef); + auto resolved = builder.buildStagePullDependency(); + ASSERT_TRUE(resolved) << resolved.error().message(); + ASSERT_TRUE(builder.projectLock); + EXPECT_FALSE(builder.pendingSources); + auto written = builder::readProjectLock(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(written) << written.error().message(); + EXPECT_TRUE(written->sources.empty()); + ASSERT_EQ(written->dependencies.size(), 1); + EXPECT_EQ(written->dependencies.front().resolved.reference, lockedReference(baseRef)); +} + +TEST_F(PullDependencyTest, LockedSourceMismatchFailsBeforeResetOrFetch) +{ + api::types::v1::BuilderProjectSource source; + source.kind = "file"; + source.name = "source"; + source.url = "https://example.com/new-source"; + source.digest = "sha256:" + std::string(64, 'a'); + auto project = builderProject(); + project.sources = std::vector{ source }; + SourceTestBuilder builder(project, + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.setBuildOptions(builder::BuilderBuildOptions{ .locked = true }); + auto oldSource = source; + oldSource.url = "https://example.com/old-source"; + builder.projectLock = api::types::v1::BuilderProjectLock{}; + builder.projectLock->sources = { { { *source.digest }, oldSource } }; + const auto cache = m_tmpDir.path() / "cache"; + utils::EnvironmentVariableGuard cacheEnv("LINGLONG_FETCH_CACHE", cache.string()); + std::filesystem::create_directories(builder.directories.sources()); + const auto manualSource = builder.directories.sources() / "manual-source"; + std::ofstream(manualSource) << "existing sources"; + + auto result = builder.buildStageFetchSource(); + ASSERT_FALSE(result); + EXPECT_THAT(result.error().message(), testing::HasSubstr("sources differ from linglong.lock")); + EXPECT_TRUE(std::filesystem::exists(manualSource)); + EXPECT_FALSE(std::filesystem::exists(cache)); + EXPECT_FALSE(builder.pendingSources); + ASSERT_TRUE(builder.projectLock); + EXPECT_EQ(builder.projectLock->sources.front().source.url, oldSource.url); +} + +TEST_F(PullDependencyTest, PrintsSourceProgressAndTerminatesRowsOnSuccessAndFailure) +{ + const auto artifact = m_tmpDir.path() / "artifact"; + std::ofstream(artifact) << "source content"; + auto hash = builder::fileHash(artifact); + ASSERT_TRUE(hash) << hash.error().message(); + api::types::v1::BuilderProjectSource source; + source.kind = "file"; + source.name = "custom-source"; + source.url = "https://example.com/" + std::string(100, 'x') + "/source"; + source.digest = *hash; + auto project = builderProject(); + project.sources = std::vector{ source }; + SourceTestBuilder builder(project, + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.setBuildOptions(builder::BuilderBuildOptions{ .locked = true, .offline = true }); + builder.projectLock = api::types::v1::BuilderProjectLock{}; + builder.projectLock->sources = { { { *hash }, source } }; + const auto cache = m_tmpDir.path() / "cache"; + utils::EnvironmentVariableGuard cacheEnv("LINGLONG_FETCH_CACHE", cache.string()); + builder::SourceCacheLayout cacheLayout(cache); + const auto cached = cacheLayout.object("file", *hash); + std::filesystem::create_directories(cached.parent_path()); + std::filesystem::copy_file(artifact, cached); + + for (const bool corrupt : { false, true }) { + SCOPED_TRACE(corrupt); + if (corrupt) { + std::ofstream(cached, std::ios::trunc) << "corrupt"; + } + testing::internal::CaptureStdout(); + auto result = builder.buildStageFetchSource(); + const auto output = testing::internal::GetCapturedStdout(); + EXPECT_EQ(result.has_value(), !corrupt); + EXPECT_THAT(output, testing::HasSubstr("[Processing Sources]")); + EXPECT_THAT(output, testing::HasSubstr(*source.name)); + EXPECT_THAT(output, testing::Not(testing::HasSubstr("Source 0"))); + EXPECT_THAT(output, testing::HasSubstr("file")); + EXPECT_THAT(output, + testing::HasSubstr("..." + source.url->substr(source.url->size() - 70))); + EXPECT_THAT(output, testing::HasSubstr("fetching ...")); + EXPECT_THAT(output, testing::Not(testing::HasSubstr("downloading"))); + EXPECT_THAT(output, testing::HasSubstr(corrupt ? "failed\n" : "complete\n")); + EXPECT_THAT(output, testing::Not(testing::HasSubstr(corrupt ? "complete\n" : "failed\n"))); + } +} + +TEST_F(PullDependencyTest, GenerateAndUpdateCreateLockAfterResolvingDependencies) +{ + auto baseRef = + package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + ASSERT_TRUE(baseRef) << baseRef.error().message(); + for (const auto mode : + { builder::ProjectLockMode::Generate, builder::ProjectLockMode::Update }) { + SCOPED_TRACE(static_cast(mode)); + TempDir workingDir; + builder::Builder builder(builderProject(), + workingDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (workingDir.path() / "linglong.yaml").string(); + builder.setBuildOptions(builder::BuilderBuildOptions{ .skipPullDepend = true }); + utils::EnvironmentVariableGuard cache("LINGLONG_FETCH_CACHE", + (workingDir.path() / "cache").string()); + if (mode == builder::ProjectLockMode::Update) { + std::ofstream(builder::projectLockPath(builder.projectYamlFile)) << "invalid old lock"; + } + + expectFreshBase(*baseRef); + auto result = builder.processProjectLock(mode); + ASSERT_TRUE(result) << result.error().message(); + ASSERT_TRUE(builder.projectLock); + EXPECT_FALSE(builder.pendingSources); + auto written = builder::readProjectLock(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(written) << written.error().message(); + EXPECT_TRUE(written->sources.empty()); + ASSERT_EQ(written->dependencies.size(), 1); + EXPECT_EQ(written->dependencies.front().resolved.reference, lockedReference(baseRef)); + testing::Mock::VerifyAndClearExpectations(m_repo.get()); + } +} + +TEST_F(PullDependencyTest, DependencyFailureKeepsFreshLockAbsent) +{ + LINGLONG_TRACE("fail dependency resolution before creating lock"); + SourceTestBuilder builder(builderProject(), + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (m_tmpDir.path() / "linglong.yaml").string(); + builder.setBuildOptions(builder::BuilderBuildOptions{ .skipPullDepend = true }); + utils::EnvironmentVariableGuard cache("LINGLONG_FETCH_CACHE", + (m_tmpDir.path() / "cache").string()); + auto fetched = builder.buildStageFetchSource(); + ASSERT_TRUE(fetched) << fetched.error().message(); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)) + .WillOnce(Return(LINGLONG_ERR("dependency is unavailable"))); + + auto resolved = builder.buildStagePullDependency(); + ASSERT_FALSE(resolved); + EXPECT_FALSE(builder.projectLock); + EXPECT_TRUE(builder.pendingSources); + EXPECT_FALSE(std::filesystem::exists(builder::projectLockPath(builder.projectYamlFile))); +} + +TEST_F(PullDependencyTest, PendingSourcesPreserveExistingLockUntilDependenciesComplete) +{ + LINGLONG_TRACE("fail dependency resolution with pending source changes"); + auto baseRef = + package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + ASSERT_TRUE(baseRef) << baseRef.error().message(); + builder::Builder builder(builderProject(), + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (m_tmpDir.path() / "linglong.yaml").string(); + builder.setBuildOptions(builder::BuilderBuildOptions{ .skipPullDepend = true }); + const builder::LockedDependency base{ *builder.project->base, + { { "binary" }, lockedReference(baseRef) }, + "base" }; + api::types::v1::BuilderProjectSource source; + source.kind = "file"; + source.url = "https://example.com/source"; + const auto oldHash = "sha256:" + std::string(64, 'a'); + const auto newHash = "sha256:" + std::string(64, 'b'); + builder.projectLock = builder::completeProjectLock({ { { oldHash }, source } }, base); + ASSERT_TRUE(builder::writeCompleteProjectLock(builder.projectYamlFile, *builder.projectLock)); + builder.pendingSources = + std::vector{ { { newHash }, source } }; + EXPECT_CALL(*m_repo, getLayerDir(*baseRef, "binary")).WillOnce(Return(layerCached())); + EXPECT_CALL(*m_repo, mergeModules()).WillOnce(Return(LINGLONG_ERR("merge failed"))); + + auto resolved = builder.buildStagePullDependency(); + ASSERT_FALSE(resolved); + ASSERT_TRUE(builder.projectLock); + EXPECT_EQ(builder.projectLock->sources.front().resolved.hash, oldHash); + ASSERT_TRUE(builder.pendingSources); + EXPECT_EQ(builder.pendingSources->front().resolved.hash, newHash); + auto written = builder::readProjectLock(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(written) << written.error().message(); + EXPECT_EQ(written->sources.front().resolved.hash, oldHash); + + EXPECT_CALL(*m_repo, getLayerDir(*baseRef, "binary")).WillOnce(Return(layerCached())); + EXPECT_CALL(*m_repo, mergeModules()).WillOnce(Return(utils::error::Result{})); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, false)).Times(0); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).Times(0); + EXPECT_CALL(*m_repo, getModuleList(_)).Times(0); + + auto retried = builder.buildStagePullDependency(); + ASSERT_TRUE(retried) << retried.error().message(); + ASSERT_TRUE(builder.projectLock); + EXPECT_EQ(builder.projectLock->sources.front().resolved.hash, newHash); + EXPECT_FALSE(builder.pendingSources); + auto updated = builder::readProjectLock(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(updated) << updated.error().message(); + EXPECT_EQ(updated->sources.front().resolved.hash, newHash); + EXPECT_EQ(updated->dependencies.front().resolved.reference, lockedReference(baseRef)); +} + +TEST_F(PullDependencyTest, LockedDependencyChangesFailBeforeFetching) +{ + const builder::LockedDependency base{ "org.deepin.base/23.0.0.0/" + currentArchitecture, + { { "binary" }, "main:org.deepin.base/23.0.0.0" }, + "base" }; + const builder::LockedDependency runtime{ + "org.deepin.runtime/23.0.0.0/" + currentArchitecture, + { { "binary" }, "main:org.deepin.runtime/23.0.0.0" }, + "runtime", + }; + for (const auto &change : { "base", "runtime", "add-runtime", "remove-runtime" }) { + SCOPED_TRACE(change); + auto project = builderProject(runtime.requested); + auto lock = builder::completeProjectLock({}, base, runtime); + if (std::string(change) == "base") { + project.base = "org.deepin.base/24.0.0.0/" + currentArchitecture; + } else if (std::string(change) == "runtime") { + project.runtime = "org.deepin.runtime/24.0.0.0/" + currentArchitecture; + } else if (std::string(change) == "add-runtime") { + lock = builder::completeProjectLock({}, base); + } else { + project.runtime.reset(); + } + builder::Builder builder(project, + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (m_tmpDir.path() / "linglong.yaml").string(); + builder.projectLock = lock; + builder.setBuildOptions(builder::BuilderBuildOptions{ .locked = true }); + ASSERT_TRUE(builder::writeCompleteProjectLock(builder.projectYamlFile, lock)); + auto before = utils::readFile(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(before); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, _)).Times(0); + EXPECT_CALL(*m_repo, getLayerItem(_, _)).Times(0); + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)).Times(0); + EXPECT_CALL(*m_repo, mergeModules()).Times(0); + + auto result = builder.buildStagePullDependency(); + ASSERT_FALSE(result); + EXPECT_THAT(result.error().message(), testing::HasSubstr("differs from linglong.lock")); + EXPECT_EQ(builder.project->base, project.base); + EXPECT_EQ(builder.project->runtime, project.runtime); + EXPECT_FALSE(builder.pendingSources); + auto after = utils::readFile(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(after); + EXPECT_EQ(*after, *before); + testing::Mock::VerifyAndClearExpectations(m_repo.get()); + } +} + +TEST_F(PullDependencyTest, CachedDependencyModulesPrintStatusInBothBuildModes) +{ + auto reference = + package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + ASSERT_TRUE(reference); + for (const bool locked : { false, true }) { + SCOPED_TRACE(locked); + builder::Builder builder(builderProject(), + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (m_tmpDir.path() / "linglong.yaml").string(); + const builder::LockedDependency entry{ *builder.project->base, + { { "binary", "develop" }, + lockedReference(reference) }, + "base" }; + builder.projectLock = builder::completeProjectLock({}, entry); + builder.setBuildOptions( + builder::BuilderBuildOptions{ .skipPullDepend = true, .locked = locked }); + ASSERT_TRUE( + builder::writeCompleteProjectLock(builder.projectYamlFile, *builder.projectLock)); + auto before = utils::readFile(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(before); + for (const auto &module : { "binary", "develop" }) { + EXPECT_CALL(*m_repo, getLayerDir(*reference, module)).WillOnce(Return(layerCached())); + } + EXPECT_CALL(*m_repo, mergeModules()).WillOnce(Return(utils::error::Result{})); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, false)).Times(0); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).Times(0); + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)).Times(0); + EXPECT_CALL(*m_repo, getModuleList(_)).Times(0); + + testing::internal::CaptureStdout(); + auto result = builder.buildStagePullDependency(); + const auto output = testing::internal::GetCapturedStdout(); + ASSERT_TRUE(result) << result.error().message(); + EXPECT_THAT(output, testing::HasSubstr("binary complete\n")); + EXPECT_THAT(output, testing::HasSubstr("develop complete\n")); + auto after = utils::readFile(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(after); + EXPECT_EQ(*after, *before); + testing::Mock::VerifyAndClearExpectations(m_repo.get()); + } +} + +TEST_F(PullDependencyTest, SkipPullRejectsMissingLockedModulesWithoutRemoteLookup) +{ + LINGLONG_TRACE("reject unavailable local locked dependency"); + auto reference = + package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + ASSERT_TRUE(reference); + builder::Builder builder(builderProject(), + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + const builder::LockedDependency entry{ *builder.project->base, + { { "binary" }, lockedReference(reference) }, + "base" }; + builder.projectLock = builder::completeProjectLock({}, entry); + builder.setBuildOptions(builder::BuilderBuildOptions{ .skipPullDepend = true }); + EXPECT_CALL(*m_repo, getLayerDir(*reference, "binary")) + .WillOnce(Return(LINGLONG_ERR("missing module"))); + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)).Times(0); + EXPECT_CALL(*m_repo, pull(_, _, _)).Times(0); + EXPECT_CALL(*m_repo, mergeModules()).Times(0); + + testing::internal::CaptureStdout(); + auto result = builder.buildStagePullDependency(); + const auto output = testing::internal::GetCapturedStdout(); + ASSERT_FALSE(result); + EXPECT_THAT(result.error().message(), testing::HasSubstr("unavailable locally")); + EXPECT_THAT(output, testing::HasSubstr("binary failed\n")); + EXPECT_FALSE(builder.pendingSources); +} + +TEST_F(PullDependencyTest, UnchangedRuntimeBaseDeclarationReusesLockedResources) +{ + auto oldBase = + package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + auto runtime = + package::Reference::parse("main:org.deepin.runtime/23.0.0.0/" + currentArchitecture); + ASSERT_TRUE(oldBase); + ASSERT_TRUE(runtime); + for (const bool locked : { false, true }) { + SCOPED_TRACE(locked); + auto project = builderProject("org.deepin.runtime/23.0.0.0/" + currentArchitecture); + project.base.reset(); + builder::Builder builder(project, + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (m_tmpDir.path() / "linglong.yaml").string(); + const builder::LockedDependency baseEntry{ "org.deepin.base/23.0.0.0/" + + currentArchitecture, + { { "binary" }, lockedReference(oldBase) }, + "base" }; + const builder::LockedDependency runtimeEntry{ *project.runtime, + { { "binary" }, lockedReference(runtime) }, + "runtime" }; + builder.projectLock = builder::completeProjectLock({}, baseEntry, runtimeEntry); + builder.setBuildOptions( + builder::BuilderBuildOptions{ .skipPullDepend = true, .locked = locked }); + ASSERT_TRUE( + builder::writeCompleteProjectLock(builder.projectYamlFile, *builder.projectLock)); + auto before = utils::readFile(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(before); + api::types::v1::RepositoryCacheLayersItem runtimeItem; + runtimeItem.info.base = baseEntry.requested; + EXPECT_CALL(*m_repo, getLayerItem(*runtime, "binary")) + .Times(1) + .WillRepeatedly([runtimeItem](const package::Reference &, const std::string &) + -> utils::error::Result { + return runtimeItem; + }); + EXPECT_CALL(*m_repo, getLayerDir(*runtime, "binary")).WillOnce(Return(layerCached())); + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)).Times(0); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).Times(0); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, false)).Times(0); + EXPECT_CALL(*m_repo, getModuleList(_)).Times(0); + EXPECT_CALL(*m_repo, getLayerDir(*oldBase, "binary")).WillOnce(Return(layerCached())); + EXPECT_CALL(*m_repo, mergeModules()).WillOnce(Return(utils::error::Result{})); + + auto result = builder.buildStagePullDependency(); + ASSERT_TRUE(result) << result.error().message(); + auto after = utils::readFile(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(after); + EXPECT_EQ(builder.project->base, project.base); + EXPECT_EQ(*after, *before); + testing::Mock::VerifyAndClearExpectations(m_repo.get()); + } +} + +TEST_F(PullDependencyTest, RemovingExplicitBaseFollowsRuntimeDeclaration) +{ + LINGLONG_TRACE("derive base declaration after removing explicit base"); + + auto oldBase = package::Reference::parse("main:org.test.Base/1.0.1/" + currentArchitecture); + auto newBase = package::Reference::parse("main:org.test.Base/1.0.2/" + currentArchitecture); + auto runtime = package::Reference::parse("main:org.test.Runtime/2.0.0/" + currentArchitecture); + ASSERT_TRUE(oldBase); + ASSERT_TRUE(newBase); + ASSERT_TRUE(runtime); + + for (const bool differentReference : { false, true }) { + SCOPED_TRACE(differentReference); + const auto &expectedBase = differentReference ? *newBase : *oldBase; + for (const bool locked : { false, true }) { + SCOPED_TRACE(locked); + auto project = builderProject("org.test.Runtime/2.0"); + project.base = "org.test.Base/1.0.1"; + const builder::LockedDependency baseEntry{ *project.base, + { { "binary" }, lockedReference(oldBase) }, + "base" }; + const builder::LockedDependency runtimeEntry{ *project.runtime, + { { "binary" }, + lockedReference(runtime) }, + "runtime" }; + const auto originalLock = builder::completeProjectLock({}, baseEntry, runtimeEntry); + auto previousState = builder::projectLockDependencyState(originalLock); + ASSERT_TRUE(previousState); + + project.base.reset(); + builder::Builder builder(project, + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (m_tmpDir.path() / "linglong.yaml").string(); + builder.projectLock = originalLock; + builder.setBuildOptions( + builder::BuilderBuildOptions{ .skipPullDepend = true, .locked = locked }); + ASSERT_TRUE(builder::writeCompleteProjectLock(builder.projectYamlFile, originalLock)); + auto before = utils::readFile(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(before); + + api::types::v1::RepositoryCacheLayersItem runtimeItem; + runtimeItem.info.base = "org.test.Base/1.0"; + EXPECT_CALL(*m_repo, getLayerDir(*runtime, "binary")).WillOnce(Return(layerCached())); + EXPECT_CALL(*m_repo, getLayerItem(*runtime, "binary")).WillOnce(Return(runtimeItem)); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, false)).Times(0); + EXPECT_CALL(*m_repo, latestRemoteReference(_)).Times(0); + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)).Times(0); + EXPECT_CALL(*m_repo, getModuleList(*runtime)).Times(0); + if (locked) { + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).Times(0); + EXPECT_CALL(*m_repo, getLayerDir(*oldBase, _)).Times(0); + EXPECT_CALL(*m_repo, getLayerDir(*newBase, _)).Times(0); + EXPECT_CALL(*m_repo, getModuleList(_)).Times(0); + EXPECT_CALL(*m_repo, mergeModules()).Times(0); + } else { + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)) + .WillOnce([&expectedBase](const package::FuzzyReference &requested, bool) { + EXPECT_EQ(requested.id, "org.test.Base"); + EXPECT_EQ(requested.version, "1.0"); + return expectedBase; + }); + EXPECT_CALL(*m_repo, getLayerDir(expectedBase, "binary")) + .WillOnce(Return(layerCached())); + EXPECT_CALL(*m_repo, getLayerDir(expectedBase, "develop")) + .WillOnce(Return(LINGLONG_ERR("develop module missing"))); + EXPECT_CALL(*m_repo, getModuleList(expectedBase)) + .WillOnce(Return(std::vector{ "binary" })); + EXPECT_CALL(*m_repo, mergeModules()).WillOnce(Return(utils::error::Result{})); + } + + auto result = builder.buildStagePullDependency(); + auto after = utils::readFile(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(after); + EXPECT_FALSE(builder.project->base); + EXPECT_EQ(builder.project->runtime, project.runtime); + if (locked) { + ASSERT_FALSE(result); + EXPECT_THAT(result.error().message(), testing::HasSubstr("base differs")); + EXPECT_EQ(*after, *before); + testing::Mock::VerifyAndClearExpectations(m_repo.get()); + continue; + } + + ASSERT_TRUE(result) << result.error().message(); + EXPECT_NE(*after, *before); + auto written = + builder::readProjectLock(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(written); + auto writtenBase = builder::lockedDependency(*written, "base"); + auto writtenRuntime = builder::lockedDependency(*written, "runtime"); + ASSERT_TRUE(writtenBase); + ASSERT_TRUE(writtenRuntime); + EXPECT_EQ(writtenBase->requested, runtimeItem.info.base); + EXPECT_EQ(writtenBase->resolved.reference, lockedReference(expectedBase, false)); + EXPECT_EQ(writtenRuntime->requested, runtimeEntry.requested); + EXPECT_EQ(writtenRuntime->resolved.reference, runtimeEntry.resolved.reference); + auto state = builder::projectLockDependencyState(*written); + ASSERT_TRUE(state); + EXPECT_EQ(*state == *previousState, !differentReference); + + TempDir workingDir; + const auto root = workingDir.path() / "repo"; + std::filesystem::create_directories(root); + auto repo = repo::OSTreeRepo::create(root, m_repo->getConfig()); + ASSERT_TRUE(repo) << repo.error().message(); + builder::Builder committed(project, + workingDir.path(), + **repo, + containerBuilder(), + builderConfig()); + committed.projectLock = builder.projectLock; + auto reference = package::Reference::fromBuilderProject(project); + ASSERT_TRUE(reference); + committed.projectRef = *reference; + committed.projectYamlFile = (workingDir.path() / "linglong.yaml").string(); + std::ofstream(committed.projectYamlFile) << "version: 1\n"; + committed.packageModules = { "binary" }; + const auto output = committed.directories.module("binary"); + std::filesystem::create_directories(output / "files"); + std::ofstream(output / "files/payload") << "content"; + + auto committedResult = committed.commitToLocalRepo(); + ASSERT_TRUE(committedResult) << committedResult.error().message(); + auto layer = (*repo)->getLayerDir(*reference, "binary"); + ASSERT_TRUE(layer); + auto info = layer->info(); + ASSERT_TRUE(info); + EXPECT_EQ(info->base, runtimeItem.info.base); + EXPECT_EQ(info->runtime, project.runtime); + testing::Mock::VerifyAndClearExpectations(m_repo.get()); + } + } +} + +TEST_F(PullDependencyTest, FreshRuntimeDerivesBaseWhenYamlOmitsBase) +{ + auto oldBase = + package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + auto oldRuntime = + package::Reference::parse("main:org.deepin.runtime/23.0.0.0/" + currentArchitecture); + auto newBase = + package::Reference::parse("main:org.deepin.base/24.0.0.0/" + currentArchitecture); + auto newRuntime = + package::Reference::parse("main:org.deepin.runtime/24.0.0.0/" + currentArchitecture); + ASSERT_TRUE(oldBase); + ASSERT_TRUE(oldRuntime); + ASSERT_TRUE(newBase); + ASSERT_TRUE(newRuntime); + for (const int scenario : { 0, 1, 2 }) { + SCOPED_TRACE(scenario); + const bool existingLock = scenario != 0; + const bool reuseBase = scenario == 2; + const auto &expectedBase = reuseBase ? *oldBase : *newBase; + auto project = builderProject("org.deepin.runtime/24.0.0.0/" + currentArchitecture); + project.base.reset(); + builder::Builder builder(project, + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (m_tmpDir.path() / "linglong.yaml").string(); + builder.setBuildOptions(builder::BuilderBuildOptions{ .skipPullDepend = true }); + if (existingLock) { + const builder::LockedDependency baseEntry{ "org.deepin.base/23.0.0.0/" + + currentArchitecture, + { { "binary" }, lockedReference(oldBase) }, + "base" }; + const builder::LockedDependency runtimeEntry{ + "org.deepin.runtime/23.0.0.0/" + currentArchitecture, + { { "binary" }, lockedReference(oldRuntime) }, + "runtime" + }; + builder.projectLock = builder::completeProjectLock({}, baseEntry, runtimeEntry); + } else { + builder.pendingSources.emplace(); + } + + if (reuseBase) { + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(*newRuntime)); + } else { + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)) + .WillOnce(Return(*newRuntime)) + .WillOnce(Return(expectedBase)); + } + EXPECT_CALL(*m_repo, clearReferenceLocal(_, false)).Times(0); + EXPECT_CALL(*m_repo, latestRemoteReference(_)).Times(0); + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)).Times(0); + EXPECT_CALL(*m_repo, getLayerDir(_, _)) + .WillRepeatedly([](const package::Reference &, + const std::string &) -> utils::error::Result { + return layerCached(); + }); + api::types::v1::RepositoryCacheLayersItem runtimeItem; + runtimeItem.info.base = reuseBase ? "org.deepin.base/23.0.0.0/" + currentArchitecture + : "org.deepin.base/24.0.0.0/" + currentArchitecture; + EXPECT_CALL(*m_repo, getLayerItem(*newRuntime, "binary")) + .Times(1) + .WillRepeatedly([runtimeItem](const package::Reference &, const std::string &) + -> utils::error::Result { + return runtimeItem; + }); + EXPECT_CALL(*m_repo, getLayerItem(expectedBase, _)).Times(0); + if (reuseBase) { + EXPECT_CALL(*m_repo, getModuleList(expectedBase)).Times(0); + } else { + EXPECT_CALL(*m_repo, getModuleList(expectedBase)) + .WillOnce(Return(std::vector{ "binary" })); + } + EXPECT_CALL(*m_repo, getModuleList(*newRuntime)) + .WillOnce(Return(std::vector{ "binary" })); + EXPECT_CALL(*m_repo, mergeModules()).WillOnce(Return(utils::error::Result{})); + + auto result = builder.buildStagePullDependency(); + ASSERT_TRUE(result) << result.error().message(); + EXPECT_EQ(builder.project->base, project.base); + EXPECT_EQ(builder.project->runtime, project.runtime); + EXPECT_FALSE(builder.pendingSources); + auto written = builder::readProjectLock(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(written); + ASSERT_EQ(written->dependencies.size(), 2); + EXPECT_EQ(written->dependencies[0].requested, runtimeItem.info.base); + EXPECT_EQ(written->dependencies[0].resolved.reference, lockedReference(expectedBase)); + EXPECT_EQ(written->dependencies[1].requested, *project.runtime); + EXPECT_EQ(written->dependencies[1].resolved.reference, lockedReference(newRuntime)); + testing::Mock::VerifyAndClearExpectations(m_repo.get()); + } +} + +TEST_F(PullDependencyTest, CommittedMetadataPreservesRuntimeDependencyDeclarations) +{ + for (const int mode : { 0, 1, 2 }) { + SCOPED_TRACE(mode); + const bool withRuntime = mode != 0; + TempDir workingDir; + const auto root = workingDir.path() / "repo"; + std::filesystem::create_directories(root); + auto repo = repo::OSTreeRepo::create(root, m_repo->getConfig()); + ASSERT_TRUE(repo) << repo.error().message(); + auto project = builderProject( + withRuntime ? std::make_optional(std::string("org.test.Runtime/2.0.0")) : std::nullopt); + if (mode == 2) { + project.base.reset(); + } else { + project.base = "org.test.Base/1.0.0"; + } + builder::Builder builder(project, + workingDir.path(), + **repo, + containerBuilder(), + builderConfig()); + const builder::LockedDependency base{ "org.test.Base/1.0.0" + + (mode == 2 ? "/" + currentArchitecture : ""), + { { "binary" }, "main:org.test.Base/1.0.0.0" }, + "base" }; + const builder::LockedDependency runtime{ "org.test.Runtime/2.0.0", + { { "binary" }, "main:org.test.Runtime/2.0.0.0" }, + "runtime" }; + builder.projectLock = + builder::completeProjectLock({}, + base, + withRuntime ? std::make_optional(runtime) : std::nullopt); + auto reference = package::Reference::fromBuilderProject(project); + ASSERT_TRUE(reference); + builder.projectRef = *reference; + builder.projectYamlFile = (workingDir.path() / "linglong.yaml").string(); + std::ofstream(builder.projectYamlFile) << "version: 1\n"; + builder.packageModules = { "binary", "develop" }; + for (const auto &module : builder.packageModules) { + const auto output = builder.directories.module(module); + std::filesystem::create_directories(output / "files"); + std::ofstream(output / "files/payload") << module; + } + + std::string expectedBase = base.requested; + if (mode == 2) { + const auto architecture = package::Architecture::currentCPUArchitecture().toString(); + for (const auto &id : { "org.test.Base", "org.test.Runtime" }) { + const auto input = workingDir.path() / id; + std::filesystem::create_directories(input / "files"); + const std::string version = + std::string(id) == "org.test.Base" ? "1.0.0.0" : "2.0.0.0"; + api::types::v1::PackageInfoV2 info{ .arch = { architecture }, + .base = expectedBase, + .channel = "main", + .id = id, + .kind = "runtime", + .packageInfoV2Module = "binary", + .version = version }; + std::ofstream(input / "info.json") << nlohmann::json(info).dump(); + std::ofstream(input / "files/payload") << id; + ASSERT_TRUE((*repo)->importLayerDir(package::LayerDir(input))); + } + ASSERT_TRUE((*repo)->mergeModules()); + auto context = builder.buildContext.resolve(project, + builder.directories.buildOutput(), + *builder.projectLock); + ASSERT_TRUE(context) << context.error().message(); + ASSERT_TRUE(builder.buildContext.getRuntimeLayer()); + EXPECT_EQ(builder.buildContext.getRuntimeLayer()->getCachedItem().info.base, + expectedBase); + } + + auto committed = builder.commitToLocalRepo(); + ASSERT_TRUE(committed) << committed.error().message(); + for (const auto &module : builder.packageModules) { + const auto layer = (*repo)->getLayerDir(*reference, module); + ASSERT_TRUE(layer); + const auto info = layer->info(); + ASSERT_TRUE(info); + EXPECT_EQ(info->base, expectedBase); + EXPECT_EQ(info->runtime, project.runtime); + } + EXPECT_EQ(builder.project->base, project.base); + EXPECT_EQ(builder.project->runtime, project.runtime); + } +} + +TEST_F(PullDependencyTest, RejectsDependencyResolutionOutsideDeclaredReference) +{ + LINGLONG_TRACE("validate dependency resolution before recording a lock entry"); + + builder::Builder builder(builderProject(), + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + const std::string requested = "main:org.test.Base/1.0.0.0/x86_64"; + EXPECT_CALL(*m_repo, getModuleList(_)).Times(0); + for (const auto &resolved : { "main:org.test.Other/1.0.0.0/x86_64", + "other:org.test.Base/1.0.0.0/x86_64", + "main:org.test.Base/9.0.0.0/x86_64", + "main:org.test.Base/1.0.0.0/arm64" }) { + SCOPED_TRACE(resolved); + auto reference = package::Reference::parse(resolved); + ASSERT_TRUE(reference) << reference.error().message(); + auto entry = + builder.recordProjectDependency(builder::ResolveDependencyRequest{ requested, "base" }, + *reference); + ASSERT_FALSE(entry); + EXPECT_THAT(entry.error().message(), testing::HasSubstr("does not match its declaration")); + } + testing::Mock::VerifyAndClearExpectations(m_repo.get()); + + auto reference = package::Reference::parse("main:org.test.Base/1.0.0.1/" + currentArchitecture); + ASSERT_TRUE(reference); + EXPECT_CALL(*m_repo, getModuleList(*reference)) + .WillOnce(Return(std::vector{ "binary" })); + auto entry = builder.recordProjectDependency( + builder::ResolveDependencyRequest{ "org.test.Base/1.0.0", "base" }, + *reference); + ASSERT_TRUE(entry) << entry.error().message(); + EXPECT_EQ(entry->resolved.reference, "main:org.test.Base/1.0.0.1/unknown"); + const auto manifest = m_tmpDir.path() / "linglong.yaml"; + ASSERT_TRUE( + builder::writeCompleteProjectLock(manifest, builder::completeProjectLock({}, *entry))); + EXPECT_TRUE(builder::readProjectLock(builder::projectLockPath(manifest))); +} + +TEST_F(PullDependencyTest, PreservesLegacyRuntimeModuleInLock) +{ + const auto reference = + package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + ASSERT_TRUE(reference); + builder::Builder builder(builderProject(), + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + EXPECT_CALL(*m_repo, getModuleList(*reference)) + .WillOnce(Return(std::vector{ "runtime", "develop" })); + + const auto entry = builder.recordProjectDependency( + builder::ResolveDependencyRequest{ *builder.project->base, "base" }, + *reference); + ASSERT_TRUE(entry) << entry.error().message(); + EXPECT_EQ(entry->resolved.modules, (std::vector{ "develop", "runtime" })); + const auto manifest = m_tmpDir.path() / "linglong.yaml"; + ASSERT_TRUE( + builder::writeCompleteProjectLock(manifest, builder::completeProjectLock({}, *entry))); + const auto loaded = builder::readProjectLock(builder::projectLockPath(manifest)); + ASSERT_TRUE(loaded); + EXPECT_EQ(loaded->dependencies.front().resolved.modules, entry->resolved.modules); +} + +TEST_F(PullDependencyTest, LockedModulesShareExactRemoteLookup) +{ + LINGLONG_TRACE("fetch locked modules with one remote lookup"); + auto reference = + package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + auto newer = package::Reference::parse("main:org.deepin.base/24.0.0.0/" + currentArchitecture); + ASSERT_TRUE(reference); + ASSERT_TRUE(newer); + builder::Builder builder(builderProject(), + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + const builder::LockedDependency entry{ *builder.project->base, + { { "binary", "develop" }, lockedReference(reference) }, + "base" }; + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)) + .WillOnce([&](const package::FuzzyReference &fuzzy, + const auto &) -> utils::error::Result { + EXPECT_EQ(fuzzy.toString(), reference->toString()); + return remotePackages({ *newer, *reference }); + }); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, _)).Times(0); + EXPECT_CALL(*m_repo, latestRemoteReference(_)).Times(0); + EXPECT_CALL(*m_repo, getLayerItem(_, _)).Times(0); + for (const std::string module : { "binary", "develop" }) { + EXPECT_CALL(*m_repo, getLayerDir(*reference, module)) + .WillOnce(Return(LINGLONG_ERR("not installed"))) + .WillOnce(Return(LINGLONG_ERR("not installed"))) + .WillOnce(Return(layerCached())); + EXPECT_CALL(*m_repo, pull(_, _, module)) + .WillOnce([&](service::Task &, + const package::ReferenceWithRepo &remote, + const auto &) -> utils::error::Result { + EXPECT_EQ(remote.reference, *reference); + EXPECT_EQ(remote.repo.name, "custom"); + return LINGLONG_OK; + }); + } + testing::internal::CaptureStdout(); + auto result = builder.fetchProjectDependency(builder::LockedDependencyRequest{ entry }); + const auto output = testing::internal::GetCapturedStdout(); + ASSERT_TRUE(result) << result.error().message(); + EXPECT_EQ(*result, *reference); + EXPECT_THAT(output, testing::HasSubstr("binary complete\n")); + EXPECT_THAT(output, testing::HasSubstr("develop complete\n")); +} + +TEST_F(PullDependencyTest, LockedDependencyRejectsOtherRemoteVersions) +{ + LINGLONG_TRACE("reject replacement versions for locked dependency"); + auto reference = + package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + auto newer = package::Reference::parse("main:org.deepin.base/24.0.0.0/" + currentArchitecture); + ASSERT_TRUE(reference); + ASSERT_TRUE(newer); + builder::Builder builder(builderProject(), + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + const builder::LockedDependency entry{ *builder.project->base, + { { "binary" }, lockedReference(reference) }, + "base" }; + EXPECT_CALL(*m_repo, getLayerDir(*reference, "binary")) + .WillOnce(Return(LINGLONG_ERR("not installed"))); + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)) + .WillOnce([&](const package::FuzzyReference &, + const auto &) -> utils::error::Result { + return remotePackages({ *newer }); + }); + EXPECT_CALL(*m_repo, pull(_, _, _)).Times(0); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, _)).Times(0); + EXPECT_CALL(*m_repo, latestRemoteReference(_)).Times(0); + auto result = builder.fetchProjectDependency(builder::LockedDependencyRequest{ entry }); + ASSERT_FALSE(result); + EXPECT_THAT(result.error().message(), testing::HasSubstr("unavailable remotely")); +} + +TEST_F(PullDependencyTest, MissingDevelopIsRequiredByLockAndOptionalForResolution) +{ + LINGLONG_TRACE("check required and optional develop modules without remote access"); + auto reference = + package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + ASSERT_TRUE(reference); + for (const bool locked : { false, true }) { + SCOPED_TRACE(locked); + builder::Builder builder(builderProject(), + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.setBuildOptions(builder::BuilderBuildOptions{ .skipPullDepend = true }); + builder::DependencyFetchRequest request = + builder::ResolveDependencyRequest{ *builder.project->base, "base" }; + if (locked) { + request = builder::LockedDependencyRequest{ { *builder.project->base, + { { "binary", "develop" }, + lockedReference(reference) }, + "base" } }; + EXPECT_CALL(*m_repo, clearReferenceLocal(_, _)).Times(0); + } else { + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(*reference)); + } + expectCachedModule(*reference, "binary"); + EXPECT_CALL(*m_repo, getLayerDir(*reference, "develop")) + .WillOnce(Return(LINGLONG_ERR("not installed"))); + EXPECT_CALL(*m_repo, latestRemoteReference(_)).Times(0); + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)).Times(0); + EXPECT_CALL(*m_repo, pull(_, _, _)).Times(0); + testing::internal::CaptureStdout(); + auto result = builder.fetchProjectDependency(request); + const auto output = testing::internal::GetCapturedStdout(); + EXPECT_THAT(output, testing::HasSubstr("binary complete\n")); + if (locked) { + ASSERT_FALSE(result); + EXPECT_THAT(output, testing::HasSubstr("develop failed\n")); + } else { + ASSERT_TRUE(result) << result.error().message(); + EXPECT_EQ(*result, *reference); + EXPECT_THAT(output, testing::HasSubstr("develop missing\n")); + } + testing::Mock::VerifyAndClearExpectations(m_repo.get()); + } +} + +TEST_F(PullDependencyTest, ResolvedBinaryFallbackFetchesDevelopForFinalReference) +{ + LINGLONG_TRACE("fetch develop for the local fallback version"); + auto local = package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + auto newer = package::Reference::parse("main:org.deepin.base/24.0.0.0/" + currentArchitecture); + ASSERT_TRUE(local); + ASSERT_TRUE(newer); + builder::Builder builder(builderProject(), + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(*local)); + EXPECT_CALL(*m_repo, latestRemoteReference(_)).WillOnce(Return(refWithRepo(*newer))); + EXPECT_CALL(*m_repo, getLayerDir(*newer, "binary")) + .WillOnce(Return(LINGLONG_ERR("not installed"))) + .WillOnce(Return(LINGLONG_ERR("not installed"))); + EXPECT_CALL(*m_repo, pull(_, _, "binary")) + .WillOnce([&](service::Task &, + const package::ReferenceWithRepo &remote, + const auto &) -> utils::error::Result { + EXPECT_EQ(remote.reference, *newer); + return LINGLONG_ERR("download failed"); + }); + expectCachedModule(*local, "binary"); + EXPECT_CALL(*m_repo, getLayerDir(*local, "develop")) + .WillOnce(Return(LINGLONG_ERR("not installed"))) + .WillOnce(Return(LINGLONG_ERR("not installed"))) + .WillOnce(Return(layerCached())); + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)) + .WillOnce([&](const package::FuzzyReference &fuzzy, + const auto &) -> utils::error::Result { + EXPECT_EQ(fuzzy.toString(), local->toString()); + return remotePackages({ *newer, *local }); + }); + EXPECT_CALL(*m_repo, pull(_, _, "develop")) + .WillOnce([&](service::Task &, + const package::ReferenceWithRepo &remote, + const auto &) -> utils::error::Result { + EXPECT_EQ(remote.reference, *local); + return LINGLONG_OK; + }); + EXPECT_CALL(*m_repo, getLayerDir(*newer, "develop")).Times(0); + EXPECT_CALL(*m_repo, getLayerItem(_, _)).Times(0); + testing::internal::CaptureStdout(); + auto result = builder.fetchProjectDependency( + builder::ResolveDependencyRequest{ "org.deepin.base", "base" }); + const auto output = testing::internal::GetCapturedStdout(); + ASSERT_TRUE(result) << result.error().message(); + EXPECT_EQ(*result, *local); + EXPECT_THAT(output, testing::HasSubstr("binary complete\n")); + EXPECT_THAT(output, testing::HasSubstr("develop complete\n")); +} + +TEST_F(PullDependencyTest, LockedDependencyFetchesExactVersionForSelectedArchitecture) +{ + EXPECT_CALL(*m_repo, clearReferenceLocal(_, _)).Times(0); + EXPECT_CALL(*m_repo, latestRemoteReference(_)).Times(0); + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)).Times(0); + EXPECT_CALL(*m_repo, pull(_, _, _)).Times(0); + for (const std::string architecture : { "unknown", "arm64" }) { + SCOPED_TRACE(architecture); + const builder::LockedDependency entry{ "org.test.Base/1.0.0", + { { "binary", "develop" }, + "main:org.test.Base/1.0.0.0/" + architecture }, + "base" }; + const auto original = nlohmann::json(entry); + auto reference = package::Reference::parse( + "main:org.test.Base/1.0.0.0/" + + (architecture == "unknown" ? currentArchitecture : architecture)); + ASSERT_TRUE(reference); + builder::Builder builder(builderProject(), + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.setBuildOptions(builder::BuilderBuildOptions{ .skipPullDepend = true }); + expectCachedModule(*reference, "binary"); + expectCachedModule(*reference, "develop"); + auto result = builder.fetchProjectDependency(builder::LockedDependencyRequest{ entry }); + ASSERT_TRUE(result) << result.error().message(); + EXPECT_EQ(*result, *reference); + EXPECT_EQ(nlohmann::json(entry), original); + } +} + +TEST_F(PullDependencyTest, SharedLockDoesNotFallBackToAnotherArchitectureOrVersion) +{ + LINGLONG_TRACE("reject a missing locked dependency for the selected architecture"); + + const builder::LockedDependency entry{ "org.test.Base/1.0.0", + { { "binary" }, "main:org.test.Base/1.0.0.0/unknown" }, + "base" }; + const auto reference = + package::Reference::parse("main:org.test.Base/1.0.0.0/" + currentArchitecture); + ASSERT_TRUE(reference); + builder::Builder builder(builderProject(), + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.setBuildOptions(builder::BuilderBuildOptions{ .skipPullDepend = true }); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, _)).Times(0); + EXPECT_CALL(*m_repo, latestRemoteReference(_)).Times(0); + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)).Times(0); + EXPECT_CALL(*m_repo, pull(_, _, _)).Times(0); + EXPECT_CALL(*m_repo, getLayerDir(*reference, "binary")) + .WillOnce(Return(LINGLONG_ERR("missing module for build host"))); + auto result = builder.fetchProjectDependency(builder::LockedDependencyRequest{ entry }); + ASSERT_FALSE(result); + EXPECT_THAT(result.error().message(), testing::HasSubstr(reference->toString())); +} + +TEST_F(PullDependencyTest, ImplicitBaseUsesCurrentRuntimeMetadataArchitecture) +{ + auto project = builderProject("org.test.Runtime/2.0.0/arm64"); + project.base.reset(); + const builder::LockedDependency base{ "org.test.Base/1.0.0/arm64", + { { "binary" }, "main:org.test.Base/1.0.0.0/arm64" }, + "base" }; + const builder::LockedDependency runtime{ *project.runtime, + { { "binary" }, + "main:org.test.Runtime/2.0.0.0/arm64" }, + "runtime" }; + const auto baseReference = package::Reference::parse("main:org.test.Base/1.0.0.0/arm64"); + const auto runtimeReference = package::Reference::parse("main:org.test.Runtime/2.0.0.0/arm64"); + ASSERT_TRUE(baseReference); + ASSERT_TRUE(runtimeReference); + builder::Builder builder(project, + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectLock = builder::completeProjectLock({}, base, runtime); + const auto original = nlohmann::json(*builder.projectLock); + builder.setBuildOptions(builder::BuilderBuildOptions{ .skipPullDepend = true, .locked = true }); + api::types::v1::RepositoryCacheLayersItem item; + item.info.base = "org.test.Base/1.0.0/arm64"; + EXPECT_CALL(*m_repo, getLayerItem(*runtimeReference, "binary")).WillOnce(Return(item)); + expectCachedModule(*baseReference, "binary"); + expectCachedModule(*runtimeReference, "binary"); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, _)).Times(0); + EXPECT_CALL(*m_repo, latestRemoteReference(_)).Times(0); + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)).Times(0); + EXPECT_CALL(*m_repo, pull(_, _, _)).Times(0); + EXPECT_CALL(*m_repo, mergeModules()).WillOnce(Return(utils::error::Result{})); + auto result = builder.buildStagePullDependency(); + ASSERT_TRUE(result) << result.error().message(); + EXPECT_EQ(nlohmann::json(*builder.projectLock), original); + EXPECT_FALSE(builder.project->base); +} + +TEST_F(PullDependencyTest, DerivedBasePreservesDeclarationConstraints) +{ + const std::string other = currentArchitecture == "arm64" ? "x86_64" : "arm64"; + const std::vector> declarations{ + { "org.test.Base", "unknown" }, + { "org.test.Base/unknown", "unknown" }, + { "org.test.Base/unknown/" + other, other }, + { "org.test.Base/1.0.0", "unknown" }, + { "org.test.Base/1.0.0/unknown", "unknown" }, + { "org.test.Base/1.0.0/" + other, other } + }; + for (const auto &[declaration, architecture] : declarations) { + SCOPED_TRACE(declaration); + TempDir workingDir; + auto project = builderProject("org.test.Runtime/2.0.0/" + other); + project.base.reset(); + const auto base = package::Reference::parse( + "main:org.test.Base/1.0.0.0/" + + (architecture == "unknown" ? currentArchitecture : architecture)); + const auto runtime = package::Reference::parse("main:org.test.Runtime/2.0.0.0/" + other); + ASSERT_TRUE(base); + ASSERT_TRUE(runtime); + builder::Builder builder(project, + workingDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (workingDir.path() / "linglong.yaml").string(); + builder.setBuildOptions(builder::BuilderBuildOptions{ .skipPullDepend = true }); + + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)) + .WillOnce(Return(*runtime)) + .WillOnce(Return(*base)); + EXPECT_CALL(*m_repo, latestRemoteReference(_)).Times(0); + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)).Times(0); + EXPECT_CALL(*m_repo, pull(_, _, _)).Times(0); + for (const auto &reference : { *base, *runtime }) { + expectCachedModule(reference, "binary"); + expectCachedModule(reference, "develop"); + EXPECT_CALL(*m_repo, getModuleList(reference)) + .WillOnce(Return(std::vector{ "binary", "develop" })); + } + api::types::v1::RepositoryCacheLayersItem item; + item.info.base = declaration; + EXPECT_CALL(*m_repo, getLayerItem(*runtime, "binary")).WillOnce(Return(item)); + EXPECT_CALL(*m_repo, mergeModules()).WillOnce(Return(utils::error::Result{})); + + auto result = builder.buildStagePullDependency(); + ASSERT_TRUE(result) << result.error().message(); + auto lock = builder::readProjectLock(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(lock) << lock.error().message(); + auto lockedBase = builder::lockedDependency(*lock, "base"); + ASSERT_TRUE(lockedBase); + EXPECT_EQ(lockedBase->requested, declaration); + EXPECT_EQ(lockedBase->resolved.reference, "main:org.test.Base/1.0.0.0/" + architecture); + testing::Mock::VerifyAndClearExpectations(m_repo.get()); + + builder.setBuildOptions( + builder::BuilderBuildOptions{ .skipPullDepend = true, .locked = true }); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, _)).Times(0); + EXPECT_CALL(*m_repo, latestRemoteReference(_)).Times(0); + EXPECT_CALL(*m_repo, getModuleList(_)).Times(0); + EXPECT_CALL(*m_repo, getLayerItem(*runtime, "binary")).WillOnce(Return(item)); + EXPECT_CALL(*m_repo, mergeModules()).WillOnce(Return(utils::error::Result{})); + for (const auto &reference : { *base, *runtime }) { + expectCachedModule(reference, "binary"); + expectCachedModule(reference, "develop"); + } + result = builder.buildStagePullDependency(); + ASSERT_TRUE(result) << result.error().message(); + EXPECT_EQ(nlohmann::json(*builder.projectLock), nlohmann::json(*lock)); + testing::Mock::VerifyAndClearExpectations(m_repo.get()); + } +} + +TEST_F(PullDependencyTest, CrossBuildGeneratesAndUpdatesLock) +{ + auto base = package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + ASSERT_TRUE(base); + for (const auto mode : + { builder::ProjectLockMode::Generate, builder::ProjectLockMode::Update }) { + SCOPED_TRACE(static_cast(mode)); + TempDir workingDir; + auto project = builderProject(); + project.package.architecture = "arm64"; + project.base = "org.deepin.base/23.0.0.0"; + builder::Builder builder(project, + workingDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (workingDir.path() / "linglong.arm64.yaml").string(); + builder.setBuildOptions(builder::BuilderBuildOptions{ .skipPullDepend = true }); + utils::EnvironmentVariableGuard cache("LINGLONG_FETCH_CACHE", + (workingDir.path() / "cache").string()); + if (mode == builder::ProjectLockMode::Update) { + const builder::LockedDependency oldBase{ *project.base, + { { "binary" }, + "main:org.deepin.base/23.0.0.0" }, + "base" }; + auto oldLock = builder::completeProjectLock({}, oldBase); + ASSERT_TRUE(builder::writeCompleteProjectLock(builder.projectYamlFile, oldLock)); + } + + expectFreshBase(*base); + auto result = builder.processProjectLock(mode); + ASSERT_TRUE(result) << result.error().message(); + auto written = builder::readProjectLock(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(written); + auto baseEntry = builder::lockedDependency(*written, "base"); + ASSERT_TRUE(baseEntry); + EXPECT_EQ(baseEntry->requested, *project.base); + EXPECT_EQ(baseEntry->resolved.reference, lockedReference(base, false)); + EXPECT_EQ(builder.project->package.architecture, "arm64"); + EXPECT_EQ(builder.project->base, project.base); + testing::Mock::VerifyAndClearExpectations(m_repo.get()); + } +} + +TEST_F(PullDependencyTest, PrepareLoadsSharedLockIndependentlyOfTargetArchitecture) +{ + for (const auto &target : { "x86_64", "arm64" }) { + SCOPED_TRACE(target); + TempDir workingDir; + // Preparation validates package metadata after loading the lock. + // An empty package keeps this check independent of installed builder helpers. + api::types::v1::BuilderProject project; + project.package.architecture = target; + builder::Builder builder(project, + workingDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (workingDir.path() / "linglong.yaml").string(); + const builder::LockedDependency base{ "org.deepin.base/23.0.0.0", + { { "binary" }, "main:org.deepin.base/23.0.0.0" }, + "base" }; + ASSERT_TRUE(builder::writeCompleteProjectLock(builder.projectYamlFile, + builder::completeProjectLock({}, base))); + auto result = builder.buildStagePrepare(); + ASSERT_FALSE(result); + EXPECT_THAT(result.error().message(), testing::HasSubstr("invalid project info")); + ASSERT_TRUE(builder.projectLock); + EXPECT_EQ(builder.projectLock->dependencies.front().resolved.reference, + base.resolved.reference); + } +} + +TEST_F(PullDependencyTest, FailedSharedLockUpdatePreservesOldLock) +{ + auto project = builderProject(); + project.base = "org.deepin.base/23.0.0.0"; + builder::Builder builder(project, + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (m_tmpDir.path() / "linglong.yaml").string(); + builder.setBuildOptions(builder::BuilderBuildOptions{ .skipPullDepend = true }); + utils::EnvironmentVariableGuard cache("LINGLONG_FETCH_CACHE", + (m_tmpDir.path() / "cache").string()); + const auto current = package::Architecture::currentCPUArchitecture(); + const builder::LockedDependency base{ *project.base, + { { "binary" }, "main:org.deepin.base/23.0.0.0" }, + "base" }; + auto lock = builder::completeProjectLock({}, base); + ASSERT_TRUE(builder::writeCompleteProjectLock(builder.projectYamlFile, lock)); + const auto path = builder::projectLockPath(builder.projectYamlFile); + auto before = utils::readFile(path); + ASSERT_TRUE(before); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)) + .WillOnce([&](const package::FuzzyReference &fuzzy, + bool) -> utils::error::Result { + LINGLONG_TRACE("look up a dependency for the new build architecture"); + EXPECT_EQ(fuzzy.arch, current); + return LINGLONG_ERR("not installed for current architecture"); + }); + EXPECT_CALL(*m_repo, latestRemoteReference(_)).Times(0); + EXPECT_CALL(*m_repo, pull(_, _, _)).Times(0); + + auto result = builder.processProjectLock(builder::ProjectLockMode::Update); + ASSERT_FALSE(result); + EXPECT_THAT(result.error().message(), testing::HasSubstr("failed to get local ref")); + auto after = utils::readFile(path); + ASSERT_TRUE(after); + EXPECT_EQ(*after, *before); +} + +TEST_F(PullDependencyTest, ResolvePreservesExplicitDependencyArchitecture) +{ + const auto current = package::Architecture::currentCPUArchitecture().toString(); + const std::string other = current == "arm64" ? "x86_64" : "arm64"; + const auto requested = "org.deepin.base/23.0.0.0/" + other; + auto reference = package::Reference::parse("main:" + requested); + ASSERT_TRUE(reference); + builder::Builder builder(builderProject(), + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.setBuildOptions(builder::BuilderBuildOptions{ .skipPullDepend = true }); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)) + .WillOnce([&](const package::FuzzyReference &fuzzy, bool) { + EXPECT_EQ(fuzzy.arch, reference->arch); + return *reference; + }); + EXPECT_CALL(*m_repo, latestRemoteReference(_)).Times(0); + expectCachedModule(*reference, "binary"); + expectCachedModule(*reference, "develop"); + + auto result = + builder.fetchProjectDependency(builder::ResolveDependencyRequest{ requested, "base" }); + ASSERT_TRUE(result) << result.error().message(); + EXPECT_EQ(*result, *reference); + + EXPECT_CALL(*m_repo, getModuleList(*reference)) + .WillOnce(Return(std::vector{ "binary", "develop" })); + auto entry = + builder.recordProjectDependency(builder::ResolveDependencyRequest{ requested, "base" }, + *result); + ASSERT_TRUE(entry) << entry.error().message(); + EXPECT_EQ(entry->requested, requested); + EXPECT_EQ(entry->resolved.reference, reference->toString()); +} + +TEST_F(PullDependencyTest, CrossBuildPreservesDependencyArchitectures) +{ + LINGLONG_TRACE("resolve and reuse host dependencies for a cross build"); + + auto base = package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + auto runtime = + package::Reference::parse("main:org.deepin.runtime/23.0.0.0/" + currentArchitecture); + ASSERT_TRUE(base); + ASSERT_TRUE(runtime); + + for (const bool withRuntime : { false, true }) { + SCOPED_TRACE(withRuntime); + for (const int scenario : { 0, 1, 2 }) { + SCOPED_TRACE(scenario); + const bool existingLock = scenario != 0; + auto project = builderProject(); + project.package.architecture = "arm64"; + project.base = "org.deepin.base/23.0.0"; + if (withRuntime) { + project.runtime = "org.deepin.runtime/23.0.0"; + project.base.reset(); + } + + builder::Builder builder(project, + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (m_tmpDir.path() + / ("cross-" + std::to_string(withRuntime) + "-" + + std::to_string(scenario) + ".yaml")) + .string(); + builder.setBuildOptions(builder::BuilderBuildOptions{ .locked = scenario == 2 }); + const builder::LockedDependency baseEntry{ "org.deepin.base/23.0.0", + { { "binary", "develop" }, + lockedReference(base, false) }, + "base" }; + std::optional runtimeEntry; + if (withRuntime) { + runtimeEntry = builder::LockedDependency{ *project.runtime, + { { "binary", "develop" }, + lockedReference(runtime, false) }, + "runtime" }; + } + std::optional before; + if (existingLock) { + builder.projectLock = builder::completeProjectLock({}, baseEntry, runtimeEntry); + ASSERT_TRUE( + builder::writeCompleteProjectLock(builder.projectYamlFile, *builder.projectLock)); + auto contents = utils::readFile(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(contents); + before = *contents; + EXPECT_CALL(*m_repo, clearReferenceLocal(_, _)).Times(0); + EXPECT_CALL(*m_repo, latestRemoteReference(_)).Times(0); + EXPECT_CALL(*m_repo, getModuleList(_)).Times(0); + } + + auto expectDependency = [&](const package::Reference &reference) { + expectCachedModule(reference, "binary"); + expectCachedModule(reference, "develop"); + if (existingLock) { + return; + } + + EXPECT_CALL( + *m_repo, + clearReferenceLocal(testing::Field(&package::FuzzyReference::id, reference.id), + true)) + .WillOnce([&reference](const package::FuzzyReference &fuzzy, + bool) -> utils::error::Result { + LINGLONG_TRACE("look up a cross build dependency locally"); + EXPECT_EQ(fuzzy.id, reference.id); + EXPECT_EQ(fuzzy.arch, package::Architecture::currentCPUArchitecture()); + return LINGLONG_ERR("not installed"); + }); + EXPECT_CALL( + *m_repo, + latestRemoteReference(testing::Field(&package::FuzzyReference::id, reference.id))) + .WillOnce([&reference](const package::FuzzyReference &fuzzy) { + EXPECT_EQ(fuzzy.id, reference.id); + EXPECT_EQ(fuzzy.arch, package::Architecture::currentCPUArchitecture()); + return refWithRepo(reference); + }); + EXPECT_CALL(*m_repo, getModuleList(reference)) + .WillOnce(Return(std::vector{ "binary", "develop" })); + }; + if (withRuntime) { + expectDependency(*runtime); + api::types::v1::RepositoryCacheLayersItem runtimeItem; + runtimeItem.info.base = baseEntry.requested; + EXPECT_CALL(*m_repo, getLayerItem(*runtime, "binary")) + .WillOnce(Return(runtimeItem)); + } + expectDependency(*base); + EXPECT_CALL(*m_repo, getLayerItem(*base, _)).Times(0); + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)).Times(0); + EXPECT_CALL(*m_repo, pull(_, _, _)).Times(0); + EXPECT_CALL(*m_repo, mergeModules()).WillOnce(Return(utils::error::Result{})); + + auto result = builder.buildStagePullDependency(); + ASSERT_TRUE(result) << result.error().message(); + auto written = + builder::readProjectLock(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(written); + auto writtenBase = builder::lockedDependency(*written, "base"); + ASSERT_TRUE(writtenBase); + EXPECT_EQ(writtenBase->resolved.reference, lockedReference(base, false)); + if (withRuntime) { + auto writtenRuntime = builder::lockedDependency(*written, "runtime"); + ASSERT_TRUE(writtenRuntime); + EXPECT_EQ(writtenRuntime->resolved.reference, lockedReference(runtime, false)); + } + EXPECT_EQ(builder.project->package.architecture, "arm64"); + EXPECT_EQ(builder.project->base, project.base); + EXPECT_EQ(builder.project->runtime, project.runtime); + if (before) { + auto after = utils::readFile(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(after); + EXPECT_EQ(*after, *before); + } + testing::Mock::VerifyAndClearExpectations(m_repo.get()); + } + } +} + +TEST_F(PullDependencyTest, CrossBuildKeepsLocalFallbackAfterDownloadFailure) +{ + LINGLONG_TRACE("use a local host dependency after a cross build download fails"); + + auto local = package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); + auto remote = package::Reference::parse("main:org.deepin.base/24.0.0.0/" + currentArchitecture); + ASSERT_TRUE(local); + ASSERT_TRUE(remote); + auto project = builderProject(); + project.package.architecture = "arm64"; + project.base = "org.deepin.base"; + builder::Builder builder(project, + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.projectYamlFile = (m_tmpDir.path() / "cross-fallback.yaml").string(); + + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(*local)); + EXPECT_CALL(*m_repo, latestRemoteReference(_)).WillOnce(Return(refWithRepo(*remote))); + EXPECT_CALL(*m_repo, getLayerDir(*remote, "binary")) + .WillOnce(Return(LINGLONG_ERR("not installed"))) + .WillOnce(Return(LINGLONG_ERR("not installed"))); + EXPECT_CALL(*m_repo, pull(_, _, "binary")).WillOnce(Return(LINGLONG_ERR("download failed"))); + expectCachedModule(*local, "binary"); + expectCachedModule(*local, "develop"); + EXPECT_CALL(*m_repo, getLayerDir(*remote, "develop")).Times(0); + EXPECT_CALL(*m_repo, getLayerItem(_, _)).Times(0); + EXPECT_CALL(*m_repo, getModuleList(*local)) + .WillOnce(Return(std::vector{ "binary", "develop" })); + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)).Times(0); + EXPECT_CALL(*m_repo, mergeModules()).WillOnce(Return(utils::error::Result{})); + + auto result = builder.buildStagePullDependency(); + ASSERT_TRUE(result) << result.error().message(); + auto written = builder::readProjectLock(builder::projectLockPath(builder.projectYamlFile)); + ASSERT_TRUE(written); + auto baseEntry = builder::lockedDependency(*written, "base"); + ASSERT_TRUE(baseEntry); + EXPECT_EQ(baseEntry->resolved.reference, lockedReference(local, false)); + EXPECT_EQ(builder.project->package.architecture, "arm64"); +} + TEST_F(PullDependencyTest, clearDependency_useRemoteFalse_returns_local_ref) { - auto ref = package::Reference::parse("main:org.example.test/1.0.0.0/x86_64"); + auto ref = package::Reference::parse("main:org.example.test/1.0.0.0/" + currentArchitecture); ASSERT_TRUE(ref.has_value()) << ref.error().message(); EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(std::move(*ref))); auto result = - builder::detail::clearDependency("org.example.test/1.0.0.0/x86_64", *m_repo, false); + builder::detail::clearDependency("org.example.test/1.0.0.0/" + currentArchitecture, + *m_repo, + false); ASSERT_TRUE(result.has_value()) << result.error().message(); EXPECT_FALSE(result->first); ASSERT_TRUE(result->second); - EXPECT_EQ(result->second->toString(), "main:org.example.test/1.0.0.0/x86_64"); + EXPECT_EQ(result->second->toString(), "main:org.example.test/1.0.0.0/" + currentArchitecture); } TEST_F(PullDependencyTest, clearDependency_useRemoteFalse_not_found_returns_error) @@ -160,14 +1917,18 @@ TEST_F(PullDependencyTest, clearDependency_useRemoteFalse_not_found_returns_erro EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(LINGLONG_ERR("not found"))); auto result = - builder::detail::clearDependency("org.example.missing/1.0.0.0/x86_64", *m_repo, false); + builder::detail::clearDependency("org.example.missing/1.0.0.0/" + currentArchitecture, + *m_repo, + false); EXPECT_FALSE(result.has_value()); } TEST_F(PullDependencyTest, clearDependency_returns_local_variant) { - auto localRef = package::Reference::parse("main:org.example.test/3.0.0.0/x86_64"); - auto remoteRef = package::Reference::parse("main:org.example.test/2.0.0.0/x86_64"); + auto localRef = + package::Reference::parse("main:org.example.test/3.0.0.0/" + currentArchitecture); + auto remoteRef = + package::Reference::parse("main:org.example.test/2.0.0.0/" + currentArchitecture); ASSERT_TRUE(localRef.has_value()); ASSERT_TRUE(remoteRef.has_value()); @@ -176,7 +1937,9 @@ TEST_F(PullDependencyTest, clearDependency_returns_local_variant) .WillOnce(Return(refWithRepo(std::move(*remoteRef)))); auto result = - builder::detail::clearDependency("org.example.test/2.0.0.0/x86_64", *m_repo, true); + builder::detail::clearDependency("org.example.test/2.0.0.0/" + currentArchitecture, + *m_repo, + true); ASSERT_TRUE(result.has_value()) << result.error().message(); EXPECT_FALSE(result->first); ASSERT_TRUE(result->second); @@ -185,14 +1948,17 @@ TEST_F(PullDependencyTest, clearDependency_returns_local_variant) TEST_F(PullDependencyTest, clearDependency_exact_version_returns_local_variant) { - auto localRef = package::Reference::parse("main:org.example.test/1.0.0.0/x86_64"); + auto localRef = + package::Reference::parse("main:org.example.test/1.0.0.0/" + currentArchitecture); ASSERT_TRUE(localRef.has_value()); EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(std::move(*localRef))); EXPECT_CALL(*m_repo, latestRemoteReference(_)).Times(0); auto result = - builder::detail::clearDependency("org.example.test/1.0.0.0/x86_64", *m_repo, true); + builder::detail::clearDependency("org.example.test/1.0.0.0/" + currentArchitecture, + *m_repo, + true); ASSERT_TRUE(result.has_value()) << result.error().message(); ASSERT_TRUE(result->second); EXPECT_FALSE(result->first); @@ -203,8 +1969,10 @@ TEST_F(PullDependencyTest, clearDependency_module_missing_returns_remote_variant { LINGLONG_TRACE("clearDependency_module_missing_returns_remote_variant"); - auto localRef = package::Reference::parse("main:org.example.test/3.0.0.0/x86_64"); - auto remoteRef = package::Reference::parse("main:org.example.test/2.0.0.0/x86_64"); + auto localRef = + package::Reference::parse("main:org.example.test/3.0.0.0/" + currentArchitecture); + auto remoteRef = + package::Reference::parse("main:org.example.test/2.0.0.0/" + currentArchitecture); ASSERT_TRUE(localRef.has_value()); ASSERT_TRUE(remoteRef.has_value()); @@ -214,7 +1982,10 @@ TEST_F(PullDependencyTest, clearDependency_module_missing_returns_remote_variant .WillOnce(Return(refWithRepo(std::move(*remoteRef), "custom"))); auto result = - builder::detail::clearDependency("org.example.test/1.0.0.0/x86_64", *m_repo, true, "develop"); + builder::detail::clearDependency("org.example.test/1.0.0.0/" + currentArchitecture, + *m_repo, + true, + "develop"); ASSERT_TRUE(result.has_value()) << result.error().message(); ASSERT_TRUE(result->first); EXPECT_FALSE(result->second); @@ -223,8 +1994,10 @@ TEST_F(PullDependencyTest, clearDependency_module_missing_returns_remote_variant TEST_F(PullDependencyTest, useRemoteTrue_prefers_remote_when_newer) { - auto localRef = package::Reference::parse("main:org.example.test/1.0.0.0/x86_64"); - auto remoteRef = package::Reference::parse("main:org.example.test/2.0.0.0/x86_64"); + auto localRef = + package::Reference::parse("main:org.example.test/1.0.0.0/" + currentArchitecture); + auto remoteRef = + package::Reference::parse("main:org.example.test/2.0.0.0/" + currentArchitecture); ASSERT_TRUE(localRef.has_value()); ASSERT_TRUE(remoteRef.has_value()); @@ -241,8 +2014,10 @@ TEST_F(PullDependencyTest, useRemoteTrue_prefers_remote_when_newer) TEST_F(PullDependencyTest, useRemoteTrue_prefers_local_when_newer) { - auto localRef = package::Reference::parse("main:org.example.test/3.0.0.0/x86_64"); - auto remoteRef = package::Reference::parse("main:org.example.test/2.0.0.0/x86_64"); + auto localRef = + package::Reference::parse("main:org.example.test/3.0.0.0/" + currentArchitecture); + auto remoteRef = + package::Reference::parse("main:org.example.test/2.0.0.0/" + currentArchitecture); ASSERT_TRUE(localRef.has_value()); ASSERT_TRUE(remoteRef.has_value()); @@ -252,8 +2027,9 @@ TEST_F(PullDependencyTest, useRemoteTrue_prefers_local_when_newer) EXPECT_CALL(*m_repo, getLayerDir(_, _)).Times(0); - auto result = - builder::detail::pullDependency("org.example.test/2.0.0.0/x86_64", *m_repo, "binary"); + auto result = builder::detail::pullDependency("org.example.test/2.0.0.0/" + currentArchitecture, + *m_repo, + "binary"); ASSERT_TRUE(result.has_value()) << result.error().message(); EXPECT_EQ(result->version.toString(), "3.0.0.0"); } @@ -262,8 +2038,10 @@ TEST_F(PullDependencyTest, useRemoteTrue_falls_back_to_local_when_remote_pull_fa { LINGLONG_TRACE("useRemoteTrue_falls_back_to_local_when_remote_pull_fails"); - auto localRef = package::Reference::parse("main:org.example.test/1.0.0.0/x86_64"); - auto remoteRef = package::Reference::parse("main:org.example.test/2.0.0.0/x86_64"); + auto localRef = + package::Reference::parse("main:org.example.test/1.0.0.0/" + currentArchitecture); + auto remoteRef = + package::Reference::parse("main:org.example.test/2.0.0.0/" + currentArchitecture); ASSERT_TRUE(localRef.has_value()); ASSERT_TRUE(remoteRef.has_value()); @@ -281,8 +2059,10 @@ TEST_F(PullDependencyTest, useRemoteTrue_falls_back_to_local_when_remote_pull_fa TEST_F(PullDependencyTest, useRemoteTrue_equal_version_prefers_local) { - auto localRef = package::Reference::parse("main:org.example.test/2.0.0.0/x86_64"); - auto remoteRef = package::Reference::parse("main:org.example.test/2.0.0.0/x86_64"); + auto localRef = + package::Reference::parse("main:org.example.test/2.0.0.0/" + currentArchitecture); + auto remoteRef = + package::Reference::parse("main:org.example.test/2.0.0.0/" + currentArchitecture); ASSERT_TRUE(localRef.has_value()); ASSERT_TRUE(remoteRef.has_value()); @@ -298,25 +2078,29 @@ TEST_F(PullDependencyTest, useRemoteTrue_equal_version_prefers_local) TEST_F(PullDependencyTest, exact_version_local_match_skips_remote_lookup) { - auto localRef = package::Reference::parse("main:org.example.test/1.0.0.0/x86_64"); + auto localRef = + package::Reference::parse("main:org.example.test/1.0.0.0/" + currentArchitecture); ASSERT_TRUE(localRef.has_value()); EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(std::move(*localRef))); EXPECT_CALL(*m_repo, latestRemoteReference(_)).Times(0); auto result = - builder::detail::clearDependency("org.example.test/1.0.0.0/x86_64", *m_repo, true); + builder::detail::clearDependency("org.example.test/1.0.0.0/" + currentArchitecture, + *m_repo, + true); ASSERT_TRUE(result.has_value()) << result.error().message(); EXPECT_FALSE(result->first); ASSERT_TRUE(result->second); - EXPECT_EQ(result->second->toString(), "main:org.example.test/1.0.0.0/x86_64"); + EXPECT_EQ(result->second->toString(), "main:org.example.test/1.0.0.0/" + currentArchitecture); } TEST_F(PullDependencyTest, useRemoteTrue_only_local_exists) { LINGLONG_TRACE("useRemoteTrue_only_local_exists"); - auto localRef = package::Reference::parse("main:org.example.test/1.0.0.0/x86_64"); + auto localRef = + package::Reference::parse("main:org.example.test/1.0.0.0/" + currentArchitecture); ASSERT_TRUE(localRef.has_value()); EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(std::move(*localRef))); @@ -326,14 +2110,15 @@ TEST_F(PullDependencyTest, useRemoteTrue_only_local_exists) auto result = builder::detail::pullDependency("org.example.test", *m_repo, "binary"); ASSERT_TRUE(result.has_value()) << result.error().message(); - EXPECT_EQ(result->toString(), "main:org.example.test/1.0.0.0/x86_64"); + EXPECT_EQ(result->toString(), "main:org.example.test/1.0.0.0/" + currentArchitecture); } TEST_F(PullDependencyTest, useRemoteTrue_only_remote_exists) { LINGLONG_TRACE("useRemoteTrue_only_remote_exists"); - auto remoteRef = package::Reference::parse("main:org.example.test/1.0.0.0/x86_64"); + auto remoteRef = + package::Reference::parse("main:org.example.test/1.0.0.0/" + currentArchitecture); ASSERT_TRUE(remoteRef.has_value()); EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(LINGLONG_ERR("not found"))); @@ -342,8 +2127,9 @@ TEST_F(PullDependencyTest, useRemoteTrue_only_remote_exists) EXPECT_CALL(*m_repo, getLayerDir(_, _)).WillOnce(Return(layerCached())); - auto result = - builder::detail::pullDependency("org.example.test/1.0.0.0/x86_64", *m_repo, "binary"); + auto result = builder::detail::pullDependency("org.example.test/1.0.0.0/" + currentArchitecture, + *m_repo, + "binary"); ASSERT_TRUE(result.has_value()) << result.error().message(); } @@ -355,7 +2141,9 @@ TEST_F(PullDependencyTest, useRemoteTrue_neither_exists_returns_error) EXPECT_CALL(*m_repo, latestRemoteReference(_)).WillOnce(Return(LINGLONG_ERR("not found"))); auto result = - builder::detail::pullDependency("org.example.missing/1.0.0.0/x86_64", *m_repo, "binary"); + builder::detail::pullDependency("org.example.missing/1.0.0.0/" + currentArchitecture, + *m_repo, + "binary"); EXPECT_FALSE(result.has_value()); } @@ -367,7 +2155,7 @@ TEST_F(PullDependencyTest, invalid_fuzzy_ref_returns_error) TEST_F(PullDependencyTest, pullResolvedRef_cached_module_returns_ok) { - auto ref = package::Reference::parse("main:org.example.test/1.0.0.0/x86_64"); + auto ref = package::Reference::parse("main:org.example.test/1.0.0.0/" + currentArchitecture); ASSERT_TRUE(ref.has_value()) << ref.error().message(); EXPECT_CALL(*m_repo, getLayerDir(_, _)).WillOnce(Return(layerCached())); @@ -380,7 +2168,7 @@ TEST_F(PullDependencyTest, pullResolvedRef_not_cached_triggers_pull) { LINGLONG_TRACE("pullResolvedRef_not_cached_triggers_pull"); - auto ref = package::Reference::parse("main:org.example.test/1.0.0.0/x86_64"); + auto ref = package::Reference::parse("main:org.example.test/1.0.0.0/" + currentArchitecture); ASSERT_TRUE(ref.has_value()) << ref.error().message(); EXPECT_CALL(*m_repo, getLayerDir(_, _)).WillOnce(Return(LINGLONG_ERR("not found"))); @@ -401,7 +2189,8 @@ TEST_F(PullDependencyTest, buildStagePullDependency_continues_when_local_develop { LINGLONG_TRACE("buildStagePullDependency_continues_when_local_develop_pull_fails"); - auto baseRef = package::Reference::parse("main:org.deepin.base/23.0.0.0/x86_64"); + auto baseRef = + package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); ASSERT_TRUE(baseRef.has_value()); builder::Builder builder(builderProject(), @@ -409,39 +2198,49 @@ TEST_F(PullDependencyTest, buildStagePullDependency_continues_when_local_develop *m_repo, containerBuilder(), builderConfig()); + builder.setBuildOptions(builder::BuilderBuildOptions{ .skipFetchSource = true }); + builder.projectYamlFile = (m_tmpDir.path() / "linglong.yaml").string(); - { - InSequence seq; - EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(*baseRef)); - EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(*baseRef)); - } - EXPECT_CALL(*m_repo, latestRemoteReference(_)).WillOnce(Return(LINGLONG_ERR("not found"))); - EXPECT_CALL(*m_repo, getLayerDir(_, "develop")) - .WillOnce(Return(LINGLONG_ERR("not found"))) + expectFreshBase(*baseRef); + EXPECT_CALL(*m_repo, getLayerDir(*baseRef, "develop")) .WillOnce(Return(LINGLONG_ERR("not found"))); - EXPECT_CALL(*m_repo, mergeModules()).WillOnce(Return(utils::error::Result{})); + EXPECT_CALL(*m_repo, matchRemoteByPriority(_, _)).WillOnce(Return(LINGLONG_ERR("not found"))); + EXPECT_CALL(*m_repo, pull(_, _, _)).Times(0); auto result = builder.buildStagePullDependency(); - EXPECT_TRUE(result.has_value()) << result.error().message(); + ASSERT_TRUE(result.has_value()) << result.error().message(); + ASSERT_TRUE(builder.project->base); + EXPECT_EQ(builder.project->base, builderProject().base); + ASSERT_TRUE(builder.projectLock); + EXPECT_TRUE(builder.projectLock->sources.empty()); + ASSERT_EQ(builder.projectLock->dependencies.size(), 1); + EXPECT_EQ(builder.projectLock->dependencies.front().resolved.reference, + lockedReference(baseRef)); + EXPECT_FALSE(builder.pendingSources); + EXPECT_TRUE(std::filesystem::exists(builder::projectLockPath(builder.projectYamlFile))); } TEST_F(PullDependencyTest, buildStagePullDependency_resolves_base_from_runtime) { LINGLONG_TRACE("buildStagePullDependency_resolves_base_from_runtime"); - auto runtimeRef = package::Reference::parse("main:org.deepin.runtime/23.0.0.0/x86_64"); - auto baseRef = package::Reference::parse("main:org.deepin.base/23.0.0.0/x86_64"); + auto runtimeRef = + package::Reference::parse("main:org.deepin.runtime/23.0.0.0/" + currentArchitecture); + auto baseRef = + package::Reference::parse("main:org.deepin.base/23.0.0.0/" + currentArchitecture); ASSERT_TRUE(runtimeRef.has_value()); ASSERT_TRUE(baseRef.has_value()); - auto project = builderProject("org.deepin.runtime/23.0.0.0/x86_64"); + auto project = builderProject("org.deepin.runtime/23.0.0.0/" + currentArchitecture); project.base.reset(); builder::Builder builder(std::move(project), m_tmpDir.path(), *m_repo, containerBuilder(), builderConfig()); - builder.setBuildOptions(builder::BuilderBuildOptions{ .skipPullDepend = true }); + builder.projectYamlFile = (m_tmpDir.path() / "linglong.yaml").string(); + builder.setBuildOptions( + builder::BuilderBuildOptions{ .skipFetchSource = true, .skipPullDepend = true }); { InSequence seq; @@ -456,76 +2255,119 @@ TEST_F(PullDependencyTest, buildStagePullDependency_resolves_base_from_runtime) }); api::types::v1::RepositoryCacheLayersItem runtimeItem; - runtimeItem.info.base = "org.deepin.base/23.0.0.0/x86_64"; + runtimeItem.info.base = "org.deepin.base/23.0.0.0/" + currentArchitecture; EXPECT_CALL(*m_repo, getLayerItem(*runtimeRef, "binary")) - .WillOnce(Return(std::move(runtimeItem))); + .Times(1) + .WillRepeatedly([runtimeItem](const package::Reference &, const std::string &) + -> utils::error::Result { + return runtimeItem; + }); + EXPECT_CALL(*m_repo, getLayerDir(_, "binary")) + .Times(2) + .WillRepeatedly([](const package::Reference &, + const std::string &) -> utils::error::Result { + return layerCached(); + }); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, false)).Times(0); + EXPECT_CALL(*m_repo, getLayerItem(*baseRef, _)).Times(0); + EXPECT_CALL(*m_repo, getModuleList(*baseRef)) + .WillOnce(Return(std::vector{ "binary" })); + EXPECT_CALL(*m_repo, getModuleList(*runtimeRef)) + .WillOnce(Return(std::vector{ "binary" })); EXPECT_CALL(*m_repo, mergeModules()).WillOnce(Return(utils::error::Result{})); auto result = builder.buildStagePullDependency(); ASSERT_TRUE(result.has_value()) << result.error().message(); - ASSERT_TRUE(builder.project->base); - EXPECT_EQ(*builder.project->base, "org.deepin.base/23.0.0.0/x86_64"); + EXPECT_FALSE(builder.project->base); + ASSERT_TRUE(builder.project->runtime); + EXPECT_EQ(*builder.project->runtime, "org.deepin.runtime/23.0.0.0/" + currentArchitecture); + ASSERT_TRUE(builder.projectLock); + EXPECT_EQ(builder::lockedDependency(*builder.projectLock, "base")->resolved.reference, + lockedReference(baseRef)); + EXPECT_EQ(builder::lockedDependency(*builder.projectLock, "runtime")->resolved.reference, + lockedReference(runtimeRef)); } TEST_F(PullDependencyTest, buildStagePullDependency_rejects_incompatible_base_and_runtime) { LINGLONG_TRACE("buildStagePullDependency_rejects_incompatible_base_and_runtime"); - auto runtimeRef = package::Reference::parse("main:org.deepin.runtime/23.0.0.0/x86_64"); - auto configuredBaseRef = package::Reference::parse("main:org.example.base/23.0.0.0/x86_64"); - ASSERT_TRUE(runtimeRef.has_value()); - ASSERT_TRUE(configuredBaseRef.has_value()); - - auto project = builderProject("org.deepin.runtime/23.0.0.0/x86_64"); - project.base = "org.example.base/23.0.0/x86_64"; - builder::Builder builder(std::move(project), - m_tmpDir.path(), - *m_repo, - containerBuilder(), - builderConfig()); - builder.setBuildOptions(builder::BuilderBuildOptions{ .skipPullDepend = true }); - - { - InSequence seq; - EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(*runtimeRef)); - EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(*configuredBaseRef)); + for (const std::string suffix : { std::string{}, "/23.0.0.0/" + currentArchitecture }) { + SCOPED_TRACE(suffix); + auto runtimeRef = + package::Reference::parse("main:org.deepin.runtime/23.0.0.0/" + currentArchitecture); + auto configuredBaseRef = + package::Reference::parse("main:org.example.base/23.0.0.0/" + currentArchitecture); + ASSERT_TRUE(runtimeRef.has_value()); + ASSERT_TRUE(configuredBaseRef.has_value()); + + auto project = builderProject("org.deepin.runtime/23.0.0.0/" + currentArchitecture); + project.base = "org.example.base/23.0.0/" + currentArchitecture; + builder::Builder builder(std::move(project), + m_tmpDir.path(), + *m_repo, + containerBuilder(), + builderConfig()); + builder.setBuildOptions( + builder::BuilderBuildOptions{ .skipFetchSource = true, .skipPullDepend = true }); + + { + InSequence seq; + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(*runtimeRef)); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, true)).WillOnce(Return(*configuredBaseRef)); + } + EXPECT_CALL(*m_repo, getLayerDir(_, "develop")) + .Times(2) + .WillRepeatedly([](const package::Reference &, + const std::string &) -> utils::error::Result { + return layerCached(); + }); + + api::types::v1::RepositoryCacheLayersItem runtimeItem; + runtimeItem.info.base = "org.deepin.base" + suffix; + EXPECT_CALL(*m_repo, getLayerItem(*runtimeRef, "binary")) + .Times(1) + .WillRepeatedly([runtimeItem](const package::Reference &, const std::string &) + -> utils::error::Result { + return runtimeItem; + }); + EXPECT_CALL(*m_repo, getLayerDir(_, "binary")) + .Times(2) + .WillRepeatedly([](const package::Reference &, + const std::string &) -> utils::error::Result { + return layerCached(); + }); + EXPECT_CALL(*m_repo, mergeModules()).Times(0); + + auto result = builder.buildStagePullDependency(); + ASSERT_FALSE(result.has_value()); + EXPECT_THAT(result.error().message(), + testing::HasSubstr("base is not compatible with runtime")); + testing::Mock::VerifyAndClearExpectations(m_repo.get()); } - EXPECT_CALL(*m_repo, getLayerDir(_, "develop")) - .Times(2) - .WillRepeatedly([](const package::Reference &, - const std::string &) -> utils::error::Result { - return layerCached(); - }); - - api::types::v1::RepositoryCacheLayersItem runtimeItem; - runtimeItem.info.base = "org.deepin.base/23.0.0.0/x86_64"; - EXPECT_CALL(*m_repo, getLayerItem(*runtimeRef, "binary")) - .WillOnce(Return(std::move(runtimeItem))); - EXPECT_CALL(*m_repo, mergeModules()).Times(0); - - auto result = builder.buildStagePullDependency(); - ASSERT_FALSE(result.has_value()); - EXPECT_THAT(result.error().message(), - testing::HasSubstr("base is not compatible with runtime")); } TEST_F(PullDependencyTest, buildStagePullDependency_accepts_semantically_compatible_base) { LINGLONG_TRACE("buildStagePullDependency_accepts_semantically_compatible_base"); - auto runtimeRef = package::Reference::parse("main:org.deepin.runtime/23.0.0.0/x86_64"); - auto baseRef = package::Reference::parse("main:org.deepin.base/23.0.0.1/x86_64"); + auto runtimeRef = + package::Reference::parse("main:org.deepin.runtime/23.0.0.0/" + currentArchitecture); + auto baseRef = + package::Reference::parse("main:org.deepin.base/23.0.0.1/" + currentArchitecture); ASSERT_TRUE(runtimeRef.has_value()); ASSERT_TRUE(baseRef.has_value()); - auto project = builderProject("org.deepin.runtime/23.0.0/x86_64"); - project.base = "org.deepin.base/23.0.0/x86_64"; + auto project = builderProject("org.deepin.runtime/23.0.0/" + currentArchitecture); + project.base = "org.deepin.base/23.0.0/" + currentArchitecture; builder::Builder builder(std::move(project), m_tmpDir.path(), *m_repo, containerBuilder(), builderConfig()); - builder.setBuildOptions(builder::BuilderBuildOptions{ .skipPullDepend = true }); + builder.projectYamlFile = (m_tmpDir.path() / "linglong.yaml").string(); + builder.setBuildOptions( + builder::BuilderBuildOptions{ .skipFetchSource = true, .skipPullDepend = true }); { InSequence seq; @@ -540,13 +2382,38 @@ TEST_F(PullDependencyTest, buildStagePullDependency_accepts_semantically_compati }); api::types::v1::RepositoryCacheLayersItem runtimeItem; - runtimeItem.info.base = "org.deepin.base/23.0.0/x86_64"; + runtimeItem.info.base = "org.deepin.base/23.0.0/" + currentArchitecture; EXPECT_CALL(*m_repo, getLayerItem(*runtimeRef, "binary")) - .WillOnce(Return(std::move(runtimeItem))); + .Times(1) + .WillRepeatedly([runtimeItem](const package::Reference &, const std::string &) + -> utils::error::Result { + return runtimeItem; + }); + EXPECT_CALL(*m_repo, getLayerDir(_, "binary")) + .Times(2) + .WillRepeatedly([](const package::Reference &, + const std::string &) -> utils::error::Result { + return layerCached(); + }); + EXPECT_CALL(*m_repo, clearReferenceLocal(_, false)).Times(0); + EXPECT_CALL(*m_repo, getLayerItem(*baseRef, _)).Times(0); + EXPECT_CALL(*m_repo, getModuleList(*baseRef)) + .WillOnce(Return(std::vector{ "binary" })); + EXPECT_CALL(*m_repo, getModuleList(*runtimeRef)) + .WillOnce(Return(std::vector{ "binary" })); EXPECT_CALL(*m_repo, mergeModules()).WillOnce(Return(utils::error::Result{})); auto result = builder.buildStagePullDependency(); - EXPECT_TRUE(result.has_value()) << result.error().message(); + ASSERT_TRUE(result.has_value()) << result.error().message(); + ASSERT_TRUE(builder.project->base); + ASSERT_TRUE(builder.project->runtime); + EXPECT_EQ(*builder.project->base, "org.deepin.base/23.0.0/" + currentArchitecture); + EXPECT_EQ(*builder.project->runtime, "org.deepin.runtime/23.0.0/" + currentArchitecture); + ASSERT_TRUE(builder.projectLock); + EXPECT_EQ(builder::lockedDependency(*builder.projectLock, "base")->resolved.reference, + lockedReference(baseRef)); + EXPECT_EQ(builder::lockedDependency(*builder.projectLock, "runtime")->resolved.reference, + lockedReference(runtimeRef)); } } // namespace diff --git a/libs/linglong/tests/ll-tests/src/linglong/builder/source_fetcher_test.cpp b/libs/linglong/tests/ll-tests/src/linglong/builder/source_fetcher_test.cpp index c4afd6a67..b210b4914 100644 --- a/libs/linglong/tests/ll-tests/src/linglong/builder/source_fetcher_test.cpp +++ b/libs/linglong/tests/ll-tests/src/linglong/builder/source_fetcher_test.cpp @@ -1,136 +1,1007 @@ -// SPDX-FileCopyrightText: 2024 UnionTech Software Technology Co., Ltd. +// SPDX-FileCopyrightText: 2024 - 2026 UnionTech Software Technology Co., Ltd. // // SPDX-License-Identifier: LGPL-3.0-or-later -#include #include +#include "common/tempdir.h" #include "linglong/builder/source_fetcher.h" +#include "linglong/utils/cmd.h" +#include "linglong/utils/env.h" #include "linglong/utils/error/error.h" +#include "linglong/utils/filelock.h" +#include "linglong/utils/unique_fd.h" -#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include using namespace linglong; namespace linglong::builder { -using ::testing::_; -using ::testing::ElementsAre; -using ::testing::EndsWith; -using ::testing::Return; -using ::testing::ReturnRef; +namespace { -class MockCommand : public linglong::utils::Cmd +class FetchProcess { public: - MockCommand(std::string command) - : Cmd(command) + explicit FetchProcess(const std::function &fetch) + { + std::array completion; + if (::pipe2(completion.data(), O_CLOEXEC) != 0) { + throw std::runtime_error("failed to create fetch completion pipe"); + } + completed.reset(completion[0]); + utils::fd::UniqueFd notify(completion[1]); + + pid = ::fork(); + if (pid < 0) { + throw std::runtime_error("failed to fork source fetch process"); + } + if (pid == 0) { + ::setpgid(0, 0); + completed.reset(); + const auto result = fetch(); + const char done = '1'; + if (::write(notify.get(), &done, 1) != 1) { + ::_exit(1); + } + ::_exit(result); + } + ::setpgid(pid, pid); + } + + ~FetchProcess() + { + if (pid > 0) { + // Also stop helper commands if a test fails while a fetch is paused. + ::kill(-pid, SIGKILL); + while (::waitpid(pid, nullptr, 0) < 0 && errno == EINTR) { } + } + } + + int wait() { + pollfd event{ completed.get(), POLLIN, 0 }; + if (::poll(&event, 1, 10000) != 1 || !(event.revents & POLLIN)) { + return -1; + } + + int status = 0; + pid_t waited; + do { + waited = ::waitpid(pid, &status, 0); + } while (waited < 0 && errno == EINTR); + if (waited != pid) { + return -1; + } + pid = -1; + return WIFEXITED(status) ? WEXITSTATUS(status) : -1; } - MOCK_METHOD(utils::error::Result, - exec, - (const std::vector &args), - (noexcept, override)); - MOCK_METHOD(bool, exists, (), (noexcept, override)); - MOCK_METHOD(Cmd &, - setEnv, - (const std::string &name, const std::string &value), - (noexcept, override)); +private: + pid_t pid = -1; + utils::fd::UniqueFd completed; }; +class FetchGate +{ +public: + explicit FetchGate(const std::filesystem::path &directory) + : readyPath(directory / "ready") + , resumePath(directory / "resume") + { + std::filesystem::create_directories(directory); + if (::mkfifo(readyPath.c_str(), 0600) != 0 || ::mkfifo(resumePath.c_str(), 0600) != 0) { + throw std::runtime_error("failed to create source fetch gate"); + } + ready.reset(::open(readyPath.c_str(), O_RDWR | O_NONBLOCK | O_CLOEXEC)); + resume.reset(::open(resumePath.c_str(), O_RDWR | O_NONBLOCK | O_CLOEXEC)); + if (!ready || !resume) { + throw std::runtime_error("failed to open source fetch gate"); + } + } + + bool wait() + { + pollfd event{ ready.get(), POLLIN, 0 }; + return ::poll(&event, 1, 10000) == 1 && (event.revents & POLLIN); + } + + bool release() { return ::write(resume.get(), "1\n", 2) == 2; } + + const std::filesystem::path readyPath; + const std::filesystem::path resumePath; + +private: + utils::fd::UniqueFd ready; + utils::fd::UniqueFd resume; +}; + +} // namespace + +auto prepareSourceDirectories(const std::filesystem::path &cache, + std::initializer_list destinations) + -> utils::error::Result +{ + LINGLONG_TRACE("prepare test source directories"); + + auto ensured = SourceCacheLayout(cache).ensure(); + if (!ensured) { + return LINGLONG_ERR(ensured); + } + + for (const auto &destination : destinations) { + std::error_code ec; + std::filesystem::create_directories(destination, ec); + if (ec) { + return LINGLONG_ERR("failed to create test source destination: " + ec.message()); + } + } + + return LINGLONG_OK; +} + class SourceFetcherTest : public ::testing::Test { protected: void SetUp() override { } }; -// 测试获取文件类型源码的正常流程 -// 场景：提供有效的文件类型source配置，包括url、digest和name -// 预期：成功获取源码并返回ok -TEST_F(SourceFetcherTest, FetchFileSource) +auto lockedRequest(api::types::v1::BuilderProjectSource source, std::string hash) + -> LockedSourceRequest +{ + api::types::v1::BuilderSourceLockResolved resolved; + resolved.hash = std::move(hash); + return LockedSourceRequest{ std::move(source), std::move(resolved) }; +} + +std::string commandOutputLine(std::string output) +{ + const auto end = output.find_first_of("\r\n"); + if (end != std::string::npos) { + output.erase(end); + } + return output; +} + +class SourceCacheProcessTest : public ::testing::Test +{ +protected: + void SetUp() override + { + ASSERT_TRUE(prepareSourceDirectories(cache, { firstDestination, secondDestination })); + std::ofstream(artifact) << "source contents"; + auto digest = fileHash(artifact); + ASSERT_TRUE(digest) << digest.error().message(); + hash = *digest; + source.kind = "file"; + source.url = artifact.string(); + source.digest = hash; + source.name = "source"; + + auto tarPath = utils::Cmd("sh").exec({ "-c", "command -v tar" }); + if (tarPath) { + realTar = std::filesystem::absolute(commandOutputLine(*tarPath)); + } + const auto commands = temporary.path() / "bin"; + std::filesystem::create_directories(commands); + std::ofstream(commands / "wget") << R"(#!/bin/sh +set -eu +/bin/cp -- "$1" "$3" +if [ "$LINGLONG_TEST_FETCH_GATE" = download ]; then + printf 'ready\n' > "$LINGLONG_TEST_FETCH_READY" + read -r resume < "$LINGLONG_TEST_FETCH_RESUME" +fi +)"; + std::ofstream(commands / "tar") << R"(#!/bin/sh +set -eu +printf 'ready\n' > "$LINGLONG_TEST_FETCH_READY" +read -r resume < "$LINGLONG_TEST_FETCH_RESUME" +if [ "$LINGLONG_TEST_FETCH_FAIL" = 1 ]; then + exit 1 +fi +exec "$LINGLONG_TEST_REAL_TAR" "$@" +)"; + for (const auto &command : { "wget", "tar" }) { + std::filesystem::permissions(commands / command, std::filesystem::perms::owner_all); + } + + const auto *originalPath = std::getenv("PATH"); + commandPath = std::make_unique( + "PATH", + commands.string() + ":" + (originalPath ? originalPath : "")); + } + + int fetch(const FetchGate &gate, + const std::filesystem::path &destination, + bool offline, + bool failMaterialization = false, + bool locked = true) + { + utils::EnvironmentVariableGuard ready("LINGLONG_TEST_FETCH_READY", gate.readyPath.string()); + utils::EnvironmentVariableGuard resume("LINGLONG_TEST_FETCH_RESUME", + gate.resumePath.string()); + utils::EnvironmentVariableGuard tar("LINGLONG_TEST_REAL_TAR", realTar.string()); + utils::EnvironmentVariableGuard failure("LINGLONG_TEST_FETCH_FAIL", + failMaterialization ? "1" : "0"); + utils::EnvironmentVariableGuard phase("LINGLONG_TEST_FETCH_GATE", + source.kind == "archive" ? "materialize" + : "download"); + SourceFetchRequest request = GenerateSourceRequest{ source }; + if (locked) { + request = lockedRequest(source, hash); + } + SourceFetcher fetcher(std::move(request), SourceCacheLayout(cache)); + auto result = fetcher.fetch(destination, offline); + return result.has_value() == !failMaterialization && (!result || result->hash == hash) ? 0 + : 1; + } + + TempDir temporary; + const std::filesystem::path cache = temporary.path() / "cache"; + const std::filesystem::path artifact = temporary.path() / "artifact"; + const std::filesystem::path firstDestination = temporary.path() / "first"; + const std::filesystem::path secondDestination = temporary.path() / "second"; + const SourceCacheLayout layout{ cache }; + api::types::v1::BuilderProjectSource source; + std::string hash; + std::filesystem::path realTar; + std::unique_ptr commandPath; +}; + +TEST_F(SourceCacheProcessTest, RechecksCacheAfterConcurrentDownloadsAndRepairsAcrossProcesses) +{ + const auto cached = layout.object(source.kind, hash); + std::filesystem::create_directories(cached.parent_path()); + std::ofstream(cached) << "corrupted cache"; + + FetchGate firstGate(temporary.path() / "first-gate"); + FetchGate secondGate(temporary.path() / "second-gate"); + FetchProcess first([&]() { + return fetch(firstGate, firstDestination, false); + }); + FetchProcess second([&]() { + return fetch(secondGate, secondDestination, false); + }); + ASSERT_TRUE(firstGate.wait()); + ASSERT_TRUE(secondGate.wait()); + + // Both downloads have finished, but neither holds the cache lock during acquisition. + { + auto probe = utils::filelock::FileLock::create(layout.lockFile(cached), + utils::filelock::LockType::Write); + ASSERT_TRUE(probe) << probe.error().message(); + auto locked = probe->tryLock(utils::filelock::LockType::Write); + ASSERT_TRUE(locked) << locked.error().message(); + ASSERT_TRUE(*locked); + } + + ASSERT_TRUE(secondGate.release()); + ASSERT_EQ(second.wait(), 0); + ASSERT_EQ(*fileHash(cached), hash); + struct stat published{}; + ASSERT_EQ(::stat(cached.c_str(), &published), 0); + + // The first fetch observed corruption before downloading. It must now reuse the repair. + ASSERT_TRUE(firstGate.release()); + ASSERT_EQ(first.wait(), 0); + struct stat reused{}; + ASSERT_EQ(::stat(cached.c_str(), &reused), 0); + EXPECT_EQ(reused.st_ino, published.st_ino); + EXPECT_EQ(*fileHash(cached), hash); + EXPECT_EQ(*fileHash(firstDestination / "source"), hash); + EXPECT_EQ(*fileHash(secondDestination / "source"), hash); +} + +class SourceCacheMaterializationTest + : public SourceCacheProcessTest, + public ::testing::WithParamInterface> +{ +}; + +TEST_P(SourceCacheMaterializationTest, HoldsEntryLockThroughMaterializationAndReleasesOnReturn) +{ + if (realTar.empty()) { + GTEST_SKIP() << "tar is not available"; + } + const auto [locked, offline, failMaterialization] = GetParam(); + const auto content = temporary.path() / "content"; + std::filesystem::create_directories(content); + std::ofstream(content / "source.txt") << "archive contents"; + auto archived = + utils::Cmd(realTar.string()).exec({ "-cf", artifact.string(), "-C", content.string(), "." }); + ASSERT_TRUE(archived) << archived.error().message(); + auto digest = fileHash(artifact); + ASSERT_TRUE(digest) << digest.error().message(); + hash = *digest; + source.kind = "archive"; + source.digest = hash; + const auto cached = layout.object(source.kind, hash); + if (offline) { + std::filesystem::create_directories(cached.parent_path()); + std::filesystem::copy_file(artifact, cached); + } + + FetchGate gate(temporary.path() / "materialize-gate"); + FetchProcess reader([&]() { + return fetch(gate, firstDestination, offline, failMaterialization, locked); + }); + ASSERT_TRUE(gate.wait()); + + // The archive helper is paused inside materialize; another process cannot replace the cache. + const auto lockPath = layout.lockFile(cached); + struct stat heldLock{}; + ASSERT_EQ(::stat(lockPath.c_str(), &heldLock), 0); + auto probe = utils::filelock::FileLock::create(lockPath, utils::filelock::LockType::Write); + ASSERT_TRUE(probe) << probe.error().message(); + auto busy = probe->tryLock(utils::filelock::LockType::Write); + ASSERT_TRUE(busy) << busy.error().message(); + EXPECT_FALSE(*busy); + + ASSERT_TRUE(gate.release()); + ASSERT_EQ(reader.wait(), 0); + auto available = probe->tryLock(utils::filelock::LockType::Write); + ASSERT_TRUE(available) << available.error().message(); + EXPECT_TRUE(*available); + struct stat releasedLock{}; + ASSERT_EQ(::stat(lockPath.c_str(), &releasedLock), 0); + EXPECT_EQ(releasedLock.st_ino, heldLock.st_ino); + EXPECT_EQ(std::filesystem::exists(firstDestination / "source/source.txt"), + !failMaterialization); + EXPECT_EQ(*fileHash(cached), hash); +} + +INSTANTIATE_TEST_SUITE_P(MaterializationResult, + SourceCacheMaterializationTest, + ::testing::Values(std::make_tuple(true, false, false), + std::make_tuple(true, false, true), + std::make_tuple(true, true, false), + std::make_tuple(true, true, true), + std::make_tuple(false, false, false), + std::make_tuple(false, false, true))); + +TEST(GitSourceHash, ExtractsSha1AndSha256CommitIds) +{ + const std::string sha1(40, 'a'); + const std::string sha256(64, 'B'); + EXPECT_EQ(gitCommitFromHash("git-commit:" + sha1), sha1); + EXPECT_EQ(gitCommitFromHash("git-commit:" + sha256), sha256); +} + +TEST(GitSourceHash, RejectsInvalidPrefixesAndCommitIds) +{ + EXPECT_FALSE(gitCommitFromHash("")); + EXPECT_FALSE(gitCommitFromHash(std::string(40, 'a'))); + EXPECT_FALSE(gitCommitFromHash("sha256:" + std::string(64, 'a'))); + EXPECT_FALSE(gitCommitFromHash("git-commit:main")); + EXPECT_FALSE(gitCommitFromHash("git-commit:" + std::string(39, 'a'))); + EXPECT_FALSE(gitCommitFromHash("git-commit:" + std::string(41, 'a'))); + EXPECT_FALSE(gitCommitFromHash("git-commit:" + std::string(40, 'g'))); +} + +TEST(SourceUrl, ExtractsFileName) +{ + EXPECT_EQ(fileNameFromUrl("https://example.com/releases/source.tar.gz"), "source.tar.gz"); + EXPECT_EQ(fileNameFromUrl("archive.tar.gz"), "archive.tar.gz"); +} + +TEST(SourceUrl, IgnoresQueryAndFragment) +{ + EXPECT_EQ(fileNameFromUrl("https://example.com/source.tar.gz?download=1#content"), + "source.tar.gz"); + EXPECT_EQ(fileNameFromUrl("file:///tmp/source.zip#content"), "source.zip"); +} + +TEST(SourceUrl, DecodesPercentEncodedFileName) +{ + EXPECT_EQ(fileNameFromUrl("https://example.com/source%20archive.tar.gz"), + "source archive.tar.gz"); +} + +TEST(SourceUrl, ReturnsEmptyNameForTrailingSlash) +{ + EXPECT_TRUE(fileNameFromUrl("https://example.com/releases/").empty()); +} + +TEST(SourceOutputName, PreservesDotGitSuffixForGitSource) +{ + api::types::v1::BuilderProjectSource source; + source.kind = "git"; + source.url = "https://example.com/project.git"; + + EXPECT_EQ(sourceOutputName(source), "project.git"); +} + +TEST(SourceOutputName, PreservesExplicitNameForGitSource) +{ + api::types::v1::BuilderProjectSource source; + source.kind = "git"; + source.url = "https://example.com/project.git"; + source.name = "custom.git"; + + EXPECT_EQ(sourceOutputName(source), "custom.git"); +} + +TEST(SourceOutputName, PreservesDotGitSuffixForFileSource) { api::types::v1::BuilderProjectSource source; source.kind = "file"; - source.url = "https://example.com/repo.git"; - source.digest = "abcdef123456"; - source.name = "repo"; + source.url = "https://example.com/project.git"; - QDir cacheDir("/tmp/cache"); - auto mockCmd = std::make_shared("mock"); - EXPECT_CALL(*mockCmd, - exec(ElementsAre(EndsWith("fetch-file-source"), - "/tmp/dest/" + source.name.value(), - source.url, - source.digest, - "/tmp/cache"))) - .WillOnce(Return("ok")); - SourceFetcher fetcher(source, cacheDir); - fetcher.setCommand(mockCmd); - auto ret = fetcher.fetch(QDir("/tmp/dest")); - EXPECT_TRUE(ret.has_value()) << ret.error().message(); -} - -// 测试获取git类型源码的正常流程 -// 场景：提供有效的git类型source配置，包括url、commit和name -// 预期：成功获取源码并返回ok -TEST_F(SourceFetcherTest, FetchGitSource) + EXPECT_EQ(sourceOutputName(source), "project.git"); +} + +TEST(SourceCache, PublishesGitDirectoryAtomicallyAndCopiesItToBuildDirectory) { + utils::Cmd git("git"); + if (!git.exists()) { + GTEST_SKIP() << "git is not available"; + } + + TempDir temporary; + const auto repository = temporary.path() / "repository"; + auto initialized = git.exec({ "init", repository.string() }); + ASSERT_TRUE(initialized.has_value()) << initialized.error().message(); + ASSERT_TRUE(git.exec({ "-C", repository.string(), "config", "user.name", "Test" })) + << "failed to configure git user name"; + ASSERT_TRUE(git.exec({ "-C", repository.string(), "config", "user.email", "test@example.com" })) + << "failed to configure git user email"; + std::ofstream(repository / "source.txt") << "cached content"; + ASSERT_TRUE(git.exec({ "-C", repository.string(), "add", "source.txt" })) + << "failed to stage test source"; + ASSERT_TRUE(git.exec({ "-C", repository.string(), "commit", "-m", "initial" })) + << "failed to commit test source"; + auto commit = git.exec({ "-C", repository.string(), "rev-parse", "HEAD" }); + ASSERT_TRUE(commit.has_value()) << commit.error().message(); + commit->erase(commit->find_first_of("\r\n")); + api::types::v1::BuilderProjectSource source; source.kind = "git"; - source.url = "https://example.com/repo.git"; - source.commit = "abcdef123456"; - source.name = "repo"; + source.url = repository.string(); + source.commit = *commit; + source.name = "source"; + const auto cache = temporary.path() / "cache"; + const auto firstDestination = temporary.path() / "first"; + const auto secondDestination = temporary.path() / "second"; + ASSERT_TRUE(prepareSourceDirectories(cache, + { firstDestination, + secondDestination, + temporary.path() / "offline", + temporary.path() / "dirty", + temporary.path() / "repaired-dirty", + temporary.path() / "repaired-object", + temporary.path() / "corrupt", + temporary.path() / "repaired" })); + + auto fetch = [&source, &cache](const std::filesystem::path &destination) { + SourceFetcher fetcher(GenerateSourceRequest{ source }, SourceCacheLayout(cache)); + return fetcher.fetch(destination); + }; + FetchProcess second([&]() { + auto result = fetch(secondDestination); + return result && result->hash == "git-commit:" + *commit ? 0 : 1; + }); + auto firstResult = fetch(firstDestination); + ASSERT_EQ(second.wait(), 0); + ASSERT_TRUE(firstResult.has_value()) << firstResult.error().message(); + EXPECT_EQ(firstResult->hash, "git-commit:" + *commit); + + const auto cached = SourceCacheLayout(cache).gitObject(*commit, true); + EXPECT_TRUE(std::filesystem::is_directory(cached)); + + struct stat cacheStatus{}; + struct stat buildStatus{}; + ASSERT_EQ(::stat((cached / "source.txt").c_str(), &cacheStatus), 0); + ASSERT_EQ(::stat((firstDestination / "source/source.txt").c_str(), &buildStatus), 0); + EXPECT_NE(cacheStatus.st_ino, buildStatus.st_ino); + EXPECT_TRUE(std::filesystem::is_directory(cached / ".git")); + EXPECT_TRUE(std::filesystem::is_directory(firstDestination / "source/.git")); + auto firstHead = + git.exec({ "-C", (firstDestination / "source").string(), "rev-parse", "--verify", "HEAD" }); + ASSERT_TRUE(firstHead.has_value()) << firstHead.error().message(); + EXPECT_TRUE(firstHead->rfind(*commit, 0) == 0); + + std::ofstream(firstDestination / "source/source.txt", std::ios::trunc) << "modified"; + EXPECT_EQ(*fileHash(cached / "source.txt"), *fileHash(repository / "source.txt")); + + SourceFetcher offlineFetcher(lockedRequest(source, firstResult->hash), + SourceCacheLayout(cache)); + auto offline = offlineFetcher.fetch(temporary.path() / "offline", true); + ASSERT_TRUE(offline.has_value()) << offline.error().message(); + EXPECT_TRUE(std::filesystem::is_directory(temporary.path() / "offline/source/.git")); + + std::ofstream(cached / "source.txt", std::ios::trunc) << "corrupted cached content"; + auto dirty = offlineFetcher.fetch(temporary.path() / "dirty", true); + ASSERT_TRUE(dirty.has_value()) << dirty.error().message(); + EXPECT_EQ(*fileHash(temporary.path() / "dirty/source/source.txt"), + *fileHash(repository / "source.txt")); + EXPECT_NE(*fileHash(cached / "source.txt"), *fileHash(repository / "source.txt")); + SourceFetcher repairDirty(lockedRequest(source, firstResult->hash), SourceCacheLayout(cache)); + auto repairedDirty = repairDirty.fetch(temporary.path() / "repaired-dirty"); + ASSERT_TRUE(repairedDirty.has_value()) << repairedDirty.error().message(); + + auto blob = git.exec({ "-C", cached.string(), "rev-parse", "HEAD:source.txt" }); + ASSERT_TRUE(blob.has_value()) << blob.error().message(); + *blob = commandOutputLine(std::move(*blob)); + const auto object = cached / ".git/objects" / blob->substr(0, 2) / blob->substr(2); + std::filesystem::create_directories(object.parent_path()); + if (std::filesystem::exists(object)) { + std::filesystem::permissions(object, + std::filesystem::perms::owner_write, + std::filesystem::perm_options::add); + } + { + std::ofstream corruptedObject(object, std::ios::binary | std::ios::trunc); + ASSERT_TRUE(corruptedObject.is_open()); + corruptedObject << "corrupt git object"; + } + const auto previousOutputHash = fileHash(firstDestination / "source/source.txt"); + ASSERT_TRUE(previousOutputHash.has_value()) << previousOutputHash.error().message(); + auto corruptObject = offlineFetcher.fetch(firstDestination, true); + EXPECT_FALSE(corruptObject.has_value()); + EXPECT_EQ(*fileHash(firstDestination / "source/source.txt"), *previousOutputHash); + SourceFetcher repairObject(lockedRequest(source, firstResult->hash), SourceCacheLayout(cache)); + auto repairedObject = repairObject.fetch(temporary.path() / "repaired-object"); + ASSERT_TRUE(repairedObject.has_value()) << repairedObject.error().message(); + EXPECT_EQ(*fileHash(temporary.path() / "repaired-object/source/source.txt"), + *fileHash(repository / "source.txt")); + + std::error_code ec; + std::filesystem::remove_all(cached / ".git", ec); + ASSERT_FALSE(ec) << ec.message(); + auto corrupt = offlineFetcher.fetch(temporary.path() / "corrupt", true); + EXPECT_FALSE(corrupt.has_value()); + + SourceFetcher repairFetcher(lockedRequest(source, firstResult->hash), SourceCacheLayout(cache)); + auto repaired = repairFetcher.fetch(temporary.path() / "repaired"); + ASSERT_TRUE(repaired.has_value()) << repaired.error().message(); + EXPECT_TRUE(std::filesystem::is_directory(cached / ".git")); +} + +TEST(SourceCache, CopiesGitTreeWithoutFollowingSymlinksOrCopyingSpecialFiles) +{ + utils::Cmd git("git"); + if (!git.exists()) { + GTEST_SKIP() << "git is not available"; + } + + TempDir temporary; + const auto repository = temporary.path() / "repository"; + ASSERT_TRUE(git.exec({ "init", repository.string() })); + ASSERT_TRUE(git.exec({ "-C", repository.string(), "config", "user.name", "Test" })); + ASSERT_TRUE( + git.exec({ "-C", repository.string(), "config", "user.email", "test@example.com" })); + std::ofstream(repository / "source.txt") << "source contents"; + ASSERT_TRUE(git.exec({ "-C", repository.string(), "add", "." })); + ASSERT_TRUE(git.exec({ "-C", repository.string(), "commit", "-m", "initial" })); + auto commit = git.exec({ "-C", repository.string(), "rev-parse", "HEAD" }); + ASSERT_TRUE(commit) << commit.error().message(); + *commit = commandOutputLine(std::move(*commit)); + + api::types::v1::BuilderProjectSource source; + source.kind = "git"; + source.url = repository.string(); + source.commit = *commit; + source.name = "source"; + const auto cache = temporary.path() / "cache"; + const auto destination = temporary.path() / "offline"; + ASSERT_TRUE(prepareSourceDirectories(cache, { temporary.path() / "initial", destination })); + SourceFetcher initialFetcher(GenerateSourceRequest{ source }, SourceCacheLayout(cache)); + auto initial = initialFetcher.fetch(temporary.path() / "initial"); + ASSERT_TRUE(initial) << initial.error().message(); + + // Keep test entries in Git metadata so reset/clean cannot hide copyTree mistakes. + const auto metadata = SourceCacheLayout(cache).gitObject(*commit, true) / ".git/copy-test"; + std::filesystem::create_directories(metadata / "nested/empty"); + std::ofstream(metadata / "nested/file") << "metadata contents"; + std::filesystem::permissions(metadata / "nested/file", std::filesystem::perms::owner_all); + const auto external = temporary.path() / "external"; + std::filesystem::create_directories(external); + std::ofstream(external / "file") << "external contents"; + const std::array, 5> links{ + { { "file-link", "nested/file" }, + { "directory-link", "nested" }, + { "dangling-link", "missing" }, + { "cycle-link", "." }, + { "external-link", external } } + }; + for (const auto &[name, target] : links) { + std::filesystem::create_symlink(target, metadata / name); + } + ASSERT_EQ(::mkfifo((metadata / "fifo").c_str(), 0600), 0) << errno; + + utils::fd::UniqueFd socket(::socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0)); + ASSERT_TRUE(socket) << errno; + sockaddr_un address{}; + address.sun_family = AF_UNIX; + // Bind at a short path to stay within sun_path even when the cache path is long. + const auto socketPath = (temporary.path() / "socket").string(); + ASSERT_LT(socketPath.size(), sizeof(address.sun_path)); + std::copy(socketPath.begin(), socketPath.end(), address.sun_path); + ASSERT_EQ(::bind(socket.get(), reinterpret_cast(&address), sizeof(address)), + 0) + << errno; + std::filesystem::rename(socketPath, metadata / "fsmonitor.socket"); + + SourceFetcher offlineFetcher(lockedRequest(source, initial->hash), SourceCacheLayout(cache)); + auto fetched = offlineFetcher.fetch(destination, true); + ASSERT_TRUE(fetched) << fetched.error().message(); + EXPECT_EQ(fetched->hash, initial->hash); + const auto copied = destination / "source/.git/copy-test"; + EXPECT_TRUE(std::filesystem::is_directory(copied / "nested/empty")); + ASSERT_TRUE(std::filesystem::is_regular_file(copied / "nested/file")); + EXPECT_EQ(*fileHash(copied / "nested/file"), *fileHash(metadata / "nested/file")); + EXPECT_EQ(std::filesystem::status(copied / "nested/file").permissions(), + std::filesystem::status(metadata / "nested/file").permissions()); + for (const auto &[name, target] : links) { + ASSERT_TRUE(std::filesystem::is_symlink(copied / name)) << name; + EXPECT_EQ(std::filesystem::read_symlink(copied / name), target) << name; + } + EXPECT_FALSE(std::filesystem::exists(copied / "fifo")); + EXPECT_FALSE(std::filesystem::exists(copied / "fsmonitor.socket")); + EXPECT_TRUE(std::filesystem::is_fifo(metadata / "fifo")); + EXPECT_TRUE(std::filesystem::is_socket(metadata / "fsmonitor.socket")); +} + +class GitSourceCacheResetTest : public ::testing::TestWithParam +{ +}; - QDir cacheDir("/tmp/cache"); - auto mockCmd = std::make_shared("mock"); - EXPECT_CALL(*mockCmd, - exec(ElementsAre(EndsWith("fetch-git-source"), - "/tmp/dest/" + source.name.value(), - source.url, - source.commit, - "/tmp/cache"))) - .WillOnce(Return("ok")); - EXPECT_CALL(*mockCmd, setEnv("GIT_SUBMODULES", "true")).WillOnce(ReturnRef(*mockCmd)); - SourceFetcher fetcher(source, cacheDir); - fetcher.setCommand(mockCmd); - auto ret = fetcher.fetch(QDir("/tmp/dest")); - EXPECT_TRUE(ret.has_value()) << ret.error().message(); -} - -// 测试git子模块处理功能 -// 场景1：默认启用子模块时，验证环境变量设置正确 -// 场景2：手动禁用子模块时，验证环境变量被清空 -// 预期：两种情况下都能成功获取源码 -TEST_F(SourceFetcherTest, gitSubmodules) +TEST_P(GitSourceCacheResetTest, RestoresHiddenChangesAndCleansExtraFilesOffline) { + utils::Cmd git("git"); + if (!git.exists()) { + GTEST_SKIP() << "git is not available"; + } + + TempDir temporary; + const auto repository = temporary.path() / "repository"; + ASSERT_TRUE(git.exec({ "init", repository.string() })); + ASSERT_TRUE(git.exec({ "-C", repository.string(), "config", "user.name", "Test" })); + ASSERT_TRUE( + git.exec({ "-C", repository.string(), "config", "user.email", "test@example.com" })); + std::ofstream(repository / "source.txt") << "original"; + std::ofstream(repository / ".gitignore") << "build/\n"; + ASSERT_TRUE(git.exec({ "-C", repository.string(), "add", "." })); + ASSERT_TRUE(git.exec({ "-C", repository.string(), "commit", "-m", "initial" })); + auto commit = git.exec({ "-C", repository.string(), "rev-parse", "HEAD" }); + ASSERT_TRUE(commit.has_value()) << commit.error().message(); + *commit = commandOutputLine(std::move(*commit)); + api::types::v1::BuilderProjectSource source; source.kind = "git"; + source.url = repository.string(); + source.commit = *commit; + source.name = "source"; + const auto cache = temporary.path() / "cache"; + ASSERT_TRUE( + prepareSourceDirectories(cache, + { temporary.path() / "initial", temporary.path() / "offline" })); + SourceFetcher initialFetcher(GenerateSourceRequest{ source }, SourceCacheLayout(cache)); + auto initial = initialFetcher.fetch(temporary.path() / "initial"); + ASSERT_TRUE(initial.has_value()) << initial.error().message(); + + const auto cached = SourceCacheLayout(cache).gitObject(*commit, true); + ASSERT_TRUE(git.exec({ "-C", cached.string(), "update-index", GetParam(), "source.txt" })); + std::ofstream(cached / "source.txt", std::ios::trunc) << "hidden modification"; + std::filesystem::create_directories(cached / "build"); + std::ofstream(cached / "build/config.h") << "stale configuration"; + ASSERT_TRUE(git.exec({ "init", (cached / "build/nested").string() })); + auto hiddenStatus = git.exec({ "-C", cached.string(), "status", "--porcelain=v1" }); + ASSERT_TRUE(hiddenStatus.has_value()) << hiddenStatus.error().message(); + EXPECT_TRUE(hiddenStatus->empty()); + std::ofstream(cached / "untracked.txt") << "extra input"; + + SourceFetcher offlineFetcher(lockedRequest(source, initial->hash), SourceCacheLayout(cache)); + const auto destination = temporary.path() / "offline"; + auto fetched = offlineFetcher.fetch(destination, true); + ASSERT_TRUE(fetched.has_value()) << fetched.error().message(); + EXPECT_EQ(fetched->hash, initial->hash); + EXPECT_EQ(*fileHash(destination / "source/source.txt"), *fileHash(repository / "source.txt")); + EXPECT_FALSE(std::filesystem::exists(destination / "source/build")); + EXPECT_FALSE(std::filesystem::exists(destination / "source/untracked.txt")); + EXPECT_TRUE(std::filesystem::is_directory(destination / "source/.git")); + EXPECT_NE(*fileHash(cached / "source.txt"), *fileHash(repository / "source.txt")); + EXPECT_TRUE(std::filesystem::exists(cached / "build/config.h")); + EXPECT_TRUE(std::filesystem::exists(cached / "build/nested/.git")); + EXPECT_TRUE(std::filesystem::exists(cached / "untracked.txt")); +} + +INSTANTIATE_TEST_SUITE_P(IndexFlags, + GitSourceCacheResetTest, + ::testing::Values("--assume-unchanged", "--skip-worktree")); + +TEST(SourceCache, SeparatesCommitsWithIdenticalWorkingTrees) +{ + utils::Cmd git("git"); + if (!git.exists()) { + GTEST_SKIP() << "git is not available"; + } + + TempDir temporary; + const auto repository = temporary.path() / "repository.git"; + ASSERT_TRUE(git.exec({ "init", repository.string() })) << "failed to initialize repository"; + ASSERT_TRUE(git.exec({ "-C", repository.string(), "config", "user.name", "Test" })); + ASSERT_TRUE( + git.exec({ "-C", repository.string(), "config", "user.email", "test@example.com" })); + std::ofstream(repository / "source.txt") << "same tree"; + ASSERT_TRUE(git.exec({ "-C", repository.string(), "add", "source.txt" })); + ASSERT_TRUE(git.exec({ "-C", repository.string(), "commit", "-m", "first" })); + auto firstCommit = git.exec({ "-C", repository.string(), "rev-parse", "HEAD" }); + ASSERT_TRUE(firstCommit.has_value()) << firstCommit.error().message(); + *firstCommit = commandOutputLine(std::move(*firstCommit)); + ASSERT_TRUE(git.exec({ "-C", repository.string(), "commit", "--allow-empty", "-m", "second" })); + auto secondCommit = git.exec({ "-C", repository.string(), "rev-parse", "HEAD" }); + ASSERT_TRUE(secondCommit.has_value()) << secondCommit.error().message(); + *secondCommit = commandOutputLine(std::move(*secondCommit)); + ASSERT_NE(*firstCommit, *secondCommit); + ASSERT_TRUE( + prepareSourceDirectories(temporary.path() / "cache", { temporary.path() / "destination" })); + + auto fetchCommit = [&](const std::string &commit, const std::string &name) { + api::types::v1::BuilderProjectSource source; + source.kind = "git"; + source.url = repository.string(); + source.commit = commit; + source.name = name; + SourceFetcher fetcher(GenerateSourceRequest{ source }, + SourceCacheLayout(temporary.path() / "cache")); + return fetcher.fetch(temporary.path() / "destination"); + }; + auto first = fetchCommit(*firstCommit, "first"); + auto second = fetchCommit(*secondCommit, "second"); + ASSERT_TRUE(first.has_value()) << first.error().message(); + ASSERT_TRUE(second.has_value()) << second.error().message(); + EXPECT_NE(first->hash, second->hash); + EXPECT_EQ(first->hash, "git-commit:" + *firstCommit); + EXPECT_EQ(second->hash, "git-commit:" + *secondCommit); + + auto firstHead = + git.exec({ "-C", (temporary.path() / "destination/first").string(), "rev-parse", "HEAD" }); + auto secondHead = + git.exec({ "-C", (temporary.path() / "destination/second").string(), "rev-parse", "HEAD" }); + ASSERT_TRUE(firstHead.has_value()) << firstHead.error().message(); + ASSERT_TRUE(secondHead.has_value()) << secondHead.error().message(); + EXPECT_EQ(commandOutputLine(std::move(*firstHead)), *firstCommit); + EXPECT_EQ(commandOutputLine(std::move(*secondHead)), *secondCommit); + + SourceCacheLayout cache(temporary.path() / "cache"); + EXPECT_TRUE(std::filesystem::is_directory(cache.gitObject(*firstCommit, true) / ".git")); + EXPECT_TRUE(std::filesystem::is_directory(cache.gitObject(*secondCommit, true) / ".git")); +} + +TEST(SourceCache, PreservesSubmoduleGitMetadata) +{ + utils::Cmd git("git"); + if (!git.exists()) { + GTEST_SKIP() << "git is not available"; + } + utils::EnvironmentVariableGuard protocol("GIT_ALLOW_PROTOCOL", "file"); + + TempDir temporary; + auto initializeRepository = [&](const std::filesystem::path &repository) { + EXPECT_TRUE(git.exec({ "init", repository.string() })); + EXPECT_TRUE(git.exec({ "-C", repository.string(), "config", "user.name", "Test" })); + EXPECT_TRUE( + git.exec({ "-C", repository.string(), "config", "user.email", "test@example.com" })); + }; + + const auto grandchild = temporary.path() / "grandchild"; + initializeRepository(grandchild); + std::ofstream(grandchild / "leaf.txt") << "leaf"; + std::ofstream(grandchild / ".gitignore") << "build/\n"; + ASSERT_TRUE(git.exec({ "-C", grandchild.string(), "add", "." })); + ASSERT_TRUE(git.exec({ "-C", grandchild.string(), "commit", "-m", "grandchild" })); + + const auto child = temporary.path() / "child"; + initializeRepository(child); + std::ofstream(child / "child.txt") << "child"; + std::ofstream(child / ".gitignore") << "build/\n"; + ASSERT_TRUE(git.exec({ "-C", child.string(), "add", "." })); + ASSERT_TRUE(git.exec({ "-C", child.string(), "commit", "-m", "child" })); + ASSERT_TRUE(git.exec({ "-C", + child.string(), + "-c", + "protocol.file.allow=always", + "submodule", + "add", + grandchild.string(), + "nested/grandchild" })); + ASSERT_TRUE(git.exec({ "-C", child.string(), "commit", "-am", "nested submodule" })); + + const auto repository = temporary.path() / "repository"; + initializeRepository(repository); + ASSERT_TRUE(git.exec({ "-C", + repository.string(), + "-c", + "protocol.file.allow=always", + "submodule", + "add", + child.string(), + "deps/child" })); + ASSERT_TRUE(git.exec({ "-C", repository.string(), "commit", "-am", "add submodule" })); + auto commit = git.exec({ "-C", repository.string(), "rev-parse", "HEAD" }); + ASSERT_TRUE(commit.has_value()) << commit.error().message(); + *commit = commandOutputLine(std::move(*commit)); + + api::types::v1::BuilderProjectSource source; + source.kind = "git"; + source.url = repository.string(); + source.commit = *commit; + source.name = "source"; + source.submodules = true; + const auto cachePath = temporary.path() / "cache"; + ASSERT_TRUE(prepareSourceDirectories(cachePath, + { temporary.path() / "destination", + temporary.path() / "offline", + temporary.path() / "without-submodules" })); + SourceFetcher fetcher(GenerateSourceRequest{ source }, SourceCacheLayout(cachePath)); + auto fetched = fetcher.fetch(temporary.path() / "destination"); + ASSERT_TRUE(fetched.has_value()) << fetched.error().message(); + + const auto checkout = temporary.path() / "destination/source"; + EXPECT_TRUE(std::filesystem::is_directory(checkout / ".git")); + EXPECT_TRUE(std::filesystem::is_regular_file(checkout / "deps/child/.git")); + auto childHead = git.exec({ "-C", (checkout / "deps/child").string(), "rev-parse", "HEAD" }); + ASSERT_TRUE(childHead.has_value()) << childHead.error().message(); + + const auto cached = SourceCacheLayout(cachePath).gitObject(*commit, true); + const auto cachedChild = cached / "deps/child"; + const auto cachedGrandchild = cachedChild / "nested/grandchild"; + ASSERT_TRUE( + git.exec({ "-C", cachedChild.string(), "update-index", "--assume-unchanged", "child.txt" })); + ASSERT_TRUE( + git.exec({ "-C", cachedGrandchild.string(), "update-index", "--skip-worktree", "leaf.txt" })); + std::ofstream(cachedChild / "child.txt", std::ios::trunc) << "hidden child modification"; + std::ofstream(cachedGrandchild / "leaf.txt", std::ios::trunc) << "hidden leaf modification"; + std::filesystem::create_directories(cachedChild / "build"); + std::filesystem::create_directories(cachedGrandchild / "build"); + std::ofstream(cachedChild / "build/config.h") << "stale child configuration"; + std::ofstream(cachedGrandchild / "build/config.h") << "stale leaf configuration"; + std::ofstream(cachedChild / "untracked.txt") << "extra child input"; + + SourceFetcher offlineFetcher(lockedRequest(source, fetched->hash), + SourceCacheLayout(cachePath)); + auto offline = offlineFetcher.fetch(temporary.path() / "offline", true); + ASSERT_TRUE(offline.has_value()) << offline.error().message(); + EXPECT_TRUE( + std::filesystem::is_regular_file(temporary.path() / "offline/source/deps/child/.git")); + const auto offlineChild = temporary.path() / "offline/source/deps/child"; + const auto offlineGrandchild = offlineChild / "nested/grandchild"; + EXPECT_EQ(*fileHash(offlineChild / "child.txt"), *fileHash(child / "child.txt")); + EXPECT_EQ(*fileHash(offlineGrandchild / "leaf.txt"), *fileHash(grandchild / "leaf.txt")); + EXPECT_FALSE(std::filesystem::exists(offlineChild / "build")); + EXPECT_FALSE(std::filesystem::exists(offlineGrandchild / "build")); + EXPECT_FALSE(std::filesystem::exists(offlineChild / "untracked.txt")); + EXPECT_TRUE(std::filesystem::is_regular_file(offlineGrandchild / ".git")); + EXPECT_NE(*fileHash(cachedChild / "child.txt"), *fileHash(child / "child.txt")); + EXPECT_NE(*fileHash(cachedGrandchild / "leaf.txt"), *fileHash(grandchild / "leaf.txt")); + EXPECT_TRUE(std::filesystem::exists(cachedChild / "build/config.h")); + EXPECT_TRUE(std::filesystem::exists(cachedGrandchild / "build/config.h")); + + source.submodules = false; + SourceFetcher noSubmodules(GenerateSourceRequest{ source }, SourceCacheLayout(cachePath)); + auto without = noSubmodules.fetch(temporary.path() / "without-submodules"); + ASSERT_TRUE(without.has_value()) << without.error().message(); + EXPECT_EQ(without->hash, fetched->hash); + EXPECT_FALSE( + std::filesystem::exists(temporary.path() / "without-submodules/source/deps/child/child.txt")); + const SourceCacheLayout cache(cachePath); + EXPECT_TRUE(std::filesystem::is_directory(cache.gitObject(*commit, true))); + EXPECT_TRUE(std::filesystem::is_directory(cache.gitObject(*commit, false))); +} + +// 测试获取文件类型源码的正常流程 +// 场景：提供有效的文件类型source配置，包括url、digest和name +// 预期：成功获取源码并返回ok +TEST_F(SourceFetcherTest, FetchFileSource) +{ + TempDir temporary; + const auto artifact = temporary.path() / "artifact"; + std::ofstream(artifact) << "file source"; + auto digest = fileHash(artifact); + ASSERT_TRUE(digest.has_value()) << digest.error().message(); + + api::types::v1::BuilderProjectSource source; + source.kind = "file"; source.url = "https://example.com/repo.git"; - source.commit = "abcdef123456"; + source.digest = *digest; source.name = "repo"; - // default is true - { - QDir cacheDir("/tmp/cache"); - auto mockCmd = std::make_shared("mock"); - EXPECT_CALL(*mockCmd, exec(_)).WillOnce(Return("ok")); - EXPECT_CALL(*mockCmd, setEnv("GIT_SUBMODULES", "true")).WillOnce(ReturnRef(*mockCmd)); - SourceFetcher fetcher(source, cacheDir); - fetcher.setCommand(mockCmd); - auto ret2 = fetcher.fetch(QDir("/tmp/dest")); - EXPECT_TRUE(ret2.has_value()) << ret2.error().message(); - } - // manually disable submodules - source.submodules = false; - { - QDir cacheDir("/tmp/cache"); - auto mockCmd = std::make_shared("mock"); - EXPECT_CALL(*mockCmd, exec(_)).WillOnce(Return("ok")); - EXPECT_CALL(*mockCmd, setEnv("GIT_SUBMODULES", "")).WillOnce(ReturnRef(*mockCmd)); - SourceFetcher fetcher(source, cacheDir); - fetcher.setCommand(mockCmd); - auto ret2 = fetcher.fetch(QDir("/tmp/dest")); - EXPECT_TRUE(ret2.has_value()) << ret2.error().message(); + + const auto cached = temporary.path() / "cache/sources/v1/file/sha256" / digest->substr(7); + std::error_code ec; + std::filesystem::create_directories(cached.parent_path(), ec); + ASSERT_FALSE(ec) << ec.message(); + std::filesystem::copy_file(artifact, cached, ec); + ASSERT_FALSE(ec) << ec.message(); + + SourceFetcher fetcher(lockedRequest(source, *digest), + SourceCacheLayout(temporary.path() / "cache")); + const auto destination = temporary.path() / "dest"; + ASSERT_TRUE(prepareSourceDirectories( + temporary.path() / "cache", + { destination, temporary.path() / "offline", temporary.path() / "corrupt" })); + auto ret = fetcher.fetch(destination); + ASSERT_TRUE(ret.has_value()) << ret.error().message(); + EXPECT_EQ(*fileHash(destination / "repo"), *digest); + + struct stat cacheStatus{}; + struct stat buildStatus{}; + ASSERT_EQ(::stat(cached.c_str(), &cacheStatus), 0); + ASSERT_EQ(::stat((destination / "repo").c_str(), &buildStatus), 0); + EXPECT_NE(cacheStatus.st_ino, buildStatus.st_ino); + + std::ofstream(destination / "repo", std::ios::trunc) << "modified"; + EXPECT_EQ(*fileHash(cached), *digest); + + SourceFetcher offlineFetcher(lockedRequest(source, *digest), + SourceCacheLayout(temporary.path() / "cache")); + auto offline = offlineFetcher.fetch(temporary.path() / "offline", true); + ASSERT_TRUE(offline.has_value()) << offline.error().message(); + EXPECT_EQ(*fileHash(temporary.path() / "offline/repo"), *digest); + + std::ofstream(cached, std::ios::trunc) << "corrupt"; + auto corrupt = offlineFetcher.fetch(temporary.path() / "corrupt", true); + EXPECT_FALSE(corrupt.has_value()); +} + +TEST_F(SourceFetcherTest, MaterializesCachedArchive) +{ + TempDir temporary; + + utils::Cmd tar("tar"); + if (!tar.exists()) { + GTEST_SKIP() << "tar is not available"; } + const auto content = temporary.path() / "content"; + std::error_code ec; + std::filesystem::create_directories(content, ec); + ASSERT_FALSE(ec) << ec.message(); + std::ofstream(content / "source.txt") << "source"; + const auto artifact = temporary.path() / "source.tar"; + auto archived = tar.exec({ "-cf", artifact.string(), "-C", content.string(), "." }); + ASSERT_TRUE(archived.has_value()) << archived.error().message(); + auto digest = fileHash(artifact); + ASSERT_TRUE(digest.has_value()) << digest.error().message(); + + api::types::v1::BuilderProjectSource source; + source.kind = "archive"; + source.url = "https://example.com/source.tar.gz"; + source.digest = *digest; + source.name = "source"; + + const auto cached = temporary.path() / "cache/sources/v1/archive/sha256" / digest->substr(7); + std::filesystem::create_directories(cached.parent_path(), ec); + ASSERT_FALSE(ec) << ec.message(); + std::filesystem::copy_file(artifact, cached, ec); + ASSERT_FALSE(ec) << ec.message(); + + SourceFetcher fetcher(lockedRequest(source, *digest), + SourceCacheLayout(temporary.path() / "cache")); + ASSERT_TRUE( + prepareSourceDirectories(temporary.path() / "cache", { temporary.path() / "destination" })); + auto fetched = fetcher.fetch(temporary.path() / "destination", true); + ASSERT_TRUE(fetched.has_value()) << fetched.error().message(); + EXPECT_EQ(fetched->hash, *digest); + EXPECT_TRUE( + std::filesystem::is_regular_file(temporary.path() / "destination/source/source.txt", ec)); + EXPECT_FALSE(ec); } // 测试无效的source类型 @@ -138,18 +1009,20 @@ TEST_F(SourceFetcherTest, gitSubmodules) // 预期：返回错误结果，错误码为-1 TEST_F(SourceFetcherTest, FetchInvalidSourceKind) { + TempDir temporary; api::types::v1::BuilderProjectSource source; source.kind = "invalid"; source.url = "https://example.com/repo.git"; source.name = "repo"; - QDir cacheDir("/tmp/cache"); - auto mockCmd = std::make_shared("mock"); - SourceFetcher fetcher(source, cacheDir); - fetcher.setCommand(mockCmd); - auto ret = fetcher.fetch(QDir("/tmp/dest")); - EXPECT_FALSE(ret.has_value()); + const auto cacheDir = temporary.path() / "cache"; + const auto destination = temporary.path() / "sources"; + ASSERT_TRUE(prepareSourceDirectories(cacheDir, { destination })); + SourceFetcher fetcher(GenerateSourceRequest{ source }, SourceCacheLayout(cacheDir)); + auto ret = fetcher.fetch(destination); + ASSERT_FALSE(ret.has_value()); EXPECT_EQ(ret.error().code(), -1); + EXPECT_NE(ret.error().message().find("unknown source kind"), std::string::npos); } // 测试缺失必填字段的情况 @@ -157,77 +1030,64 @@ TEST_F(SourceFetcherTest, FetchInvalidSourceKind) // 预期：返回错误结果，错误码为-1 TEST_F(SourceFetcherTest, FetchMissingRequiredField) { + TempDir temporary; api::types::v1::BuilderProjectSource source; source.kind = "git"; // 缺少 url 和 name source.commit = "abcdef123456"; - QDir cacheDir("/tmp/cache"); - auto mockCmd = std::make_shared("mock"); - SourceFetcher fetcher(source, cacheDir); - fetcher.setCommand(mockCmd); - auto ret = fetcher.fetch(QDir("/tmp/dest")); - EXPECT_FALSE(ret.has_value()); + const auto cacheDir = temporary.path() / "cache"; + const auto destination = temporary.path() / "sources"; + ASSERT_TRUE(prepareSourceDirectories(cacheDir, { destination })); + SourceFetcher fetcher(GenerateSourceRequest{ source }, SourceCacheLayout(cacheDir)); + auto ret = fetcher.fetch(destination); + ASSERT_FALSE(ret.has_value()); EXPECT_EQ(ret.error().code(), -1); + EXPECT_NE(ret.error().message().find("source URL is missing"), std::string::npos); } -// 测试无效的git commit -// 场景1：未设置commit字段 -// 场景2：设置无效的commit值 -// 预期：两种情况都返回错误结果，场景2的错误码为-2 +// 测试未设置git commit +// 预期：在调用git脚本前返回错误结果 TEST_F(SourceFetcherTest, FetchInvalidGitCommit) { LINGLONG_TRACE("FetchInvalidGitCommit"); + TempDir temporary; api::types::v1::BuilderProjectSource source; source.kind = "git"; source.url = "https://example.com/repo.git"; source.name = "repo"; - QDir cacheDir("/tmp/cache"); + const auto cacheDir = temporary.path() / "cache"; + const auto destination = temporary.path() / "sources"; + ASSERT_TRUE(prepareSourceDirectories(cacheDir, { destination })); { - SourceFetcher fetcher(source, cacheDir); - auto ret = fetcher.fetch(QDir("/tmp/dest")); + SourceFetcher fetcher(GenerateSourceRequest{ source }, SourceCacheLayout(cacheDir)); + auto ret = fetcher.fetch(destination); EXPECT_FALSE(ret.has_value()); } - source.commit = "invalid_commit"; - SourceFetcher fetcher(source, cacheDir); - auto mockCmd = std::make_shared("mock"); - EXPECT_CALL(*mockCmd, exec(_)).WillOnce(Return(LINGLONG_ERR("Invalid commit", -2))); - EXPECT_CALL(*mockCmd, setEnv("GIT_SUBMODULES", "true")).WillOnce(ReturnRef(*mockCmd)); - fetcher.setCommand(mockCmd); - auto ret = fetcher.fetch(QDir("/tmp/dest")); - EXPECT_FALSE(ret.has_value()); - EXPECT_EQ(ret.error().code(), -2); } // 测试无效的文件digest -// 场景1：未设置digest字段 -// 场景2：设置无效的digest值 -// 预期：两种情况都返回错误结果，场景2的错误码为-3 +// 场景：设置无效的digest值 +// 预期：在调用下载脚本前返回错误结果 TEST_F(SourceFetcherTest, FetchInvalidFileDigest) { LINGLONG_TRACE("FetchInvalidFileDigest"); + TempDir temporary; api::types::v1::BuilderProjectSource source; source.kind = "file"; source.url = "https://example.com/repo.git"; source.name = "repo"; - QDir cacheDir("/tmp/cache"); - { - SourceFetcher fetcher(source, cacheDir); - auto ret = fetcher.fetch(QDir("/tmp/dest")); - EXPECT_FALSE(ret.has_value()); - } - + const auto cacheDir = temporary.path() / "cache"; + const auto destination = temporary.path() / "sources"; + ASSERT_TRUE(prepareSourceDirectories(cacheDir, { destination })); source.digest = "invalid_digest"; - SourceFetcher fetcher(source, cacheDir); - auto mockCmd = std::make_shared("mock"); - EXPECT_CALL(*mockCmd, exec(_)).WillOnce(Return(LINGLONG_ERR("Invalid digest", -3))); - fetcher.setCommand(mockCmd); - auto ret = fetcher.fetch(QDir("/tmp/dest")); + SourceFetcher fetcher(GenerateSourceRequest{ source }, SourceCacheLayout(cacheDir)); + auto ret = fetcher.fetch(destination); EXPECT_FALSE(ret.has_value()); - EXPECT_EQ(ret.error().code(), -3); + EXPECT_EQ(ret.error().code(), -1); } // 测试未设置name字段的情况 @@ -235,17 +1095,29 @@ TEST_F(SourceFetcherTest, FetchInvalidFileDigest) // 预期：仍能成功获取源码 TEST_F(SourceFetcherTest, FetchNoSetName) { + TempDir temporary; + const auto artifact = temporary.path() / "artifact"; + std::ofstream(artifact) << "file source"; + auto digest = fileHash(artifact); + ASSERT_TRUE(digest.has_value()) << digest.error().message(); + api::types::v1::BuilderProjectSource source; source.kind = "file"; source.url = "https://example.com/repo.git"; - source.digest = "digest"; - - QDir cacheDir("/tmp/cache"); - auto mockCmd = std::make_shared("mock"); - EXPECT_CALL(*mockCmd, exec(_)).WillOnce(Return("ok")); - SourceFetcher fetcher(source, cacheDir); - fetcher.setCommand(mockCmd); - auto ret = fetcher.fetch(QDir("/tmp/dest")); + source.digest = *digest; + + const auto cached = temporary.path() / "cache/sources/v1/file/sha256" / digest->substr(7); + std::error_code ec; + std::filesystem::create_directories(cached.parent_path(), ec); + ASSERT_FALSE(ec) << ec.message(); + std::filesystem::copy_file(artifact, cached, ec); + ASSERT_FALSE(ec) << ec.message(); + + SourceFetcher fetcher(lockedRequest(source, *digest), + SourceCacheLayout(temporary.path() / "cache")); + ASSERT_TRUE( + prepareSourceDirectories(temporary.path() / "cache", { temporary.path() / "dest" })); + auto ret = fetcher.fetch(temporary.path() / "dest"); EXPECT_TRUE(ret.has_value()); } diff --git a/libs/linglong/tests/ll-tests/src/linglong/mocks/linglong_builder_mock.h b/libs/linglong/tests/ll-tests/src/linglong/mocks/linglong_builder_mock.h index eed498dbf..7137bc3f5 100644 --- a/libs/linglong/tests/ll-tests/src/linglong/mocks/linglong_builder_mock.h +++ b/libs/linglong/tests/ll-tests/src/linglong/mocks/linglong_builder_mock.h @@ -8,11 +8,13 @@ #include "ocppi/cli/crun/Crun.hpp" #include +#include namespace linglong::builder { class BuilderMock : public Builder { public: + using Builder::buildStageFetchSource; using Builder::layerExportFilename; using Builder::uabExportFilename; @@ -27,6 +29,13 @@ class BuilderMock : public Builder initTempContainerBuilder(), initBuilderConfig()) { }; + BuilderMock(api::types::v1::BuilderProject project, const std::filesystem::path &workingDir) + : Builder(std::optional{ std::move(project) }, + workingDir, + initTempRepo(), + initTempContainerBuilder(), + initBuilderConfig()) { }; + private: static linglong::repo::OSTreeRepo &initTempRepo() { diff --git a/libs/linglong/tests/ll-tests/src/linglong/repo/ostree_repo_test.cpp b/libs/linglong/tests/ll-tests/src/linglong/repo/ostree_repo_test.cpp index 2462656c3..864e63c52 100644 --- a/libs/linglong/tests/ll-tests/src/linglong/repo/ostree_repo_test.cpp +++ b/libs/linglong/tests/ll-tests/src/linglong/repo/ostree_repo_test.cpp @@ -431,6 +431,90 @@ TEST_F(RepoTest, moduleMergesUseBinaryInfo) EXPECT_EQ(persistentMergedInfo->packageInfoV2Module, "binary"); } +TEST_F(RepoTest, moduleMergesKeepChannelsSeparate) +{ + for (const bool otherChannelHasMainModule : { false, true }) { + SCOPED_TRACE(otherChannelHasMainModule); + TempDir tempDir; + const auto repoRoot = tempDir.path() / "repo-root"; + ASSERT_TRUE(fs::create_directories(repoRoot)); + auto repo = OSTreeRepo::create(repoRoot, createRepoConfig()); + ASSERT_TRUE(repo) << repo.error().message(); + + auto importModule = [&](const std::string &channel, const std::string &module) { + const auto name = channel + "-" + module; + const auto directory = tempDir.path() / name; + fs::create_directories(directory / "files"); + const api::types::v1::PackageInfoV2 info{ + .arch = { "x86_64" }, + .channel = channel, + .id = "org.test.merge", + .kind = "app", + .packageInfoV2Module = module, + .version = "1.0.0", + }; + std::ofstream(directory / "info.json") << nlohmann::json(info).dump(); + std::ofstream(directory / "files" / name) << name; + + auto imported = repo->get()->importLayerDir(package::LayerDir(directory)); + ASSERT_TRUE(imported) << imported.error().message(); + }; + + importModule("main", "binary"); + importModule("main", "develop"); + importModule("testing", "locale"); + if (otherChannelHasMainModule) { + importModule("testing", "binary"); + importModule("testing", "develop"); + } + + const auto mainRef = package::Reference::parse("main:org.test.merge/1.0.0/x86_64"); + ASSERT_TRUE(mainRef); + fs::path mainMergedPath; + for (int merge = 0; merge < 2; ++merge) { + SCOPED_TRACE(merge); + auto merged = repo->get()->mergeModules(); + ASSERT_TRUE(merged) << merged.error().message(); + + auto mainLayer = repo->get()->getMergedModuleDir(*mainRef, false); + ASSERT_TRUE(mainLayer) << mainLayer.error().message(); + EXPECT_TRUE(fs::exists(mainLayer->path() / "files/main-binary")); + EXPECT_TRUE(fs::exists(mainLayer->path() / "files/main-develop")); + EXPECT_FALSE(fs::exists(mainLayer->path() / "files/testing-locale")); + EXPECT_FALSE(fs::exists(mainLayer->path() / "files/testing-binary")); + EXPECT_FALSE(fs::exists(mainLayer->path() / "files/testing-develop")); + + auto mainInfo = mainLayer->info(); + ASSERT_TRUE(mainInfo) << mainInfo.error().message(); + EXPECT_EQ(mainInfo->channel, "main"); + EXPECT_EQ(mainInfo->packageInfoV2Module, "binary"); + if (merge == 0) { + mainMergedPath = mainLayer->path(); + } else { + EXPECT_EQ(mainLayer->path(), mainMergedPath); + } + + if (otherChannelHasMainModule) { + auto testingRef = package::Reference::parse("testing:org.test.merge/1.0.0/x86_64"); + ASSERT_TRUE(testingRef); + auto testingLayer = repo->get()->getMergedModuleDir(*testingRef, false); + ASSERT_TRUE(testingLayer) << testingLayer.error().message(); + EXPECT_NE(testingLayer->path(), mainLayer->path()); + EXPECT_TRUE(fs::exists(testingLayer->path() / "files/testing-binary")); + EXPECT_TRUE(fs::exists(testingLayer->path() / "files/testing-develop")); + EXPECT_TRUE(fs::exists(testingLayer->path() / "files/testing-locale")); + EXPECT_FALSE(fs::exists(testingLayer->path() / "files/main-binary")); + EXPECT_FALSE(fs::exists(testingLayer->path() / "files/main-develop")); + + auto testingInfo = testingLayer->info(); + ASSERT_TRUE(testingInfo) << testingInfo.error().message(); + EXPECT_EQ(testingInfo->channel, "testing"); + EXPECT_EQ(testingInfo->packageInfoV2Module, "binary"); + } + } + } +} + TEST_F(RepoTest, createPrefersRepoLocalConfigOverFallbackConfig) { TempDir tempDir; diff --git a/libs/linglong/tests/ll-tests/src/linglong/runtime/run_context_test.cpp b/libs/linglong/tests/ll-tests/src/linglong/runtime/run_context_test.cpp index e2b8205de..27b3cdae2 100644 --- a/libs/linglong/tests/ll-tests/src/linglong/runtime/run_context_test.cpp +++ b/libs/linglong/tests/ll-tests/src/linglong/runtime/run_context_test.cpp @@ -6,6 +6,7 @@ #include #include "../../common/tempdir.h" +#include "linglong/api/types/v1/Generators.hpp" #include "linglong/cli/cli.h" #include "linglong/oci-cfg-generators/container_cfg_builder.h" #include "linglong/package/fuzzy_reference.h" @@ -131,6 +132,167 @@ class RunContextTest : public ::testing::Test std::unique_ptr repo; }; +TEST(RunContextBuildModules, UsesExactBaseAndRuntimeModulesIncludingLegacyMainModule) +{ + for (const auto &binaryName : { "binary", "runtime" }) { + SCOPED_TRACE(binaryName); + TempDir temporary; + const auto repoRoot = temporary.path() / "repo"; + std::filesystem::create_directories(repoRoot); + auto repo = repo::OSTreeRepo::create( + repoRoot, + api::types::v1::RepoConfigV2{ + .defaultRepo = "stable", + .repos = { api::types::v1::Repo{ .name = "stable", + .priority = 0, + .url = "https://example.com/repo" } }, + .version = 2 }); + ASSERT_TRUE(repo) << repo.error().message(); + + for (const auto &module : { binaryName, "develop", "locale" }) { + const auto directory = temporary.path() / module; + std::filesystem::create_directories(directory / "files"); + const api::types::v1::PackageInfoV2 info{ .arch = { "x86_64" }, + .channel = "main", + .id = "org.test.Base", + .kind = "runtime", + .packageInfoV2Module = module, + .version = "1.0.0.0" }; + std::ofstream(directory / "info.json") << nlohmann::json(info).dump(); + std::ofstream(directory / "files" / module) << module; + auto imported = (*repo)->importLayerDir(package::LayerDir(directory)); + ASSERT_TRUE(imported) << imported.error().message(); + } + ASSERT_TRUE((*repo)->mergeModules()); + + api::types::v1::BuilderProject project; + project.version = "1"; + project.package.id = "org.test.App"; + project.package.kind = "app"; + project.package.version = "1.0.0.0"; + project.package.architecture = "arm64"; + project.base = "org.test.Base/1.0.0.0/x86_64"; + project.runtime = project.base; + const auto output = temporary.path() / "output"; + const std::string reference = "main:org.test.Base/1.0.0.0/x86_64"; + + // A newly generated lock can include all installed modules. + const std::vector allModules{ binaryName, "develop", "locale" }; + const api::types::v1::BuilderProjectLock complete{ + .dependencies = { { *project.base, { allModules, reference }, "base" }, + { *project.runtime, { allModules, reference }, "runtime" } }, + .sources = {}, + .version = 1 + }; + runtime::RunContext defaultContext(**repo); + ASSERT_TRUE(defaultContext.resolve(project, output, complete)); + const auto defaultLayer = defaultContext.getBaseLayerPath(); + ASSERT_TRUE(defaultLayer); + EXPECT_TRUE(std::filesystem::exists(*defaultLayer / "files/locale")); + + for (const auto &modules : { std::vector{ binaryName }, + std::vector{ binaryName, "develop" } }) { + SCOPED_TRACE(modules.size()); + runtime::RunContext lockedContext(**repo); + const api::types::v1::BuilderProjectLockDependency base{ *project.base, + { modules, reference }, + "base" }; + const api::types::v1::BuilderProjectLockDependency runtime{ *project.runtime, + { modules, reference }, + "runtime" }; + const api::types::v1::BuilderProjectLock lock{ .dependencies = { base, runtime }, + .sources = {}, + .version = 1 }; + auto resolved = lockedContext.resolve(project, output, lock); + ASSERT_TRUE(resolved) << resolved.error().message(); + + const auto baseLayer = lockedContext.getBaseLayerPath(); + const auto runtimeLayer = lockedContext.getRuntimeLayerPath(); + ASSERT_TRUE(baseLayer); + ASSERT_TRUE(runtimeLayer); + for (const auto &layer : { *baseLayer, *runtimeLayer }) { + EXPECT_TRUE(std::filesystem::exists(layer / "files" / binaryName)); + EXPECT_EQ(std::filesystem::exists(layer / "files/develop"), modules.size() == 2); + EXPECT_FALSE(std::filesystem::exists(layer / "files/locale")); + } + + auto baseOnly = lock; + baseOnly.dependencies.pop_back(); + ASSERT_TRUE(lockedContext.resolve(project, output, baseOnly)); + EXPECT_FALSE(lockedContext.hasRuntime()); + } + } +} + +TEST(RunContextBuildModules, BaseUsesLockedArchitectureOrBuildHost) +{ + for (const bool explicitArchitecture : { false, true }) { + SCOPED_TRACE(explicitArchitecture); + const auto host = package::Architecture::currentCPUArchitecture().toString(); + const std::string other = host == "arm64" ? "x86_64" : "arm64"; + const auto baseArchitecture = explicitArchitecture ? other : host; + const std::string baseDeclaration = + "org.test.Base/1.0.0" + (explicitArchitecture ? "/" + other : ""); + TempDir temporary; + const auto root = temporary.path() / "repo"; + std::filesystem::create_directories(root); + auto repository = repo::OSTreeRepo::create( + root, + api::types::v1::RepoConfigV2{ + .defaultRepo = "stable", + .repos = { api::types::v1::Repo{ .name = "stable", + .priority = 0, + .url = "https://example.com/repo" } }, + .version = 2 }); + ASSERT_TRUE(repository) << repository.error().message(); + for (const auto &id : { "org.test.Base", "org.test.Runtime" }) { + const auto directory = temporary.path() / id; + std::filesystem::create_directories(directory / "files"); + api::types::v1::PackageInfoV2 info{ + .arch = { std::string(id) == "org.test.Base" ? baseArchitecture : other }, + .base = baseDeclaration, + .channel = "main", + .id = id, + .kind = "runtime", + .packageInfoV2Module = "binary", + .version = "1.0.0.0" + }; + std::ofstream(directory / "info.json") << nlohmann::json(info).dump(); + std::ofstream(directory / "files" / id) << id; + ASSERT_TRUE((*repository)->importLayerDir(package::LayerDir(directory))); + } + ASSERT_TRUE((*repository)->mergeModules()); + api::types::v1::BuilderProject project; + project.version = "1"; + project.package.id = "org.test.App"; + project.package.kind = "app"; + project.package.version = "1.0.0.0"; + project.package.architecture = other; + project.runtime = "org.test.Runtime/1.0.0/" + other; + api::types::v1::BuilderProjectLock lock{ + .dependencies = { { baseDeclaration, + { { "binary" }, + "main:org.test.Base/1.0.0.0/" + + (explicitArchitecture ? other : "unknown") }, + "base" }, + { *project.runtime, + { { "binary" }, "main:org.test.Runtime/1.0.0.0/" + other }, + "runtime" } }, + .sources = {}, + .version = 1 + }; + const auto original = nlohmann::json(lock); + runtime::RunContext context(**repository); + auto resolved = context.resolve(project, temporary.path() / "output", lock); + ASSERT_TRUE(resolved) << resolved.error().message(); + ASSERT_TRUE(context.getBaseLayer()); + ASSERT_TRUE(context.getRuntimeLayer()); + EXPECT_EQ(context.getBaseLayer()->getReference().arch.toString(), baseArchitecture); + EXPECT_EQ(context.getRuntimeLayer()->getReference().arch.toString(), other); + EXPECT_EQ(nlohmann::json(lock), original); + } +} + TEST_F(RunContextTest, layerExist) { // Create a valid reference diff --git a/libs/linglong/tests/ll-tests/src/linglong/utils/file_test.cpp b/libs/linglong/tests/ll-tests/src/linglong/utils/file_test.cpp index 63438bbf4..09ab6ff4a 100644 --- a/libs/linglong/tests/ll-tests/src/linglong/utils/file_test.cpp +++ b/libs/linglong/tests/ll-tests/src/linglong/utils/file_test.cpp @@ -7,13 +7,18 @@ #include "common/scoped_umask.h" #include "common/tempdir.h" #include "linglong/utils/file.h" +#include "linglong/utils/unique_fd.h" #include +#include #include #include #include #include +#include +#include +#include #include namespace fs = std::filesystem; @@ -57,7 +62,8 @@ TEST_F(FileTest, CopyDirectory) return path.filename() != "ignored.txt"; }; - linglong::utils::copyDirectory(src_dir, dest_dir, matcher); + auto result = linglong::utils::copyDirectory(src_dir, dest_dir, matcher); + ASSERT_TRUE(result) << result.error().message(); EXPECT_TRUE(fs::exists(dest_dir / "file1.txt")); std::ifstream ifs(dest_dir / "file1.txt"); std::string content((std::istreambuf_iterator(ifs)), (std::istreambuf_iterator())); @@ -82,7 +88,8 @@ TEST_F(FileTest, CopyDirectory_MatcherFileInUnmatchedDir) return path.filename() == "file2.txt"; }; - linglong::utils::copyDirectory(src_dir, dest_dir, matcher); + auto result = linglong::utils::copyDirectory(src_dir, dest_dir, matcher); + ASSERT_TRUE(result) << result.error().message(); EXPECT_FALSE(fs::exists(dest_dir / "file1.txt")); EXPECT_FALSE(fs::exists(dest_dir / "ignored.txt")); EXPECT_TRUE(fs::exists(dest_dir / "subdir1" / "file2.txt")); @@ -100,7 +107,8 @@ TEST_F(FileTest, CopyDirectory_MatcherSubDirInUnmatchedDir) return path.string().rfind("subdir1/subdir2", 0) == 0; }; - linglong::utils::copyDirectory(src_dir, dest_dir, matcher); + auto result = linglong::utils::copyDirectory(src_dir, dest_dir, matcher); + ASSERT_TRUE(result) << result.error().message(); EXPECT_FALSE(fs::exists(dest_dir / "file1.txt")); EXPECT_FALSE(fs::exists(dest_dir / "ignored.txt")); EXPECT_FALSE(fs::exists(dest_dir / "subdir1" / "file2.txt")); @@ -114,23 +122,23 @@ TEST_F(FileTest, CopyDirectory_MatcherSubDirInUnmatchedDir) EXPECT_EQ(content, "content3"); } -TEST_F(FileTest, CopyDirectory_OverwriteExisting) +TEST_F(FileTest, CopyDirectory_RepeatedCopiesPreserveExistingEntries) { - std::ofstream(dest_dir / "file1.txt") << "existing_content"; - auto matcher = [](const fs::path &path) { return path.filename() != "ignored.txt"; }; - linglong::utils::copyDirectory(src_dir, - dest_dir, - matcher, - fs::copy_options::overwrite_existing - | fs::copy_options::copy_symlinks); + auto first = linglong::utils::copyDirectory(src_dir, dest_dir, matcher); + ASSERT_TRUE(first) << first.error().message(); + std::ofstream(src_dir / "file1.txt") << "changed source"; + std::ofstream(src_dir / "new.txt") << "new contents"; + auto result = linglong::utils::copyDirectory(src_dir, dest_dir, matcher); + ASSERT_TRUE(result) << result.error().message(); EXPECT_TRUE(fs::exists(dest_dir / "file1.txt")); std::ifstream ifs(dest_dir / "file1.txt"); std::string content((std::istreambuf_iterator(ifs)), (std::istreambuf_iterator())); EXPECT_EQ(content, "content1"); + EXPECT_TRUE(fs::is_regular_file(dest_dir / "new.txt")); EXPECT_TRUE(fs::is_directory(dest_dir / "subdir1")); EXPECT_TRUE(fs::exists(dest_dir / "subdir1" / "file2.txt")); @@ -152,7 +160,8 @@ TEST_F(FileTest, CopyDirectory_DestinationExists) return path.filename() != "ignored.txt"; }; - linglong::utils::copyDirectory(src_dir, dest_dir, matcher); + auto result = linglong::utils::copyDirectory(src_dir, dest_dir, matcher); + ASSERT_TRUE(result) << result.error().message(); EXPECT_TRUE(fs::exists(dest_dir / "file1.txt")); std::ifstream ifs(dest_dir / "file1.txt"); std::string content((std::istreambuf_iterator(ifs)), (std::istreambuf_iterator())); @@ -170,6 +179,125 @@ TEST_F(FileTest, CopyDirectory_DestinationExists) EXPECT_FALSE(fs::exists(dest_dir / "ignored.txt")); } +TEST_F(FileTest, CopyDirectory_CreatesEmptyDestinationWithSourcePermissions) +{ + const auto emptySource = src_dir / "empty"; + const auto destination = dest_dir / "parent/empty"; + fs::create_directory(emptySource); + fs::permissions(emptySource, fs::perms::owner_all); + + auto result = linglong::utils::copyDirectory(emptySource, destination); + ASSERT_TRUE(result) << result.error().message(); + EXPECT_TRUE(fs::is_directory(destination)); + EXPECT_TRUE(fs::is_empty(destination)); + EXPECT_EQ(fs::status(destination).permissions(), fs::status(emptySource).permissions()); +} + +TEST_F(FileTest, CopyDirectory_PreservesEmptyDirectoriesAndSymlinks) +{ + fs::create_directory(src_dir / "subdir1/empty"); + fs::create_symlink("subdir1", src_dir / "directory-link"); + fs::create_symlink("missing", src_dir / "dangling-link"); + fs::create_symlink(".", src_dir / "cycle-link"); + + auto result = linglong::utils::copyDirectory(src_dir, dest_dir); + ASSERT_TRUE(result) << result.error().message(); + EXPECT_TRUE(fs::is_directory(dest_dir / "subdir1/empty")); + EXPECT_TRUE(fs::is_empty(dest_dir / "subdir1/empty")); + for (const auto &name : { "directory-link", "dangling-link", "cycle-link" }) { + ASSERT_TRUE(fs::is_symlink(dest_dir / name)) << name; + EXPECT_EQ(fs::read_symlink(dest_dir / name), fs::read_symlink(src_dir / name)); + } +} + +TEST_F(FileTest, CopyDirectory_SkipsSocketsAndFifos) +{ + const auto socketPath = (src_dir / "socket").string(); + linglong::utils::fd::UniqueFd socket(::socket(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0)); + ASSERT_TRUE(socket) << errno; + sockaddr_un address{}; + address.sun_family = AF_UNIX; + ASSERT_LT(socketPath.size(), sizeof(address.sun_path)); + std::copy(socketPath.begin(), socketPath.end(), address.sun_path); + ASSERT_EQ(::bind(socket.get(), reinterpret_cast(&address), sizeof(address)), + 0) + << errno; + ASSERT_EQ(::mkfifo((src_dir / "subdir1/fifo").c_str(), 0600), 0) << errno; + + auto result = linglong::utils::copyDirectory(src_dir, dest_dir); + ASSERT_TRUE(result) << result.error().message(); + EXPECT_TRUE(fs::is_regular_file(dest_dir / "subdir1/file2.txt")); + EXPECT_FALSE(fs::exists(dest_dir / "socket")); + EXPECT_FALSE(fs::exists(dest_dir / "subdir1/fifo")); + EXPECT_TRUE(fs::is_socket(src_dir / "socket")); + EXPECT_TRUE(fs::is_fifo(src_dir / "subdir1/fifo")); +} + +TEST_F(FileTest, CopyDirectory_SkipsExistingDanglingSymlinks) +{ + fs::create_symlink("missing", dest_dir / "symlink1"); + + auto result = linglong::utils::copyDirectory(src_dir, dest_dir); + ASSERT_TRUE(result) << result.error().message(); + EXPECT_EQ(fs::read_symlink(dest_dir / "symlink1"), "missing"); +} + +TEST_F(FileTest, CopyDirectory_KeepsDanglingDestinationLinksForRegularFiles) +{ + const auto missingTarget = src_dir / "missing"; + fs::create_symlink(missingTarget, dest_dir / "file1.txt"); + + auto result = linglong::utils::copyDirectory(src_dir, dest_dir); + ASSERT_TRUE(result) << result.error().message(); + EXPECT_EQ(fs::read_symlink(dest_dir / "file1.txt"), missingTarget); + EXPECT_FALSE(fs::exists(missingTarget)); +} + +TEST_F(FileTest, CopyDirectory_MatcherExcludesFilesInsideMatchedDirectories) +{ + auto matcher = [](const fs::path &path) { + return path.filename() != "file2.txt"; + }; + auto result = linglong::utils::copyDirectory(src_dir, dest_dir, matcher); + ASSERT_TRUE(result) << result.error().message(); + EXPECT_FALSE(fs::exists(dest_dir / "subdir1/file2.txt")); + EXPECT_TRUE(fs::is_regular_file(dest_dir / "subdir1/subdir2/file3.txt")); +} + +TEST_F(FileTest, CopyDirectory_ReportsMissingSource) +{ + const auto missing = src_dir / "missing"; + auto result = linglong::utils::copyDirectory(missing, dest_dir / "output"); + ASSERT_FALSE(result); + EXPECT_NE(result.error().message().find(missing.string()), std::string::npos); + EXPECT_FALSE(fs::exists(dest_dir / "output")); +} + +TEST_F(FileTest, CopyDirectory_ReportsDestinationConflicts) +{ + std::ofstream(dest_dir / "subdir1") << "existing file"; + + auto result = linglong::utils::copyDirectory(src_dir, dest_dir); + ASSERT_FALSE(result); + EXPECT_NE(result.error().message().find((dest_dir / "subdir1").string()), std::string::npos); + EXPECT_TRUE(fs::is_regular_file(dest_dir / "subdir1")); +} + +TEST_F(FileTest, CopyDirectory_ReportsPermissionDenied) +{ + if (::geteuid() == 0) { + GTEST_SKIP() << "root can traverse unreadable directories"; + } + const auto unreadable = src_dir / "subdir1"; + const auto permissions = fs::status(unreadable).permissions(); + fs::permissions(unreadable, fs::perms::none); + auto result = linglong::utils::copyDirectory(src_dir, dest_dir); + fs::permissions(unreadable, permissions); + fs::permissions(dest_dir / "subdir1", permissions); + + ASSERT_FALSE(result); +} + TEST_F(FileTest, MoveFiles) { auto matcher = [](const fs::path &path) { diff --git a/libs/utils/src/linglong/utils/file.cpp b/libs/utils/src/linglong/utils/file.cpp index da396f8e2..8175b197a 100644 --- a/libs/utils/src/linglong/utils/file.cpp +++ b/libs/utils/src/linglong/utils/file.cpp @@ -189,40 +189,86 @@ calculateDirectorySize(const std::filesystem::path &dir) noexcept return size; } -// recursive copy src to dest with matcher -// symlinks are preserved -void copyDirectory(const std::filesystem::path &src, - const std::filesystem::path &dest, - std::function matcher, - std::filesystem::copy_options options) +linglong::utils::error::Result +copyDirectory(const std::filesystem::path &src, + const std::filesystem::path &dest, + std::function matcher) { + LINGLONG_TRACE("copy directory"); + std::error_code ec; - for (const auto &entry : std::filesystem::recursive_directory_iterator( - src, - std::filesystem::directory_options::skip_permission_denied, - ec)) { - const auto &fromPath = entry.path(); - auto relativePath = fromPath.lexically_relative(src); + auto iterator = std::filesystem::recursive_directory_iterator(src, ec); + if (ec) { + return LINGLONG_ERR(fmt::format("failed to iterate directory {}", src), ec); + } - if (matcher && !matcher(relativePath)) { - continue; + if (!dest.parent_path().empty()) { + std::filesystem::create_directories(dest.parent_path(), ec); + if (ec) { + return LINGLONG_ERR(fmt::format("failed to create directory {}", dest.parent_path()), + ec); } + } + std::filesystem::create_directory(dest, src, ec); + if (ec) { + return LINGLONG_ERR(fmt::format("failed to create directory {}", dest), ec); + } - const auto toPath = dest / relativePath; - LogD("{} -> {}", fromPath, toPath); - - std::filesystem::create_directories(toPath.parent_path(), ec); + const auto end = std::filesystem::recursive_directory_iterator{}; + while (iterator != end) { + const auto &fromPath = iterator->path(); + const auto status = iterator->symlink_status(ec); if (ec) { - LogW("failed to create directory {}: {}", toPath.parent_path(), ec.message()); - continue; + return LINGLONG_ERR(fmt::format("failed to inspect source entry {}", fromPath), ec); + } + const auto relativePath = fromPath.lexically_relative(src); + if ((!matcher || matcher(relativePath)) + && (std::filesystem::is_directory(status) || std::filesystem::is_regular_file(status) + || std::filesystem::is_symlink(status))) { + const auto toPath = dest / relativePath; + LogD("{} -> {}", fromPath, toPath); + std::filesystem::create_directories(toPath.parent_path(), ec); + if (ec) { + return LINGLONG_ERR( + fmt::format("failed to create directory {}", toPath.parent_path()), + ec); + } + + if (std::filesystem::is_directory(status)) { + std::filesystem::create_directory(toPath, fromPath, ec); + } else { + // Inspect the link itself so dangling destination links are also kept. + const auto targetStatus = std::filesystem::symlink_status(toPath, ec); + if (ec == std::errc::no_such_file_or_directory) { + ec.clear(); + } + if (ec) { + return LINGLONG_ERR(fmt::format("failed to inspect destination {}", toPath), + ec); + } + if (!std::filesystem::exists(targetStatus)) { + if (std::filesystem::is_regular_file(status)) { + std::filesystem::copy_file(fromPath, + toPath, + std::filesystem::copy_options::skip_existing, + ec); + } else if (std::filesystem::is_symlink(status)) { + std::filesystem::copy_symlink(fromPath, toPath, ec); + } + } + } + if (ec) { + return LINGLONG_ERR(fmt::format("failed to copy {} to {}", fromPath, toPath), ec); + } } - // preserve symlinks - std::filesystem::copy(fromPath, toPath, options, ec); + + iterator.increment(ec); if (ec) { - LogW("failed to copy {} to {}: {}", fromPath, toPath, ec.message()); - continue; + return LINGLONG_ERR(fmt::format("failed to iterate directory {}", src), ec); } } + + return LINGLONG_OK; } linglong::utils::error::Result diff --git a/libs/utils/src/linglong/utils/file.h b/libs/utils/src/linglong/utils/file.h index b76460f41..f27a950fe 100644 --- a/libs/utils/src/linglong/utils/file.h +++ b/libs/utils/src/linglong/utils/file.h @@ -6,6 +6,7 @@ #include "linglong/utils/error/error.h" #include +#include #include namespace linglong::utils { @@ -24,12 +25,13 @@ linglong::utils::error::Result concatFile(const std::filesystem::path &sou linglong::utils::error::Result calculateDirectorySize(const std::filesystem::path &dir) noexcept; -void copyDirectory( - const std::filesystem::path &src, - const std::filesystem::path &dest, - std::function matcher = {}, - std::filesystem::copy_options options = std::filesystem::copy_options::copy_symlinks - | std::filesystem::copy_options::skip_existing); +// Merge matching entries into dest without following symlinks. Special files are skipped. +// Unmatched directories are still traversed so their matching descendants can be copied. +// Existing files and symlinks are kept; existing directories are merged. +[[nodiscard]] linglong::utils::error::Result +copyDirectory(const std::filesystem::path &src, + const std::filesystem::path &dest, + std::function matcher = {}); linglong::utils::error::Result moveFiles(const std::filesystem::path &src, diff --git a/misc/libexec/linglong/fetch-archive-source b/misc/libexec/linglong/fetch-archive-source index ef7ba1eb4..ec2a7bb1f 100755 --- a/misc/libexec/linglong/fetch-archive-source +++ b/misc/libexec/linglong/fetch-archive-source @@ -1,47 +1,34 @@ #!/bin/sh # SPDX-FileCopyrightText: 2023 - 2026 UnionTech Software Technology Co., Ltd. -# # SPDX-License-Identifier: LGPL-3.0-or-later + set -e -outputdir=$1 -url=$2 -digest=$3 -cachedir=$4 +action=$1 +input=$2 +output=$3 + +case "$action" in +download) + if ! command -v wget >/dev/null 2>&1; then + echo "wget not found, please install wget first" >&2 + exit 1 + fi -# Check command tools -if ! command -v wget -then - echo "wget not found, please install wget first" - exit 1; -fi + wget "$input" -O "$output" + ;; +materialize) + if ! command -v tar >/dev/null 2>&1; then + echo "tar not found, please install tar first" >&2 + exit 1 + fi -# Clean up old directory and create parent directory -mkdir -p "$outputdir" -rm -r "$outputdir" -# Check cache -if [ -d "$cachedir/archive_$digest" ]; then - cp -r "$cachedir/archive_$digest" "$outputdir" - exit; -fi -# Create a temporary directory -tmpdir=$(mktemp -d) -# Change directory -cd "$tmpdir" -# Download dsc and tar -name=$(basename "$url") -wget "$url" -O "$name" -# Compare digest -actual_hash=$(sha256sum "$name" | awk '{print $1}') -if [ "X$actual_hash" != "X$digest" ]; then - echo "File SHA256 digest is $actual_hash, expected $digest" - exit 1; -fi -# Extract the archive -mkdir -p "$cachedir/tmp_$digest" -tar --no-same-owner -xvf "$name" -C "$cachedir/tmp_$digest" -mv "$cachedir/tmp_$digest" "$cachedir/archive_$digest" -cp -r "$cachedir/archive_$digest" "$outputdir" -# Clean temporary directory -rm -r "$tmpdir" + mkdir "$output" + tar --no-same-owner -xf "$input" -C "$output" + ;; +*) + echo "unsupported archive source action: $action" >&2 + exit 1 + ;; +esac diff --git a/misc/libexec/linglong/fetch-dsc-source b/misc/libexec/linglong/fetch-dsc-source index 5a0d27a1f..519524c82 100755 --- a/misc/libexec/linglong/fetch-dsc-source +++ b/misc/libexec/linglong/fetch-dsc-source @@ -1,48 +1,46 @@ #!/bin/sh # SPDX-FileCopyrightText: 2023 - 2026 UnionTech Software Technology Co., Ltd. -# # SPDX-License-Identifier: LGPL-3.0-or-later + set -e -outputdir=$1 -url=$2 -digest=$3 -cachedir=$4 - -# Check command tools -if ! command -v dget -then - echo "dget not found, please install devscripts first" - exit 1; -fi - -# Clean up old directory and create parent directory -mkdir -p "$outputdir" -rm -r "$outputdir" -# Check cache -if [ -d "$cachedir/dsc_$digest" ]; then - cp -r "$cachedir/dsc_$digest" "$outputdir" - exit; -fi -# Create a temporary directory -tmpdir=$(mktemp -d) -# change directory -cd "$tmpdir" -# Download dsc and tar -name=$(basename "$url") -dget -u -d "$url" -# Compare digest -actual_hash=$(sha256sum "$name" | awk '{print $1}') -if [ "X$actual_hash" != "X$digest" ]; then - echo "File SHA256 digest is $actual_hash, expected $digest" - exit 1; -fi -# Extract the archive -mkdir -p "$cachedir" -rm -r "$cachedir/tmp_$digest" || true -dpkg-source -x --no-copy "$name" "$cachedir/tmp_$digest" -mv "$cachedir/tmp_$digest" "$cachedir/dsc_$digest" -cp -r "$cachedir/dsc_$digest" "$outputdir" -# Clean temporary directory -rm -r "$tmpdir" +action=$1 +artifact=$2 +argument=$3 + +case "$action" in +download) + if ! command -v dget >/dev/null 2>&1; then + echo "dget not found, please install devscripts first" >&2 + exit 1 + fi + + mkdir "$argument" + cd "$argument" + dget -u -d "$artifact" + ;; +verify) + if ! command -v dscverify >/dev/null 2>&1; then + echo "dscverify not found, please install devscripts first" >&2 + exit 1 + fi + + dscverify --nosigcheck "$artifact/$argument" + ;; +materialize) + output=$4 + + if ! command -v dpkg-source >/dev/null 2>&1; then + echo "dpkg-source not found, please install dpkg-dev first" >&2 + exit 1 + fi + + cd "$artifact" + dpkg-source -x --no-copy "$argument" "$output" + ;; +*) + echo "unsupported dsc source action: $action" >&2 + exit 1 + ;; +esac diff --git a/misc/libexec/linglong/fetch-file-source b/misc/libexec/linglong/fetch-file-source index 883b957a8..c5a3ae9bd 100755 --- a/misc/libexec/linglong/fetch-file-source +++ b/misc/libexec/linglong/fetch-file-source @@ -1,33 +1,22 @@ #!/bin/sh -# SPDX-FileCopyrightText: 2023 UnionTech Software Technology Co., Ltd. -# +# SPDX-FileCopyrightText: 2023 - 2026 UnionTech Software Technology Co., Ltd. # SPDX-License-Identifier: LGPL-3.0-or-later + set -e -outputfile=$1 +action=$1 url=$2 -digest=$3 -cachedir=$4 +output=$3 -# Check command tools -if ! command -v wget -then - echo "wget not found, please install wget first" - exit 1; -fi -# Check cache -if [ -f "$cachedir/file_$digest" ]; then - # Using hard links is faster - cp --remove-destination --link "$cachedir/file_$digest" "$outputfile" || cp "$cachedir/file_$digest" "$outputfile" - exit; +if [ "$action" != "download" ]; then + echo "unsupported file source action: $action" >&2 + exit 1 fi -# Download file -wget "$url" -O "$cachedir/tmp_$digest" -actual_hash=$(sha256sum "$cachedir/tmp_$digest" | awk '{print $1}') -if [ "X$actual_hash" != "X$digest" ]; then - echo "File SHA256 digest is $actual_hash, expected $digest" - exit 1; + +if ! command -v wget >/dev/null 2>&1; then + echo "wget not found, please install wget first" >&2 + exit 1 fi -mv "$cachedir/tmp_$digest" "$cachedir/file_$digest" -cp --remove-destination --link "$cachedir/file_$digest" "$outputfile" || cp "$cachedir/file_$digest" "$outputfile" + +wget "$url" -O "$output" diff --git a/misc/libexec/linglong/fetch-git-source b/misc/libexec/linglong/fetch-git-source index 8836d5d5a..c64ae2da2 100755 --- a/misc/libexec/linglong/fetch-git-source +++ b/misc/libexec/linglong/fetch-git-source @@ -1,40 +1,35 @@ #!/bin/sh -# SPDX-FileCopyrightText: 2023 UnionTech Software Technology Co., Ltd. -# +# SPDX-FileCopyrightText: 2023 - 2026 UnionTech Software Technology Co., Ltd. # SPDX-License-Identifier: LGPL-3.0-or-later set -e -workdir=$1 +action=$1 url=$2 -commit=$3 -# cachedir=$4 - -# Check command tools -if ! command -v git -then - echo "git not found, please install git first" - exit 1; +ref=$3 +workdir=$4 + +if [ "$action" != "checkout" ]; then + echo "unsupported git source action: $action" >&2 + exit 1 fi -mkdir -p "$workdir" || true -cd "$workdir" - -# Init repo -if [ -d ".git" ]; then - git remote set-url origin "$url" -else - git init - git remote add origin "$url" + +if ! command -v git >/dev/null 2>&1; then + echo "git not found, please install git first" >&2 + exit 1 fi -# Fetch commit -git fetch origin "$commit" --depth 1 -n -git add :/ -git reset --hard FETCH_HEAD +mkdir "$workdir" +git -C "$workdir" init +git -C "$workdir" remote add origin "$url" -# Fetch submodule -if [ -n "$GIT_SUBMODULES" ]; then - git submodule update --init --recursive --depth 1 - git submodule foreach git reset --hard HEAD +git -C "$workdir" fetch origin "$ref" --depth 1 -n +git -C "$workdir" reset --hard FETCH_HEAD + +if [ -n "${GIT_SUBMODULES:-}" ]; then + git -C "$workdir" submodule update --init --recursive --depth 1 + git -C "$workdir" submodule foreach git reset --hard HEAD fi + +git -C "$workdir" rev-parse --verify HEAD