diff --git a/Makefile b/Makefile index 4a3a10578c..f1708d4d18 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ TEST_TIMEOUT := -timeout 10m TEST_BINARY := $(CURDIR)/dist/test/crw TEST_ENV := CRW_TEST_BINARY=$(TEST_BINARY) -.PHONY: build test test-binary test-part lint dist crw-dev gui gui-assets +.PHONY: build test test-binary test-part lint dist crw-dev gui gui-assets ci-local build: @if ! $(GO) list ./... 2>/dev/null | grep -q .; then echo "no Go packages yet: build skipped"; else $(GO) build -o $(BINARY) -trimpath -ldflags="$(LDFLAGS)" ./cmd/crw; fi @@ -83,6 +83,13 @@ endif crw-dev: $(GO) build -tags dev -o dist/crw-dev ./cmd/crw-dev +# The local full verification (CRW-964): every job and step of .github/workflows/ci.yml, in a +# clean worktree of HEAD, with the result written to a verification-record/1. A heavy step goes +# through the gate CRW_CI_HEAVY_GATE names when it is set. make ci-local and +# go run -tags dev ./cmd/crw-dev ci local are the same run; docs/CI.md has the mapping. +ci-local: + $(GO) run -tags dev ./cmd/crw-dev ci local + # The screens: install from the committed lockfile, run the screen tests, build them into a # temporary tree under dist/ (gitignored, and never the committed internal/gui/assets), and refuse # a build that does not match the committed tree byte for byte. Node is needed here and in the gui diff --git a/cmd/crw-dev/main_test.go b/cmd/crw-dev/main_test.go index 236acda7ec..23afa2ea6b 100644 --- a/cmd/crw-dev/main_test.go +++ b/cmd/crw-dev/main_test.go @@ -18,7 +18,7 @@ func TestCommandTreeDispatches(t *testing.T) { {[]string{"nope"}, 2, "", `invalid command "nope"`}, {[]string{"--help"}, 0, "usage: crw-dev {ci,cxc,fuzz,skills,stop-events,trial-ledger}", ""}, {[]string{"ci"}, 2, "", "crw-dev ci: error: the following arguments are required: check"}, - {[]string{"ci", "--help"}, 0, "usage: crw-dev ci {contracts,dispatch-cases,gui-drift,operations,plugin,refactor-backlog,validate}", ""}, + {[]string{"ci", "--help"}, 0, "usage: crw-dev ci {contracts,dispatch-cases,gui-drift,local,operations,plugin,refactor-backlog,validate}", ""}, {[]string{"ci", "nope"}, 2, "", `invalid choice: "nope"`}, {[]string{"fuzz"}, 2, "", "crw-dev fuzz: error: the following arguments are required: target"}, {[]string{"fuzz", "nope", "--cases", "1"}, 2, "", `crw-dev fuzz: error: unknown target "nope"`}, diff --git a/docs/CI.md b/docs/CI.md index 4e1b4df316..0191c077cb 100644 --- a/docs/CI.md +++ b/docs/CI.md @@ -25,6 +25,146 @@ Installing and operating the runtime is [runtime installation](runtime-install.m under `scripts/ci` and `scripts/check_operations_contract.py` left in refactor R3 (decision R3R-1). See the [workflow](../.github/workflows/ci.yml) for the exact job inputs. +## The local run + +`crw-dev ci local` (and `make ci-local`) runs every job and step of +[the workflow](../.github/workflows/ci.yml) locally, in the same order and at the same pinned +versions, and writes a `verification-record/1`. It exists so the whole verification can be +run without GitHub; the hosted run stays the merge evidence. + +The run makes a **clean worktree** of the commit being verified under the work root (the XDG state directory, or --work-root; never TMPDIR, /tmp or /var/tmp) +(`git worktree add --detach`), with `HOME` and the XDG directories pointed into a +temporary home and `TZ=UTC`, so uncommitted changes in the caller's checkout and the host's +caches cannot change the result. The record names the commit and its tree, never the caller's +working state. The worktree is removed afterwards. `GOCACHE` and `NPM_CONFIG_CACHE` are inherited: they decide how fast a step runs, not what it decides. The module cache is not inherited, and GOFLAGS is set by the engine. + +Light mode and the body-only edit mirror are never applied: the local run is always full. + +### The local step to GitHub job mapping + +| ci.yml job | ci.yml step | local | +| --- | --- | --- | +| `validate` | the sparse `scripts/ci` checkout and the edit mirror | not run: a hosted body-only edit lookup | +| `validate` | checkout | the clean worktree of the verified commit | +| `validate` | `setup-go` | the pinned Go is resolved and recorded | +| `validate` | `go build -tags dev -o "$RUNNER_TEMP/crw-dev" ./cmd/crw-dev` | run | +| `validate` | `"$RUNNER_TEMP/crw-dev" ci validate` | run, over the record's range (`BLOB_RANGE_BASE`) | +| `validate` | `"$RUNNER_TEMP/crw-dev" ci plugin` | run | +| `validate` | `"$RUNNER_TEMP/crw-dev" ci contracts` | run | +| `secrets` | the mirror pair, checkout | as above | +| `secrets` | `bash scripts/ci/secrets.sh` | run, with `GITHUB_EVENT_NAME=pull_request` and `PR_BASE_SHA`, so it scans base..head as a pull request run does | +| `skill-scripts-node` | the mirror pair, checkout | as above | +| `skill-scripts-node` | the changed-path decision | run: the same script, its answer recorded; the local run performs every step either way | +| `skill-scripts-node` | `setup-node` | the pinned Node is resolved and recorded | +| `skill-scripts-node` | the staged skill-script tests | run | +| `gui` | the mirror pair, checkout | as above | +| `gui` | the changed-path decision (`scripts/ci/gui_paths.sh`) | run, its answer recorded | +| `gui` | `setup-go`, `setup-node` | as above | +| `gui` | `npm ci`, `npm test`, `npm run build -- --outDir "$RUNNER_TEMP/gui-built" --emptyOutDir` | run (in `web/`) | +| `gui` | `ci gui-drift --built "$RUNNER_TEMP/gui-built"` | run | +| `go-product` | the light mode notice | not run: light mode is never applied | +| `go-product` | the mirror pair, checkout, `setup-go` | as above | +| `go-product` | `make lint` (leg `lint`) | run | +| `go-product` | `make test-part TEST_PART=` (legs `test-1`..`test-rest`) | run, one leg at a time | +| `go-product` | the three `make dist` builds and `sha256sum` (leg `dist`) | run | +| `go-product` | `CRW_TEST_BINARY=... go test -tags integration ./internal/runtime/integration/...` (leg `dist`) | run | +| `go-product` | the four `upload-artifact` steps | not run: nothing is uploaded locally | +| `dev-gate` | "Require every prerequisite to succeed" | run: every prerequisite job must have passed | + +`internal/dev/ci/local_plan_test.go` holds the table to the workflow job by job and step by +step, and the runner refuses a ci.yml the table does not cover. + +### The verification record + +The record is `verification-record/1`: `repository`, `baseCommit`, +`headCommit`, `treeHash`, `ciDigest` (the sha256 of ci.yml), +`tools` (Go as the module at the commit selects it, probed with the steps' isolation, and Node as observed on the host; gitleaks and staticcheck at the version +their step runs, which is the pin, because secrets.sh runs its own pinned Gitleaks and the lint +leg runs the staticcheck the tree requires), `pins` (the versions the tree +pins: go.mod's toolchain and staticcheck, ci.yml's Node, secrets.sh's Gitleaks), +`pinMismatch`, `goFlags` (the GOFLAGS the steps inherit, without the flags that select or skip tests, so a full run runs every test; a record whose GOFLAGS name a modfile or an overlay is never reused), `range` (the sha256 of the commits between base and head, the input of the blob and secret steps), `goEnv` (GOENV is unset in the +steps, so Go reads its default file under the run's own empty home, and the host's GOENV never +applies), +`dependencies` (the sha256 of `go.sum` and +`web/package-lock.json`), `os`, `arch`, `result`, `runner`, and `jobs` +— one entry per GitHub check, each with its steps' `command`, `scope`, +`result`, `seconds` and `reason`. `digest` is the sha256 of the +canonical serialization: the record as JSON with its keys sorted, no indentation and no HTML +escaping, and the `digest` member removed. A tool whose observed version differs from its +pin is named in `pinMismatch`, and such a record is never reused. + +A step runs under `bash --noprofile --norc -eo pipefail`, as the runner does, so its first +failed command fails it. A failed step's `reason` gives the exit status, the failing test lines +of its output (a `--- FAIL`, `FAIL` or `panic:` line, at most 20) and the last 40 lines of the +output, at most 4 KB; a Go test that outlived its `-timeout` is named with a `timeout:` prefix, +so a load timeout reads as a timeout, not as a verdict on the code. The steps inherit `TMPDIR` +and `PATH`, so a test that makes a socket or a temporary file uses the caller's temporary root. + +A step's result is `passed`, `failed`, `missing_tool`, +`skipped` or `not_applicable`. The whole result is `pass` only when no step +is `failed`, `missing_tool` or `skipped`: one step that fails, does not run +or lacks its tool fails the run, and the record names it with its reason. + +### Sealed runs + +A step runs in a sealed environment built from an allowlist, not from the caller's environment +minus a denylist: HOME and the XDG directories point into the run's own home, TZ is UTC, +GOTOOLCHAIN is local, and GOFLAGS is set by the engine alone (its parallelism, from +--parallel). The caller's other variables do not reach a step, and the record names the Go, +cgo, Node and npm variables it left out in ignoredEnv (names only). A caller's GOFLAGS with +-exec, -toolexec, -overlay, -modfile, -run or -count therefore cannot change what a step runs. + +Each job's Node pin is read from its own setup-node step in the commit's ci.yml, not from a +shared first match. The host's Node is observed in the same isolation, and a job whose pin +differs is named in pinMismatch as `job:node`; such a record is kept but never reused. The +gitleaks and staticcheck pins are the ones secrets.sh and go.mod fix, and the steps fetch +or run those pins, so they are recorded at the pin. + +The clean worktree is made under a work root: the directory given by --work-root, or the XDG +state directory of the real user. A root inside TMPDIR, /tmp or /var/tmp is refused with the +name work_root_in_tmp, because the repository's own checkout tests assert the checkout lies +outside those directories. + +### Reuse validation + +A record answers a run only when every key above matches and the record is sound: its digest +is valid, it is sealed, it was made from the plan the commit's table gives (planDigest), it +matches that plan one job and one step at a time (name, command and scope), every job and +step passed (or is a not-applicable step the plan marks as such), its result equals the result its steps recompute to, and it has +no pinMismatch. Any other record is not reused: the engine runs the table again and says why. + +### Reuse + +`--reuse ` answers an existing record instead of running only when the **tree**, the +**ci.yml digest**, the **tool versions**, the **dependency digests**, the **OS and +architecture** and the **base commit** and the **commit range** all match, +and the record passed with no pin mismatch. The head commit is not a key on its own. The commit range (the commits between base and head) is one: a rebase or an amend that changes that list reruns the checks, and a record is reused only when the list and the tree both match. Changing any one key re-runs. + +### The heavy-check gate and TMPDIR + +A heavy step (a Go build, vet, test, the dist builds, the integration test, `npm ci`) runs +through the command named by `CRW_CI_HEAVY_GATE` when it is set — ` ` +— and directly when it is unset. The gate is the host's; no host path is written into the +repository. Temporary files go under `TMPDIR`, which the caller sets. +Only the gate's executable is resolved, once, against the directory the run starts in. Its other +arguments are not resolved: a relative argument is read in each step's own directory, so give absolute paths. + +### The pre-push hook + +`crw-dev ci local hook install` writes a `pre-push` hook into the repository the +working directory belongs to (`git rev-parse --git-path hooks/pre-push`), and +`... hook status` reports `installed`, `absent` or `foreign`. The hook +refuses a push whose range (`..`, or the whole local sha for a new +ref) brings in a blob over 2 MiB or a secret Gitleaks finds — the same two checks the hosted +`secrets` and `validate` jobs run, moved before the push because a public history +cannot drop what has already been pushed. It fails closed: a missing Gitleaks or an unreadable +range blocks the push. An existing `pre-push` this tool did not write is left untouched and +the install refuses; the tool writes no git config. The hook's size rule carries no allow list, +unlike `crw-dev ci validate`: the list is empty today, and a future entry must be paired +with the hook at that time. Tests install the hook only into temporary repositories; the real +shared checkout is the operator's to install. + + ## The workflow Every job runs on every event: a pull request (GitHub's merge candidate), a push to `dev` (the diff --git a/internal/dev/ci/ci.go b/internal/dev/ci/ci.go index 19bae45dac..343627acf4 100644 --- a/internal/dev/ci/ci.go +++ b/internal/dev/ci/ci.go @@ -19,6 +19,7 @@ var Checks = map[string]Check{ "contracts": Contracts, "dispatch-cases": DispatchCases, "gui-drift": GuiDrift, + "local": Local, "operations": OperationsContract, "plugin": Plugin, "refactor-backlog": RefactorBacklog, diff --git a/internal/dev/ci/ci_test.go b/internal/dev/ci/ci_test.go index 9662a3f078..9bb7857f6e 100644 --- a/internal/dev/ci/ci_test.go +++ b/internal/dev/ci/ci_test.go @@ -10,7 +10,7 @@ import ( // An unknown check is a usage error that names the argument and the checks there are. func TestAnUnknownCheckIsNamed(t *testing.T) { got := runCommand(t, repoRoot(), nil, crwDev, "ci", "nope\t") - line := `crw-dev ci: error: invalid choice: "nope\t" (choose from contracts, dispatch-cases, gui-drift, operations, plugin, refactor-backlog, validate)` + "\n" + line := `crw-dev ci: error: invalid choice: "nope\t" (choose from contracts, dispatch-cases, gui-drift, local, operations, plugin, refactor-backlog, validate)` + "\n" if got.code != 2 || !strings.HasSuffix(got.stderr, line) { t.Errorf("crw-dev ci nope: exit %d\n%s\nwant the line %s", got.code, got.stderr, line) } diff --git a/internal/dev/ci/local.go b/internal/dev/ci/local.go new file mode 100644 index 0000000000..333961237a --- /dev/null +++ b/internal/dev/ci/local.go @@ -0,0 +1,935 @@ +//go:build dev + +package ci + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "io" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" + "time" +) + +// CRW-964: `crw-dev ci local`. It runs the local step table in a clean worktree of the +// commit being verified, with an isolated home and UTC, and writes a verification-record/1. The +// clean tree is what makes the result the commit's rather than the caller's working state, and the +// record is what makes it reusable when nothing that decides it has changed. + +const ( + // localRecordDefault is where the record is written when --record is not given. + localRecordDefault = ".codexclaw/ci-local/verification-record.json" + // localHeavyGateEnv names the host's heavy-check gate. When it is set, a heavy step runs as + // ; when it is unset the command runs directly. No host path is ever written + // into the repository. + localHeavyGateEnv = "CRW_CI_HEAVY_GATE" + // localTempPrefix names the temporary root a run makes for its worktree and its home. + localTempPrefix = "crw-ci-local-" +) + +// localOptions is one local run's inputs. Plan is the table (the default is localPlan); tests pass +// a smaller one. +type localOptions struct { + Root string + Commit string + Base string + Record string + Runner string + HeavyGate string + Keep bool + // WorkRoot is the directory the clean worktree is made in; it may not be inside TMPDIR, /tmp or /var/tmp. + WorkRoot string + // Parallel is the Go test parallelism every step runs with (GOFLAGS=-p=N); 0 means 4. + Parallel int + // allowTempRoot lets a test work root sit under TMPDIR. Only tests set it; the command never does. + allowTempRoot bool + Plan []localJob + // Env is extra environment for every step, after the isolated home and TZ (tests use it). + Env []string + // output is the file the changed-path decision writes its answer to, set once per run. + output string + // script counts the step scripts written under the run's temporary root. + script int +} + +// Local is `crw-dev ci local`: run every ci.yml job and step locally, or install the +// pre-push hook. `crw-dev ci local hook install|status` is the hook subcommand. +func Local(args []string, stdout, stderr io.Writer) int { + localScrubGitEnv() + if len(args) > 0 && args[0] == "hook" { + return localHook(args[1:], stdout, stderr) + } + flags := newFlags("local") + commit := flags.String("commit", "HEAD", "the commit to verify") + base := flags.String("base", "", "the base commit the record names (default: merge-base with origin/dev)") + record := flags.String("record", localRecordDefault, "where to write the verification record") + reuse := flags.String("reuse", "", "answer this record instead of running, when every key matches") + heavyGate := flags.String("heavy-gate", "", "the heavy-check gate to run heavy steps through (default $"+localHeavyGateEnv+")") + runner := flags.String("runner", "local", "where this run happened; the record names it") + keep := flags.Bool("keep", false, "keep the clean worktree for debugging") + workRoot := flags.String("work-root", "", "the directory the clean worktree is made in (default: the XDG state directory); never inside TMPDIR, /tmp or /var/tmp") + parallel := flags.Int("parallel", 4, "the Go test parallelism every step runs with (GOFLAGS=-p=N)") + description := "Run every job and step of .github/workflows/ci.yml locally, in a clean worktree of the\n" + + "commit being verified, and write a verification-record/1. The record is answered without\n" + + "running again only when the tree, the ci.yml digest, the tool versions, the dependency\n" + + "digests and the OS and architecture all match." + if code := parseFlags(flags, description, args, stdout, stderr); code >= 0 { + return code + } + root, err := repositoryRoot() + if err != nil { + return failf(stderr, "local: %s", err) + } + opts := localOptions{ + Root: root, + Commit: *commit, + Base: *base, + Record: *record, + Runner: *runner, + HeavyGate: *heavyGate, + Keep: *keep, + WorkRoot: *workRoot, + Parallel: *parallel, + } + if opts.Record == "" { + opts.Record = filepath.Join(root, localRecordDefault) + } else if !filepath.IsAbs(opts.Record) { + opts.Record = filepath.Join(root, opts.Record) + } + if opts.HeavyGate == "" { + opts.HeavyGate = os.Getenv(localHeavyGateEnv) + } + reusePath := "" + if *reuse != "" { + reusePath = localAbs(root, *reuse) + } + made, reused, err := localVerify(opts, reusePath, stdout) + if err != nil { + return failf(stderr, "local: %s", err) + } + if reused { + fmt.Fprintf(stdout, "local: reused %s (%s)\n", *reuse, made.Digest) + return 0 + } + sealed, err := writeRecord(opts.Record, made) + if err != nil { + return failf(stderr, "local: %s", err) + } + if sealed.Result == localPass { + fmt.Fprintf(stdout, "local: pass at %s (%s)\n", sealed.HeadCommit, sealed.Digest) + return 0 + } + fmt.Fprintf(stderr, "local: fail at %s (%s)\n", sealed.HeadCommit, sealed.Digest) + for _, job := range made.Jobs { + for _, step := range job.Steps { + if step.Result != localPassed && step.Result != localNotApplicableResult { + fmt.Fprintf(stderr, " %s / %s: %s (%s)\n", job.Name, step.Name, step.Result, step.Reason) + } + } + } + return 1 +} + +// localVerify answers a reused record when every key matches, or runs the table and returns the +// record it made. +func localVerify(opts localOptions, reusePath string, stdout io.Writer) (verificationRecord, bool, error) { + localScrubGitEnv() + opts.HeavyGate = localAbsGate(opts.HeavyGate) + // Replacement refs (refs/replace) would let another object stand in for a commit; the run reads the + // objects as they are. + os.Setenv("GIT_NO_REPLACE_OBJECTS", "1") + plan := opts.Plan + if plan == nil { + plan = localPlan() + } + current, err := localCurrentKeys(opts) + if err != nil { + return verificationRecord{}, false, err + } + if reusePath != "" { + // A record path that does not exist is a usage error, not a silent full run: the caller + // asked for that record. + reused, err := readRecord(reusePath) + if err != nil { + return verificationRecord{}, false, fmt.Errorf("--reuse: %w", err) + } + ok, why := localReuse(reused, current) + if ok { + ok, why = localValidateReuse(reused, plan) + } + if ok { + ok, why = localRecomputedReuse(opts, plan, current) + } + if ok { + // The record answers: write it to --record so the caller finds it where it asked. + if _, err := writeRecord(opts.Record, reused); err != nil { + return verificationRecord{}, false, err + } + fmt.Fprintf(stdout, "local: every key matches (%s); %s answers this tree\n", why, reusePath) + return reused, true, nil + } + fmt.Fprintf(stdout, "local: %s does not answer this tree: %s\n", reusePath, why) + } + // The workflow-coverage refusal guards the real table: a step ci.yml grows cannot go + // unverified. An injected table (tests) owns its own coverage. The workflow is read from the + // verified commit, not the caller's working tree, so a dirty edit to ci.yml cannot change what + // this run is held to. + if opts.Plan == nil { + if err := checkWorkflowAt(opts.Root, current.HeadCommit, plan); err != nil { + return verificationRecord{}, false, err + } + } + return localExecute(opts, plan, current, stdout) +} + +// localCurrentKeys is the state a record is judged against: the tree, the ci.yml digest, the tool +// versions, the dependency digests and the platform. It is computed from the verified commit, not +// from the caller's working tree, so a dirty checkout does not change it (comment-c6). +func localCurrentKeys(opts localOptions) (verificationRecord, error) { + head, err := localRev(opts.Root, opts.Commit+"^{commit}") + if err != nil { + return verificationRecord{}, err + } + tree, err := localRev(opts.Root, head+"^{tree}") + if err != nil { + return verificationRecord{}, err + } + ciDigest, err := localBlobDigest(opts.Root, head, ".github/workflows/ci.yml") + if err != nil { + return verificationRecord{}, err + } + dependencies := map[string]string{} + for _, name := range []string{"go.sum", "web/package-lock.json"} { + digest, err := localBlobDigest(opts.Root, head, name) + if err != nil { + return verificationRecord{}, err + } + dependencies[name] = digest + } + // The range the blob and secret steps judge is base..head, so the base is part of what the + // record covers: a different base is a different verification even at the same tree. + base := opts.Base + if base == "" { + base, err = localDefaultBase(opts.Root, head) + if err != nil { + return verificationRecord{}, err + } + } + // The base is recorded as the full commit it names, however it was given. + if base != "" { + base, err = localRev(opts.Root, base+"^{commit}") + if err != nil { + return verificationRecord{}, err + } + } + // The commits between base and head are the input the blob and secret steps judge, so the record + // names them: a different commit list is a different verification even at the same tree. + commits, err := runGit(opts.Root, "rev-list", base+".."+head) + if err != nil { + return verificationRecord{}, err + } + rangeSum := sha256.Sum256(commits) + // The Go probe runs in a module holding the commit's go.mod, so it reports the toolchain the steps select. + goMod, _ := runGit(opts.Root, "show", head+":go.mod") + pins, err := localToolPinsFrom(func(path string) ([]byte, error) { return runGit(opts.Root, "show", head+":"+path) }) + if err != nil { + return verificationRecord{}, err + } + return verificationRecord{ + Schema: recordSchema, + Runner: opts.Runner, + Range: fmt.Sprintf("sha256:%x", rangeSum), + Repository: localRepository(opts.Root), + BaseCommit: base, + HeadCommit: head, + TreeHash: tree, + CiDigest: ciDigest, + Tools: localObservedVersions(localToolVersionsIn(localPathEnv(opts.Env), goMod, opts.HeavyGate), pins), + GoFlags: localEngineFlags(opts.Parallel), + HeavyGate: localGateDigest(opts.HeavyGate), + GoEnv: localIsolatedGoEnv, + Dependencies: dependencies, + OS: localHostOS(), + Arch: localHostArch(), + }, nil +} + +// localEngineFlags is the GOFLAGS every step runs with: the engine's own parallelism. The caller's +// GOFLAGS never reach a step, so the record names the engine's value. +func localEngineFlags(parallel int) string { + if parallel <= 0 { + parallel = 4 + } + return fmt.Sprintf("-p=%d", parallel) +} + +// localRepository is the origin remote's URL, the identity a record names. It is read from the +// checkout rather than written down, so the record travels with the repository it describes. +func localRepository(root string) string { + out, err := runGit(root, "remote", "get-url", "origin") + if err != nil { + return "" + } + return localWithoutUserinfo(strings.TrimSpace(string(out))) +} + +// localWithoutUserinfo removes the credentials from a remote address before the record names it: the user, +// password and query of a URL, or the user@ of an scp-style address (pre-merge finding d1). A URL that +// url.Parse rejects, such as one whose password holds a % or a space that git accepts verbatim, is cut +// textually at its last @; when the text before that @ holds a slash, the boundary is unknown and the +// name is empty rather than the raw remote. +func localWithoutUserinfo(remote string) string { + if u, err := url.Parse(remote); err == nil && u.Scheme != "" && u.Host != "" { + u.User = nil + u.RawQuery = "" + u.Fragment = "" + return u.String() + } + if i := strings.Index(remote, "://"); i >= 0 { + rest := remote[i+len("://"):] + at := strings.LastIndex(rest, "@") + tail := rest + if at >= 0 { + if strings.Contains(rest[:at], "/") { + return "" + } + tail = rest[at+1:] + } + if cut := strings.IndexAny(tail, "?#"); cut >= 0 { + tail = tail[:cut] + } + return remote[:i+len("://")] + tail + } + // An scp-style address is [user@]host:path; without the colon the text is a local path and keeps its @. + if colon := strings.Index(remote, ":"); colon >= 0 && !strings.Contains(remote[:colon], "/") { + if at := strings.LastIndex(remote[:colon], "@"); at >= 0 { + return remote[at+1:] + } + } + return remote +} + +// localScrubGitEnv removes the caller's git repository and configuration variables from this process, so no +// git command the run makes reads another repository, index or configuration (review finding, P2). +func localScrubGitEnv() { + for _, entry := range os.Environ() { + if name, _, _ := strings.Cut(entry, "="); strings.HasPrefix(name, "GIT_") { + os.Unsetenv(name) + } + } +} + +// localExecute runs the table in a clean worktree of the verified commit and assembles the record. +func localExecute(opts localOptions, plan []localJob, current verificationRecord, stdout io.Writer) (verificationRecord, bool, error) { + root, err := localWorkRoot(opts) + if err != nil { + return verificationRecord{}, false, err + } + if err := os.MkdirAll(root, 0o755); err != nil { + return verificationRecord{}, false, err + } + temp, err := os.MkdirTemp(root, localTempPrefix) + if err != nil { + return verificationRecord{}, false, err + } + worktree := filepath.Join(temp, "tree") + home := filepath.Join(temp, "home") + cleanup := func() { + if opts.Keep { + fmt.Fprintf(stdout, "local: kept %s\n", temp) + return + } + // Remove this run's own worktree through git, then the whole root. No repository-wide prune runs: + // it would also drop the administrative state of other tasks' worktrees. + runGit(opts.Root, "worktree", "remove", "--force", worktree) + os.RemoveAll(temp) + } + defer cleanup() + checkoutErr := "" + if out, err := runGit(opts.Root, "-c", "core.hooksPath=/dev/null", "worktree", "add", "--detach", worktree, current.HeadCommit); err != nil { + checkoutErr = fmt.Sprintf("%v%s", err, localReason(string(out))) + } else if status, err := runGit(worktree, "status", "--porcelain"); err != nil { + checkoutErr = fmt.Sprintf("%v%s", err, localReason(string(status))) + } else if dirty := strings.TrimSpace(string(status)); dirty != "" { + // Attributes or filters changed the checked-out content, so the tests would not be the commit's. + checkoutErr = "the clean worktree differs from the commit" + localReason(dirty) + } else if diff := localCheckoutDiff(worktree, current.HeadCommit); diff != "" { + checkoutErr = "the clean worktree differs from the commit: " + diff + } + if checkoutErr != "" { + // A checkout that fails is a failed record, named with its reason, not a missing one. + failed := current + failed.Sealed = true + failed.PlanDigest = planDigest(plan) + failed.IgnoredEnv = localIgnoredEnv(os.Environ()) + failed.Result = localFail + failed.Jobs = []recordJob{{Name: "clean worktree", Result: localFail, Steps: []recordStep{{Job: "clean worktree", Name: "git worktree add", Scope: "full", Result: localFailed, Reason: checkoutErr}}}} + return failed, false, nil + } + opts.output = filepath.Join(temp, "runner", "output") + env, err := localStepEnv(home, temp, opts) + if err != nil { + return verificationRecord{}, false, err + } + pins, err := localToolPinsFrom(func(path string) ([]byte, error) { return os.ReadFile(filepath.Join(worktree, path)) }) + if err != nil { + return verificationRecord{}, false, err + } + record := current + record.Sealed = true + record.PlanDigest = planDigest(plan) + record.IgnoredEnv = localIgnoredEnv(os.Environ()) + record.Pins = pins + record.Tools = localObservedVersions(current.Tools, pins) + workflowData, err := os.ReadFile(filepath.Join(worktree, ".github", "workflows", "ci.yml")) + if err != nil { + return verificationRecord{}, false, err + } + workflowJobs, err := parseWorkflow(string(workflowData)) + if err != nil { + return verificationRecord{}, false, err + } + // The record's node pin is the distinct setup-node pins of the workflow; the mismatch is per job. + var nodePins []string + for _, job := range workflowJobs { + for _, step := range job.steps { + if step.nodeVersion != "" { + nodePins = append(nodePins, step.nodeVersion) + } + } + } + if nodes := localSortedUnique(nodePins); len(nodes) > 0 { + pins["node"] = strings.Join(nodes, ",") + } + record.PinMismatch = localSortedUnique(append(localPinMismatch(pins, record.Tools), localNodeMismatch(plan, workflowJobs, record.Tools["node"])...)) + // The steps read the same base the record names (the blob and secret range) and the commit + // itself (the changed-path decisions), so the environment carries the resolved values rather + // than the caller's flags. + opts.Base = record.BaseCommit + opts.Commit = current.HeadCommit + if len(record.PinMismatch) > 0 { + fmt.Fprintf(stdout, "local: pin mismatch (%s); this record will not be reused\n", strings.Join(record.PinMismatch, ", ")) + } + + for _, job := range plan { + legs := job.legs + if legs == nil { + legs = []string{""} + } + for _, leg := range legs { + name := job.name + if leg != "" { + name = job.name + " (" + leg + ")" + } + recorded := recordJob{Name: name} + failed := false + for _, step := range job.steps { + if step.legs != nil && !localContains(step.legs, leg) { + continue + } + entry := recordStep{Job: name, Name: step.name, Command: step.command, Scope: step.scope} + if entry.Name == "" { + entry.Name = localStepLabel(step) + } + start := time.Now() + offset := localOutputSize(opts.output) + switch { + case step.kind == localNotApplicable: + entry.Result, entry.Reason = localNotApplicableResult, step.note + case failed: + // GitHub stops a job's later steps once one fails; the local run does the same + // and records the rest as skipped, which is itself a failure (answer 5). + entry.Result, entry.Reason = localSkipped, "an earlier step of this job did not pass" + default: + entry.Result, entry.Reason = localRunStep(opts, step, worktree, leg, env, record.Jobs) + if entry.Result != localPassed { + failed = true + } + } + entry.Decision = localDecisionSince(opts.output, offset) + entry.Seconds = time.Since(start).Seconds() + recorded.Steps = append(recorded.Steps, entry) + fmt.Fprintf(stdout, "local: %-26s %-44s %s\n", name, entry.Name, entry.Result) + } + recorded.Result = localPassed + for _, step := range recorded.Steps { + if step.Result != localPassed && step.Result != localNotApplicableResult { + recorded.Result = localFail + break + } + } + record.Jobs = append(record.Jobs, recorded) + } + } + record.Result = localPass + for _, job := range record.Jobs { + if job.Result != localPassed { + record.Result = localFail + break + } + } + return record, false, nil +} + +// localRunStep performs one step and returns its result and reason. done is the record's jobs so +// far, which the aggregate step judges. +func localRunStep(opts localOptions, step localStep, worktree, leg string, env []string, done []recordJob) (string, string) { + if step.tool != "" && !localToolPresent(step.tool, localPathEnv(env)) { + return localMissingTool, step.tool + " is not on PATH" + } + switch step.action { + case localCheckout: + return localPassed, "the clean worktree of " + shortHash(opts.Commit) + case localGo, localNode: + return localPassed, step.tool + " " + localObserveTool(step.tool, localPathEnv(env), opts.HeavyGate) + case localAggregate: + // dev-gate: every prerequisite job must have succeeded, judged from the jobs already + // recorded rather than from a hosted needs object. + for _, job := range done { + if job.Result != localPassed { + return localFailed, "prerequisite job " + job.Name + " did not pass" + } + } + return localPassed, "every prerequisite job succeeded" + } + command := step.command + dir := worktree + if step.workdir != "" { + dir = filepath.Join(worktree, step.workdir) + } + stepEnv := append(append([]string{}, env...), localStepEnvValues(step.env, opts, leg)...) + // The command runs as a script file through bash, so a ci.yml run text that is a block scalar + // (a loop, a heredoc) runs as the runner runs it, and a gate between this process and bash + // never sees the command's own characters. systemd-run, for one, expands the arguments it is + // handed, which would otherwise mangle a run text that uses a shell variable. + script, err := localWriteScript(opts, command) + if err != nil { + return localFailed, err.Error() + } + argv := localGateArgv(opts.HeavyGate, step.heavy, script, dir, stepEnv) + shell := exec.Command(argv[0], argv[1:]...) + shell.Dir = dir + shell.Env = stepEnv + var output bytes.Buffer + shell.Stdout, shell.Stderr = &output, &output + if err := shell.Run(); err != nil { + var exit *exec.ExitError + if errors.As(err, &exit) { + reason := fmt.Sprintf("exit status %d%s", exit.ExitCode(), localReason(output.String())) + if strings.Contains(output.String(), localTimeoutMark) { + reason = "timeout: " + reason + } + return localFailed, reason + } + return localFailed, err.Error() + } + return localPassed, "" +} + +// localIsolatedGoEnv is the record's goEnv: GOENV is unset in a step, so Go reads its default file +// under the run's own XDG_CONFIG_HOME, which is empty. +const localIsolatedGoEnv = "unset: Go reads its default file under the run's own XDG_CONFIG_HOME" + +// localGateArgv is the argv a step runs: bash