From c1af4a537b60ae059cd5e53c4ae22c10dc3fdb15 Mon Sep 17 00:00:00 2001 From: Russell Stern Date: Mon, 28 Sep 2026 18:50:28 +0000 Subject: [PATCH 1/3] Increase cre test timeouts (#23841) --- .github/workflows/cre-system-tests.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/cre-system-tests.yaml b/.github/workflows/cre-system-tests.yaml index 1a59dd0ff94..89daed46fc1 100644 --- a/.github/workflows/cre-system-tests.yaml +++ b/.github/workflows/cre-system-tests.yaml @@ -190,7 +190,7 @@ jobs: # http://docs.github.com/en/actions/how-tos/deploy/configure-and-manage-deployments/control-deployments#using-environments-without-deployments name: integration deployment: false - timeout-minutes: 10 + timeout-minutes: 13 env: ENABLE_AUTO_QUARANTINE: "true" BILLING_PLATFORM_SERVICE_IMAGE: @@ -415,7 +415,7 @@ jobs: continue-on-error: ${{ env.ENABLE_AUTO_QUARANTINE == 'true' }} env: TEST_NAME: ${{ matrix.tests.test_name }} - TEST_TIMEOUT: 7m # let's leave 3 minutes for other steps (the whole job times out after 10 minutes) + TEST_TIMEOUT: 10m # let's leave 3 minutes for other steps (the whole job times out after 13 minutes) RUN_QUARANTINED_TESTS: "true" # always run quarantined tests in CI TOPOLOGY_NAME: ${{ matrix.tests.topology }} CONFIDENTIAL_COMPUTE_ROOT: ${{ github.workspace }}/chainlink-confidential-compute From 5e84e21688d668b32e858f945013677a35d6df28 Mon Sep 17 00:00:00 2001 From: Jordan Krage Date: Mon, 28 Sep 2026 19:46:45 +0000 Subject: [PATCH 2/3] add postgres support for v18; drop v14 (#23762) --- .changeset/gold-turtles-chew.md | 5 +++++ README.md | 4 ++-- core/chainlink.Dockerfile | 2 +- core/config/docs/secrets.toml | 2 +- docs/SECRETS.md | 2 +- plugins/chainlink.Dockerfile | 6 +++--- 6 files changed, 13 insertions(+), 8 deletions(-) create mode 100644 .changeset/gold-turtles-chew.md diff --git a/.changeset/gold-turtles-chew.md b/.changeset/gold-turtles-chew.md new file mode 100644 index 00000000000..66f85c9d812 --- /dev/null +++ b/.changeset/gold-turtles-chew.md @@ -0,0 +1,5 @@ +--- +"chainlink": minor +--- + +postgres: adding support for v18, removing support for v14 #nops diff --git a/README.md b/README.md index 0b6c923e456..87cfac48bf3 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,8 @@ regarding Chainlink social accounts, news, and networking. - Example Path for macOS `export PATH=$GOPATH/bin:$PATH` & `export GOPATH=/Users/$USER/go` 2. Install [NodeJS v20](https://nodejs.org/en/download/package-manager/) & [pnpm v10 via npm](https://pnpm.io/installation#using-npm). - It might be easier long term to use [nvm](https://nodejs.org/en/download/package-manager/#nvm) to switch between node versions for different projects. For example, assuming $NODE_VERSION was set to a valid version of NodeJS, you could run: `nvm install $NODE_VERSION && nvm use $NODE_VERSION` -3. Install [Postgres (>= 12.x)](https://wiki.postgresql.org/wiki/Detailed_installation_guides). It is recommended to run the latest major version of postgres. - - Note if you are running the official Chainlink docker image, the highest supported Postgres version is 17.x due to the bundled client. +3. Install [Postgres (>= 15.x)](https://wiki.postgresql.org/wiki/Detailed_installation_guides). It is recommended to run the latest major version of postgres. + - Note if you are running the official Chainlink docker image, the highest supported Postgres version is 18.x due to the bundled client. - You should [configure Postgres](https://www.postgresql.org/docs/current/ssl-tcp.html) to use SSL connection (or for testing you can set `?sslmode=disable` in your Postgres query string). 4. Download Chainlink: `git clone https://github.com/smartcontractkit/chainlink && cd chainlink` 5. Build and install Chainlink: `make install` diff --git a/core/chainlink.Dockerfile b/core/chainlink.Dockerfile index eb7cbad5a40..04439913834 100644 --- a/core/chainlink.Dockerfile +++ b/core/chainlink.Dockerfile @@ -129,7 +129,7 @@ RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ && echo "deb [signed-by=/usr/share/keyrings/postgresql-archive-keyring.gpg] \ https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ >/etc/apt/sources.list.d/pgdg.list \ - && apt-get update && apt-get install -y --no-install-recommends postgresql-client-17=17.10-1.pgdg24.04+1 \ + && apt-get update && apt-get install -y --no-install-recommends postgresql-client-18 \ && rm -rf /var/lib/apt/lists/* # Prod images (CHAINLINK_USER=chainlink) run as UID:GID 14933:14933 for deterministic diff --git a/core/config/docs/secrets.toml b/core/config/docs/secrets.toml index be0525f0a92..1a33d993deb 100644 --- a/core/config/docs/secrets.toml +++ b/core/config/docs/secrets.toml @@ -1,5 +1,5 @@ [Database] -# URL is the PostgreSQL URI to connect to your database. Chainlink nodes require Postgres versions >= 11. See +# URL is the PostgreSQL URI to connect to your database. Chainlink nodes require Postgres versions >= 15. See # [Running a Chainlink Node](https://docs.chain.link/docs/running-a-chainlink-node/#set-the-remote-database_url-config) for an example. # # Environment variable: `CL_DATABASE_URL` diff --git a/docs/SECRETS.md b/docs/SECRETS.md index b58f66a2bb2..99e5e45287e 100644 --- a/docs/SECRETS.md +++ b/docs/SECRETS.md @@ -29,7 +29,7 @@ AllowSimplePasswords = false # Default ```toml URL = "postgresql://user:pass@localhost:5432/dbname?sslmode=disable" # Example ``` -URL is the PostgreSQL URI to connect to your database. Chainlink nodes require Postgres versions >= 11. See +URL is the PostgreSQL URI to connect to your database. Chainlink nodes require Postgres versions >= 15. See [Running a Chainlink Node](https://docs.chain.link/docs/running-a-chainlink-node/#set-the-remote-database_url-config) for an example. Environment variable: `CL_DATABASE_URL` diff --git a/plugins/chainlink.Dockerfile b/plugins/chainlink.Dockerfile index 7e27d5bd06c..2864acac29a 100644 --- a/plugins/chainlink.Dockerfile +++ b/plugins/chainlink.Dockerfile @@ -126,9 +126,9 @@ RUN curl -fsSL https://www.postgresql.org/media/keys/ACCC4CF8.asc \ --keyring /usr/share/keyrings/postgresql-archive-keyring.gpg \ --fingerprint B97B0AFCAA1A47F044F244A07FCC7D46ACCC4CF8 \ && echo "deb [signed-by=/usr/share/keyrings/postgresql-archive-keyring.gpg] \ - https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ - >/etc/apt/sources.list.d/pgdg.list \ - && apt-get update && apt-get install -y --no-install-recommends postgresql-client-17=17.10-1.pgdg24.04+1 \ + https://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" \ + >/etc/apt/sources.list.d/pgdg.list \ + && apt-get update && apt-get install -y --no-install-recommends postgresql-client-18 \ && rm -rf /var/lib/apt/lists/* # Prod images (CHAINLINK_USER=chainlink) run as UID:GID 14933:14933 for deterministic ownership. From 5f39fef7cfad7074d5b4e1ea94aa23a978861a6c Mon Sep 17 00:00:00 2001 From: Prashant Yadav <34992934+prashantkumar1982@users.noreply.github.com> Date: Mon, 28 Sep 2026 20:02:37 +0000 Subject: [PATCH 3/3] test(cre): add SkipIfMixedEnv guard for new-flag e2e tests (#23842) A PR that adds a new CRE settings flag (new cresettings key + the code that reads it) AND an e2e test that enables it via ApplyCRESettings fails mixed-env: the override is validated only in the test process against the PR schema and delivered to every node, but the baseline nodes silently ignore the unknown key and have no reader for it. The 2 PR nodes act on the flag, the 2 baseline nodes cannot, so the DON diverges 2-vs-2 and trips the non-determinism gate (consensus stall + log-marker scan) even though the PR is correct. Add t_helpers.IsMixedEnvTopology() and t_helpers.SkipIfMixedEnv(t, reason), keyed strictly on TOPOLOGY_NAME, so such a test can opt out of mixed-env while still running in full under the normal single-image suite. The helper carries a strong warning to use it only when a 2-vs-2 split is unavoidable and expected. Convert the existing DON Time skip to the new helper and route nonDeterminismCheckEnabled() through IsMixedEnvTopology() for one source of truth. Document the caveat and the warning in cresettings-override-README.md and mixed-env.md. Co-authored-by: Claude Opus 4.8 --- .../docs/cresettings-override-README.md | 43 +++++++++++++++++++ .../scripts/cre/environment/docs/mixed-env.md | 34 +++++++++++++++ .../tests/smoke/cre/cre_suite_test.go | 4 +- .../smoke/cre/nondeterminism_check_test.go | 2 +- .../tests/test-helpers/mixed_env_test.go | 25 +++++++++++ system-tests/tests/test-helpers/t_helpers.go | 33 ++++++++++++++ 6 files changed, 137 insertions(+), 4 deletions(-) create mode 100644 system-tests/tests/test-helpers/mixed_env_test.go diff --git a/core/scripts/cre/environment/docs/cresettings-override-README.md b/core/scripts/cre/environment/docs/cresettings-override-README.md index baae3bacfde..2303c719b93 100644 --- a/core/scripts/cre/environment/docs/cresettings-override-README.md +++ b/core/scripts/cre/environment/docs/cresettings-override-README.md @@ -189,6 +189,49 @@ say *what* to change; the helper makes sure it lands everywhere it could be read (Internally it still merges each DON's own boot baseline, since DONs can boot with different `CL_CRE_SETTINGS` — but that's handled for you.) +## Mixed-env caveat: a NEW flag will diverge (guard the test) + +[Mixed-env](./mixed-env.md) runs each multi-node DON as **2 nodes on the PR image + 2 on +the develop/baseline image** and fails the run if the two halves disagree. That interacts +with this helper in one specific case: + +- **Overriding a flag that already exists in the baseline image** → fine. Both halves know + the key and behave the same; no divergence. +- **Overriding a flag that is NEW in your PR** (you added the `cresettings` key *and* the + code that reads it in the same PR) → **the 2 PR nodes act on it, the 2 baseline nodes + cannot** (their binary has neither the key nor a reader — the override is validated only + in the test process against the PR schema, delivered to every node, then silently ignored + on the baseline nodes). The DON diverges 2-vs-2, which trips the mixed-env non-determinism + gate (a consensus stall and/or the post-run log-marker scan) — a **false failure**, since + your PR is correct. + +For that second case, guard the test so it does not run under mixed-env: + +```go +func Test_CRE_MyNewFlag(t *testing.T) { + // Skips ONLY under mixed-env; the test still runs in full under the normal + // single-image suite. Call it before deploy/register/ApplyCRESettings. + t_helpers.SkipIfMixedEnv(t, "enables PerWorkflow., new in this PR; baseline "+ + "nodes lack it and would diverge. Remove once is in the release baseline.") + + testEnv := t_helpers.SetupTestEnvironmentWithPerTestKeys(t, t_helpers.GetDefaultTestConfig(t)) + t_helpers.ApplyCRESettings(t, testEnv, t_helpers.Workflow(myWorkflowID, "...")) + // ... +} +``` + +> **⚠️ Do not reach for `SkipIfMixedEnv` unless you know precisely why *this* test must skip +> mixed-env.** Mixed-env is a required merge gate for cross-version compatibility; every skip +> is a hole in it. It is justified **only** when the test deliberately exercises behavior +> that exists in the PR image but not the baseline (a brand-new flag or capability), so a +> 2-vs-2 split is unavoidable and expected — **never** to silence a real divergence a +> reviewer should see. The `reason` is mandatory; note removing the guard once the flag +> reaches the baseline image. See [mixed-env.md](./mixed-env.md#skipping-a-single-test). + +(Enabling a new flag via a topology's boot `CL_CRE_SETTINGS` rather than this helper diverges +from boot regardless of any test — this guard won't help there; keep the new flag out of the +mixed-env topology instead.) + ## Run these serially — the override guard Because overrides mutate settings on the shared environment, **only one override may be diff --git a/core/scripts/cre/environment/docs/mixed-env.md b/core/scripts/cre/environment/docs/mixed-env.md index 47f8d223dfd..eb28c93fa9d 100644 --- a/core/scripts/cre/environment/docs/mixed-env.md +++ b/core/scripts/cre/environment/docs/mixed-env.md @@ -69,6 +69,40 @@ These lines are **silent** when every node runs the same code, so a single occur - It flags **any** divergence from `develop` — including **intentional** report/payload changes. Those are genuine incompatibilities: if the change is deliberate and will be rolled out safely, bypass the check with the [`skip-mixed-env` label](#required-check--emergency-bypass) rather than "fixing" it. - It compares against the `develop` commit your PR is based on — not the very latest `develop`. So a change that only conflicts with `develop` commits landed **after** you branched is caught by the nightly full-matrix sweep (which pins the latest `develop`), not by the per-PR run. Keep your branch reasonably current for the tightest signal. +## Skipping a single test + +Sometimes a test *legitimately* cannot pass under mixed-env because it deliberately +exercises behavior that exists in the PR image but **not** the baseline image — most commonly +a **brand-new `cresettings` flag** (added, and enabled via `ApplyCRESettings`, in the same PR) +or a **newly added capability**. The 2 PR nodes act on it while the 2 baseline nodes cannot, +so the DON diverges 2-vs-2 and the check fires — even though the PR is correct. (See the +[CRE settings override caveat](./cresettings-override-README.md#mixed-env-caveat-a-new-flag-will-diverge-guard-the-test).) + +For that case, guard the individual test with the helper in the test-helpers package: + +```go +func Test_CRE_MyNewThing(t *testing.T) { + t_helpers.SkipIfMixedEnv(t, "enables , new in this PR; baseline nodes lack it "+ + "and would diverge. Remove once is in the release baseline.") + // ... rest of the test ... +} +``` + +It skips **only** under mixed-env (it keys on `TOPOLOGY_NAME`); the test still runs in full +under the normal single-image suite, and mixed-env covers it again automatically once the +flag/behavior lands in the baseline image. Place the call at the top of the test/scenario, +before any deploy/register/`ApplyCRESettings`, so the skipped test emits no divergent traffic. +For one scenario inside a bucket suite, put it inside that scenario's `t.Run(...)` closure. + +> **⚠️ Do not reach for `SkipIfMixedEnv` unless you know precisely why *this* test must skip +> mixed-env.** Every skip is a hole in a required cross-version gate. Use it **only** when a +> 2-vs-2 split is unavoidable and expected (new-in-PR flag or capability) — **never** to +> silence a real divergence a reviewer should see. If the divergence is an *intentional but +> already-in-baseline* report/payload change affecting a whole run, prefer the +> [`skip-mixed-env` label](#required-check--emergency-bypass) instead. To drop a test from +> mixed-env entirely (not just skip at runtime), remove its name from `defaultCREMixedEnvTests` +> in `tools/ci/internal/matrix/mixedenv.go`. + ## When it runs - **CI:** automatically on CRE-affecting PRs, in its **own** workflow `.github/workflows/cre-mixed-env-tests.yaml` (kept separate from `cre-system-tests.yaml` so that file stays simple). It runs the OCR3/DON2DON-heavy tests — `Test_CRE_V2_Suite_Bucket_A`, `Test_CRE_V2_Suite_Bucket_B`, and the `Test_CRE_V2_EVM_Read_*` suite — under mixed-env. The PR image and the develop image are both already built (per-PR and nightly), so no extra image builds are added. A dedicated **Check for non-determinism** step scans the live node containers after the suite and fails the job on any marker — kept separate from the auto-quarantined test step so the failure can't be swallowed. diff --git a/system-tests/tests/smoke/cre/cre_suite_test.go b/system-tests/tests/smoke/cre/cre_suite_test.go index 95286d2f1ec..dfe44341453 100644 --- a/system-tests/tests/smoke/cre/cre_suite_test.go +++ b/system-tests/tests/smoke/cre/cre_suite_test.go @@ -164,9 +164,7 @@ func runSuiteScenario(t *testing.T, topology string, scenario suite_config.Suite }) case suite_config.SuiteScenarioDONTime: t.Run("DON Time - "+topology, func(t *testing.T) { - if nonDeterminismCheckEnabled() { - t.Skip("DONTime registry launch requires all nodes to support jobless capability startup") - } + t_helpers.SkipIfMixedEnv(t, "DONTime registry launch requires all nodes to support jobless capability startup") if parallelEnabled { t.Parallel() } diff --git a/system-tests/tests/smoke/cre/nondeterminism_check_test.go b/system-tests/tests/smoke/cre/nondeterminism_check_test.go index 089429cf691..8e50c5373cf 100644 --- a/system-tests/tests/smoke/cre/nondeterminism_check_test.go +++ b/system-tests/tests/smoke/cre/nondeterminism_check_test.go @@ -36,7 +36,7 @@ func nonDeterminismCheckEnabled() bool { if strings.EqualFold(os.Getenv("CRE_NONDETERMINISM_CHECK"), "true") { return true } - return strings.Contains(strings.ToLower(os.Getenv("TOPOLOGY_NAME")), "mixed-env") + return t_helpers.IsMixedEnvTopology() } // reportNonDeterminism scans all container logs for the markers and prints any diff --git a/system-tests/tests/test-helpers/mixed_env_test.go b/system-tests/tests/test-helpers/mixed_env_test.go new file mode 100644 index 00000000000..36403c88d58 --- /dev/null +++ b/system-tests/tests/test-helpers/mixed_env_test.go @@ -0,0 +1,25 @@ +package helpers + +import "testing" + +func TestIsMixedEnvTopology(t *testing.T) { + cases := []struct { + name string + topologyName string + want bool + }{ + {name: "empty", topologyName: "", want: false}, + {name: "single-image topology", topologyName: "workflow-gateway-capabilities-don", want: false}, + {name: "exact mixed-env", topologyName: "mixed-env", want: true}, + {name: "mixed-env confidential variant", topologyName: "mixed-env-confidential-workflows", want: true}, + {name: "case-insensitive", topologyName: "MIXED-ENV", want: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv("TOPOLOGY_NAME", tc.topologyName) + if got := IsMixedEnvTopology(); got != tc.want { + t.Fatalf("IsMixedEnvTopology() with TOPOLOGY_NAME=%q = %v, want %v", tc.topologyName, got, tc.want) + } + }) + } +} diff --git a/system-tests/tests/test-helpers/t_helpers.go b/system-tests/tests/test-helpers/t_helpers.go index c593be0a3e9..8d5641e531d 100644 --- a/system-tests/tests/test-helpers/t_helpers.go +++ b/system-tests/tests/test-helpers/t_helpers.go @@ -967,6 +967,39 @@ func ParallelEnabled() bool { return v == "1" || v == "true" || v == "yes" } +// IsMixedEnvTopology reports whether the suite is running under a mixed-env topology, +// i.e. each multi-node DON runs 2 nodes on the PR image and 2 on the develop/baseline +// image. CI sets TOPOLOGY_NAME=mixed-env for these runs. This is the signal that a DON +// can legitimately behave two ways at once, so it is the correct gate for +// SkipIfMixedEnv. See core/scripts/cre/environment/docs/mixed-env.md. +func IsMixedEnvTopology() bool { + return strings.Contains(strings.ToLower(os.Getenv("TOPOLOGY_NAME")), "mixed-env") +} + +// SkipIfMixedEnv skips the current test when running under a mixed-env topology (see +// IsMixedEnvTopology). Call it at the very top of the test/scenario body, BEFORE any +// deploy/register/ApplyCRESettings, so the skipped test emits no divergent traffic. +// +// WARNING: do NOT use this unless you know precisely why THIS test must skip mixed-env. +// Mixed-env is a required merge gate that catches cross-version consensus/DON2DON +// divergence between PR and develop nodes; every skip is a hole in that coverage. It is +// justified ONLY when the test deliberately exercises behavior that exists in the PR +// image but not the baseline image — e.g. a brand-new CRE settings flag enabled via +// ApplyCRESettings, or a newly added capability — so a 2-vs-2 split is unavoidable and +// expected. It is NOT a way to silence a real divergence a reviewer should see. The test +// still runs in full under the normal single-image suite; only the (impossible) +// cross-version check is skipped, and mixed-env covers it again automatically once the +// flag/behavior lands in the baseline image. +// +// reason is mandatory: state why the divergence is expected, and note removing this guard +// once the flag/behavior reaches the baseline image. +func SkipIfMixedEnv(t *testing.T, reason string) { + t.Helper() + if IsMixedEnvTopology() { + t.Skipf("skipping in mixed-env: %s", reason) + } +} + // ─── Stellar ReadContract test helpers (shared by smoke + regression) ─── // // These are the generic, chain-family-agnostic bits reused by both the smoke