diff --git a/apps/dokploy/__test__/server/host-stack-lock.test.ts b/apps/dokploy/__test__/server/host-stack-lock.test.ts new file mode 100644 index 00000000000..e8fca1a8136 --- /dev/null +++ b/apps/dokploy/__test__/server/host-stack-lock.test.ts @@ -0,0 +1,31 @@ +import { + acquireHostStackUpdateLock, + isHostStackUpdateRunning, +} from "@dokploy/server/setup/host-stack-lock"; +import { describe, expect, it } from "vitest"; + +describe("acquireHostStackUpdateLock", () => { + it("admits one holder at a time and frees the lock on release", () => { + const release = acquireHostStackUpdateLock(); + + expect(release).not.toBeNull(); + expect(isHostStackUpdateRunning()).toBe(true); + expect(acquireHostStackUpdateLock()).toBeNull(); + + release?.(); + expect(isHostStackUpdateRunning()).toBe(false); + }); + + it("ignores a repeated release, so it cannot free a lock taken after it", () => { + const first = acquireHostStackUpdateLock(); + first?.(); + const second = acquireHostStackUpdateLock(); + first?.(); + + expect(isHostStackUpdateRunning()).toBe(true); + expect(acquireHostStackUpdateLock()).toBeNull(); + + second?.(); + expect(isHostStackUpdateRunning()).toBe(false); + }); +}); diff --git a/apps/dokploy/__test__/server/host-stack-rows.test.ts b/apps/dokploy/__test__/server/host-stack-rows.test.ts new file mode 100644 index 00000000000..a3c303e44c3 --- /dev/null +++ b/apps/dokploy/__test__/server/host-stack-rows.test.ts @@ -0,0 +1,305 @@ +import { PINNED_VERSIONS } from "@dokploy/server/setup/component-versions"; +import { + buildHostStackRows, + HOST_STACK_COMPONENTS, + type HostStackReadings, + type HostStackRow, + hostStackRefusals, + isHostStackUiComponent, +} from "@dokploy/server/setup/host-stack-rows"; +import { TRAEFIK_VERSION } from "@dokploy/server/setup/traefik-setup"; +import { describe, expect, it } from "vitest"; + +const pinnedPostgresMajor = PINNED_VERSIONS.postgres.split(".")[0]; + +const upToDate: HostStackReadings = { + docker: PINNED_VERSIONS.docker, + traefikImage: `traefik:v${TRAEFIK_VERSION}`, + postgresImage: `postgres:${PINNED_VERSIONS.postgres}`, + redisImage: `redis:${PINNED_VERSIONS.redis}`, + panel: { + installed: "agenthits-dev", + latest: "agenthits-dev", + updateAvailable: false, + }, +}; + +const rowsFor = (overrides: Partial) => + buildHostStackRows({ ...upToDate, ...overrides }); + +const rowNamed = (rows: HostStackRow[], name: HostStackRow["name"]) => { + const row = rows.find((candidate) => candidate.name === name); + if (!row) { + throw new Error(`no row named ${name}`); + } + return row; +}; + +describe("buildHostStackRows", () => { + it("lists the panel host components in stack order", () => { + expect(rowsFor({}).map((row) => row.name)).toEqual([ + "panel", + "docker", + "traefik", + "postgres", + "redis", + ]); + }); + + it("reports every component as current when the installed versions match the pins", () => { + for (const row of rowsFor({})) { + expect(row.status, row.name).toBe("current"); + expect(row.outdated, row.name).toBe(false); + } + }); + + it("keeps the reason of the maintenance-window steps even when they are current", () => { + const rows = rowsFor({}); + + expect(rowNamed(rows, "docker")).toMatchObject({ + action: "manual", + command: "DOCKER_ENGINE_UPGRADE=1 bash install-agenthits.sh update", + }); + expect(rowNamed(rows, "docker").reason).toContain("maintenance window"); + expect(rowNamed(rows, "postgres")).toMatchObject({ + action: "manual", + command: null, + }); + expect(rowNamed(rows, "postgres").reason).toContain("maintenance window"); + }); + + it("gives Traefik and Redis no reason or command, since the panel runs them", () => { + const rows = rowsFor({ + traefikImage: "traefik:v3.6.1", + redisImage: "redis:7", + }); + + for (const name of ["traefik", "redis"] as const) { + expect(rowNamed(rows, name)).toMatchObject({ + action: "ui", + reason: null, + command: null, + }); + } + }); + + it("marks Docker Engine behind its pin as a maintenance-window step with the engine upgrade command", () => { + const docker = rowNamed(rowsFor({ docker: "29.4.1" }), "docker"); + + expect(docker).toMatchObject({ + installed: "29.4.1", + target: PINNED_VERSIONS.docker, + outdated: true, + action: "manual", + command: "DOCKER_ENGINE_UPGRADE=1 bash install-agenthits.sh update", + }); + expect(docker.reason).toContain("maintenance window"); + }); + + it("compares Docker Engine versions numerically", () => { + expect(rowNamed(rowsFor({ docker: "29.10.0" }), "docker").outdated).toBe( + false, + ); + expect(rowNamed(rowsFor({ docker: "29.8.1" }), "docker").outdated).toBe( + true, + ); + }); + + it("reads the Traefik version from its image tag and marks it updatable from the panel", () => { + const traefik = rowNamed( + rowsFor({ traefikImage: "traefik:v3.6.1" }), + "traefik", + ); + + expect(traefik).toMatchObject({ + installed: "3.6.1", + target: TRAEFIK_VERSION, + outdated: true, + action: "ui", + reason: null, + command: null, + }); + }); + + it("treats Traefik at the pinned version as current", () => { + expect( + rowNamed( + rowsFor({ traefikImage: `traefik:v${TRAEFIK_VERSION}` }), + "traefik", + ).outdated, + ).toBe(false); + }); + + it("compares Postgres by major version and reports a major change as manual", () => { + const postgres = rowNamed( + rowsFor({ postgresImage: "postgres:16" }), + "postgres", + ); + + expect(postgres).toMatchObject({ + installed: "16", + target: PINNED_VERSIONS.postgres, + outdated: true, + action: "manual", + command: "POSTGRES_IMAGE=postgres:16 bash install-agenthits.sh update", + }); + expect(postgres.reason).toBe( + `Major version change (16 to ${pinnedPostgresMajor}) needs a migration. The panel never changes the Postgres major version.`, + ); + }); + + it("compares the full Postgres version within the pinned major", () => { + const current = rowNamed( + rowsFor({ postgresImage: `postgres:${PINNED_VERSIONS.postgres}-alpine` }), + "postgres", + ); + + expect(current).toMatchObject({ + installed: PINNED_VERSIONS.postgres, + status: "current", + outdated: false, + command: null, + }); + + const minorBehind = rowNamed( + rowsFor({ postgresImage: `postgres:${pinnedPostgresMajor}` }), + "postgres", + ); + + expect(minorBehind).toMatchObject({ + installed: pinnedPostgresMajor, + target: PINNED_VERSIONS.postgres, + status: "outdated", + outdated: true, + action: "manual", + command: null, + }); + expect(minorBehind.reason).toContain("maintenance window"); + }); + + it("compares Postgres minor versions numerically", () => { + expect( + rowNamed( + rowsFor({ postgresImage: `postgres:${pinnedPostgresMajor}.10` }), + "postgres", + ).outdated, + ).toBe(false); + }); + + it("keeps the image digest out of the Postgres command", () => { + const postgres = rowNamed( + rowsFor({ postgresImage: "postgres:16@sha256:abcdef0123" }), + "postgres", + ); + + expect(postgres.command).toBe( + "POSTGRES_IMAGE=postgres:16 bash install-agenthits.sh update", + ); + }); + + it("marks Redis behind its pin as updatable from the panel, comparing versions numerically", () => { + expect(rowNamed(rowsFor({ redisImage: "redis:7" }), "redis")).toMatchObject( + { + installed: "7", + target: PINNED_VERSIONS.redis, + outdated: true, + action: "ui", + }, + ); + expect( + rowNamed(rowsFor({ redisImage: "redis:8.9.9" }), "redis").outdated, + ).toBe(true); + expect( + rowNamed(rowsFor({ redisImage: "redis:8.10.2" }), "redis").outdated, + ).toBe(false); + }); + + it("reports components whose version cannot be read as unknown rather than outdated", () => { + const rows = rowsFor({ + docker: null, + traefikImage: null, + postgresImage: "postgres:latest", + redisImage: "redis:latest", + }); + + for (const name of ["docker", "traefik", "postgres", "redis"] as const) { + const row = rowNamed(rows, name); + expect(row.status, name).toBe("unknown"); + expect(row.outdated, name).toBe(false); + expect(row.installed, name).toBeNull(); + expect(row.command, name).toBe( + name === "docker" + ? "DOCKER_ENGINE_UPGRADE=1 bash install-agenthits.sh update" + : null, + ); + expect(row.reason, name).not.toBeNull(); + } + }); + + it("marks the panel outdated when its update check reports a newer build", () => { + expect( + rowNamed( + rowsFor({ + panel: { + installed: "agenthits-dev", + latest: "agenthits-dev-2", + updateAvailable: true, + }, + }), + "panel", + ), + ).toMatchObject({ + installed: "agenthits-dev", + target: "agenthits-dev-2", + status: "outdated", + outdated: true, + action: "panel", + }); + }); + + it("reports the panel as unknown when the update check did not return a version", () => { + expect( + rowNamed( + rowsFor({ + panel: { + installed: "agenthits-dev", + latest: null, + updateAvailable: null, + }, + }), + "panel", + ), + ).toMatchObject({ + status: "unknown", + outdated: false, + action: "panel", + target: null, + }); + }); +}); + +describe("host stack update rules", () => { + it("allows the panel to update only Traefik and Redis", () => { + expect(HOST_STACK_COMPONENTS.filter(isHostStackUiComponent)).toEqual([ + "traefik", + "redis", + ]); + }); + + it("refuses docker, postgres and panel with a message that names the next step", () => { + const refusals = hostStackRefusals([ + "traefik", + "docker", + "postgres", + "panel", + ]); + + expect(refusals).toHaveLength(3); + expect(refusals[0]).toContain( + "DOCKER_ENGINE_UPGRADE=1 bash install-agenthits.sh update", + ); + expect(refusals[1]).toContain("POSTGRES_IMAGE"); + expect(refusals[2]).toContain("Web Server Update"); + expect(hostStackRefusals(["traefik", "redis"])).toEqual([]); + }); +}); diff --git a/apps/dokploy/__test__/server/host-stack-script.test.ts b/apps/dokploy/__test__/server/host-stack-script.test.ts new file mode 100644 index 00000000000..ebfc91447f4 --- /dev/null +++ b/apps/dokploy/__test__/server/host-stack-script.test.ts @@ -0,0 +1,349 @@ +import { spawnSync } from "node:child_process"; +import { + chmodSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { PINNED_VERSIONS } from "@dokploy/server/setup/component-versions"; +import { buildHostStackUpdateScript } from "@dokploy/server/setup/host-stack"; +import type { HostStackUiComponent } from "@dokploy/server/setup/host-stack-rows"; +import { + buildComponentUpdateSteps, + COMPONENTS_UPDATE_DONE, +} from "@dokploy/server/setup/server-components"; +import { TRAEFIK_VERSION } from "@dokploy/server/setup/traefik-setup"; +import { describe, expect, it } from "vitest"; + +const SPAWN_TEST_TIMEOUT_MS = 30_000; + +const FAKE_DOCKER = `#!/bin/sh +printf 'docker %s\\n' "$*" >> "$CALL_LOG" +case "$1" in + inspect) + case "$*" in + *Config.Image*) cat "$STATE_DIR/traefik_image" ;; + *RestartCount*) echo "true 0" ;; + esac + ;; + pull) [ "$FAKE_PULL_FAILS" = 1 ] && exit 1 ;; + ps) echo cid-redis ;; + exec) [ "$FAKE_PING_FAILS" = 1 ] || echo PONG ;; + service) + case "$2" in + inspect) + case "$*" in + *ContainerSpec.Image*) cat "$STATE_DIR/image" ;; + *Version.Index*) cat "$STATE_DIR/index" ;; + *UpdateStatus.State*) cat "$STATE_DIR/state" ;; + esac + ;; + update) + [ "$FAKE_UPDATE_REFUSED" = 1 ] && exit 1 + previous="" + for argument in "$@"; do + if [ "$previous" = "--image" ]; then + printf '%s\\n' "$argument" > "$STATE_DIR/image" + fi + previous="$argument" + done + if [ "$FAKE_NEVER_MOVES" != 1 ]; then + index=$(cat "$STATE_DIR/index") + echo $((index + 1)) > "$STATE_DIR/index" + fi + printf '%s\\n' "$FAKE_UPDATE_STATE" > "$STATE_DIR/state" + ;; + esac + ;; +esac +exit 0 +`; + +const FAKE_SUDO = `#!/bin/sh +exec "$@" +`; + +const FAKE_SLEEP = `#!/bin/sh +exit 0 +`; + +type HostFixture = { + image?: string; + traefikImage?: string; + env?: Record; +}; + +const runHostScript = ( + components: HostStackUiComponent[], + fixture: HostFixture = {}, +) => { + const dir = mkdtempSync(path.join(tmpdir(), "host-stack-")); + try { + const callLog = path.join(dir, "calls.log"); + writeFileSync(callLog, ""); + writeFileSync(path.join(dir, "image"), `${fixture.image ?? "redis:7"}\n`); + writeFileSync( + path.join(dir, "traefik_image"), + `${fixture.traefikImage ?? "traefik:v1.0.0"}\n`, + ); + writeFileSync(path.join(dir, "index"), "1\n"); + writeFileSync(path.join(dir, "state"), "\n"); + for (const [name, body] of Object.entries({ + docker: FAKE_DOCKER, + sudo: FAKE_SUDO, + sleep: FAKE_SLEEP, + })) { + writeFileSync(path.join(dir, name), body); + chmodSync(path.join(dir, name), 0o755); + } + const result = spawnSync("bash", [], { + input: buildHostStackUpdateScript(components), + encoding: "utf8", + env: { + ...process.env, + PATH: `${dir}:${process.env.PATH}`, + CALL_LOG: callLog, + STATE_DIR: dir, + FAKE_UPDATE_STATE: "completed", + DOKPLOY_REDIS_TIMEOUT: "0", + ...fixture.env, + }, + }); + return { + status: result.status, + stdout: result.stdout, + stderr: result.stderr, + calls: readFileSync(callLog, "utf8").split("\n").filter(Boolean), + }; + } finally { + rmSync(dir, { force: true, recursive: true }); + } +}; + +const REDIS_UPDATE_CALL = `docker service update --detach --update-order stop-first --update-failure-action rollback --image redis:${PINNED_VERSIONS.redis} dokploy-redis`; + +describe("buildHostStackUpdateScript", () => { + it("reuses the component update steps for Traefik unchanged", () => { + expect(buildHostStackUpdateScript(["traefik"])).toContain( + buildComponentUpdateSteps(["traefik"]), + ); + }); + + it("pulls Redis before any change and updates it after Traefik is replaced", () => { + const script = buildHostStackUpdateScript(["redis", "traefik"]); + const redisPull = script.indexOf("redis_pull() {"); + const traefikDownload = script.indexOf("abort_before_change() {"); + const traefikReplace = script.indexOf('if ! traefik_create "traefik:v'); + const redisUpdate = script.indexOf("redis_settled() {"); + + expect(redisPull).toBeGreaterThanOrEqual(0); + expect(redisPull).toBeLessThan(traefikDownload); + expect(traefikDownload).toBeLessThan(traefikReplace); + expect(traefikReplace).toBeLessThan(redisUpdate); + }); + + it("builds the same script whatever order the components are requested in", () => { + expect(buildHostStackUpdateScript(["redis", "traefik"])).toBe( + buildHostStackUpdateScript(["traefik", "redis"]), + ); + }); + + it("prints the done marker once, as the last line", () => { + const script = buildHostStackUpdateScript(["traefik", "redis"]); + + expect(script.split(COMPONENTS_UPDATE_DONE)).toHaveLength(2); + expect(script.trimEnd().endsWith(`echo "${COMPONENTS_UPDATE_DONE}"`)).toBe( + true, + ); + }); + + it("pins Redis to the pinned image and waits for PONG within a timeout", () => { + const script = buildHostStackUpdateScript(["redis"]); + + expect(script).toContain(`redis_image="redis:${PINNED_VERSIONS.redis}"`); + expect(script).toContain( + 'docker service update --detach --update-order stop-first --update-failure-action rollback --image "$redis_image" dokploy-redis', + ); + expect(script).toContain("redis-cli ping"); + expect(script).toContain('= "PONG"'); + expect(script).toContain("DOKPLOY_REDIS_TIMEOUT:-240"); + }); + + it("does not touch the Postgres or panel services", () => { + const script = buildHostStackUpdateScript(["traefik", "redis"]); + + expect(script).not.toContain("dokploy-postgres"); + expect(script).not.toMatch(/service update [^\n]* dokploy\n/); + }); +}); + +describe("Traefik update run", () => { + it( + "leaves Traefik alone when its container already runs the target image, digest aside", + () => { + const run = runHostScript(["traefik"], { + traefikImage: `traefik:v${TRAEFIK_VERSION}@sha256:0123abcd`, + }); + + expect(run.status).toBe(0); + expect( + run.calls.filter((call) => + /^docker (stop|create|start) |^docker rm dokploy-traefik$/.test(call), + ), + ).toEqual([]); + expect(run.stdout).toContain( + `Traefik already runs traefik:v${TRAEFIK_VERSION}`, + ); + expect(run.stdout).toContain(COMPONENTS_UPDATE_DONE); + }, + SPAWN_TEST_TIMEOUT_MS, + ); + + it( + "replaces the Traefik container when it runs another image", + () => { + const run = runHostScript(["traefik"], { + traefikImage: "traefik:v1.0.0", + }); + + expect(run.status).toBe(0); + expect(run.calls).toContain("docker stop dokploy-traefik"); + expect( + run.calls.some( + (call) => + call.startsWith("docker create ") && + call.includes(`traefik:v${TRAEFIK_VERSION}`), + ), + ).toBe(true); + expect(run.stdout).toContain( + `Traefik version ${TRAEFIK_VERSION} installed`, + ); + }, + SPAWN_TEST_TIMEOUT_MS, + ); +}); + +describe("Redis update run", () => { + it( + "updates Redis to the pinned image, waits for PONG, and reports done", + () => { + const run = runHostScript(["redis"]); + + expect(run.status).toBe(0); + expect(run.calls).toContain(`docker pull redis:${PINNED_VERSIONS.redis}`); + expect(run.calls).toContain(REDIS_UPDATE_CALL); + expect(run.stdout).toContain( + `Redis version ${PINNED_VERSIONS.redis} installed`, + ); + expect(run.stdout).toContain(COMPONENTS_UPDATE_DONE); + }, + SPAWN_TEST_TIMEOUT_MS, + ); + + it( + "leaves Redis alone when it already runs the pinned image", + () => { + const run = runHostScript(["redis"], { + image: `redis:${PINNED_VERSIONS.redis}`, + }); + + expect(run.status).toBe(0); + expect(run.calls.some((call) => call.includes("service update"))).toBe( + false, + ); + expect(run.stdout).toContain("already runs"); + expect(run.stdout).toContain(COMPONENTS_UPDATE_DONE); + }, + SPAWN_TEST_TIMEOUT_MS, + ); + + it( + "fails loudly when the swarm rolls the Redis update back", + () => { + const run = runHostScript(["redis"], { + env: { FAKE_UPDATE_STATE: "rollback_completed" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain( + `dokploy-redis did not start on redis:${PINNED_VERSIONS.redis}`, + ); + expect(run.stdout).not.toContain(COMPONENTS_UPDATE_DONE); + }, + SPAWN_TEST_TIMEOUT_MS, + ); + + it( + "fails loudly when the Redis update pauses", + () => { + const run = runHostScript(["redis"], { + env: { FAKE_UPDATE_STATE: "paused" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain("is paused after its update"); + }, + SPAWN_TEST_TIMEOUT_MS, + ); + + it( + "fails loudly when Redis never answers PONG", + () => { + const run = runHostScript(["redis"], { + env: { FAKE_PING_FAILS: "1" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain("was not ready"); + expect(run.stdout).not.toContain(COMPONENTS_UPDATE_DONE); + }, + SPAWN_TEST_TIMEOUT_MS, + ); + + it( + "fails loudly when the swarm never registers the update", + () => { + const run = runHostScript(["redis"], { + env: { FAKE_NEVER_MOVES: "1" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain("was not ready"); + }, + SPAWN_TEST_TIMEOUT_MS, + ); + + it( + "changes nothing when the pinned Redis image cannot be pulled", + () => { + const run = runHostScript(["redis"], { + env: { FAKE_PULL_FAILS: "1" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain( + `Pre-download failed: redis:${PINNED_VERSIONS.redis}. Nothing was changed.`, + ); + expect(run.calls.some((call) => call.includes("service update"))).toBe( + false, + ); + }, + SPAWN_TEST_TIMEOUT_MS, + ); + + it( + "fails loudly when the swarm refuses the update", + () => { + const run = runHostScript(["redis"], { + env: { FAKE_UPDATE_REFUSED: "1" }, + }); + + expect(run.status).toBe(1); + expect(run.stderr).toContain("docker service update was refused"); + }, + SPAWN_TEST_TIMEOUT_MS, + ); +}); diff --git a/apps/dokploy/__test__/server/server-components.test.ts b/apps/dokploy/__test__/server/server-components.test.ts index 2f209f67b3f..83dd9a6315b 100644 --- a/apps/dokploy/__test__/server/server-components.test.ts +++ b/apps/dokploy/__test__/server/server-components.test.ts @@ -125,7 +125,13 @@ describe("buildComponentUpdateScript", () => { expect(script).toContain(`docker pull traefik:v${TRAEFIK_VERSION}`); expect(script).toContain(`traefik_create "traefik:v${TRAEFIK_VERSION}"`); - expect(script).toContain(buildTraefikCreateWithImage('"$1"').trim()); + expect(script).toContain( + buildTraefikCreateWithImage('"$1"', "$traefik_http_publish").trim(), + ); + expect(script).toContain('traefik_http_publish="-p 80:80"'); + expect(script).toContain( + "docker network disconnect bridge dokploy-traefik", + ); expect(script).not.toContain("docker rename"); }); it("stops and removes the old Traefik, creates the new one, attaches its networks, and starts it last", () => { diff --git a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts index 32ae0e340ef..897017d1a81 100644 --- a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts +++ b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts @@ -79,6 +79,13 @@ vi.mock("@dokploy/server", async () => ({ getDokployImageKeepCount: vi.fn(), getDokployImages: vi.fn(), IS_CLOUD: false, + HOST_STACK_COMPONENTS: ["panel", "docker", "traefik", "postgres", "redis"], + ...(await import("@dokploy/server/setup/host-stack-lock")), + hostStackRefusals: () => [], + isHostStackUiComponent: (component: string) => + component === "traefik" || component === "redis", + readHostStackRows: vi.fn(), + updateHostStackComponents: vi.fn(() => Promise.resolve()), checkGPUStatus: mocks.checkGPUStatus, checkPortInUse: mocks.checkPortInUse, checkPostgresHealth: mocks.checkPostgresHealth, @@ -800,3 +807,77 @@ describe("settings Traefik file access", () => { }); }); }); + +describe("panel host update lock", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.isServerUpdateRunning.mockReturnValue(false); + mocks.getUpdateData.mockResolvedValue({ + updateAvailable: true, + updateSource: "official", + latestVersion: "v2.0.0", + latestOfficialVersion: "v2.0.0", + }); + }); + + const startHostUpdate = () => + createCaller().updateHostStack({ components: ["redis"] }); + + const runToCompletion = ( + stream: Awaited>, + ) => + new Promise((resolve, reject) => { + stream.subscribe({ + next: () => {}, + error: reject, + complete: () => resolve(), + }); + }); + + const holdAudit = () => { + let recorded = () => {}; + mocks.audit.mockReturnValueOnce( + new Promise((resolve) => { + recorded = resolve; + }), + ); + return () => recorded(); + }; + + it("admits one host-stack update while another is still being recorded", async () => { + const recorded = holdAudit(); + const first = startHostUpdate(); + const second = startHostUpdate(); + + await expect(second).rejects.toMatchObject({ code: "CONFLICT" }); + recorded(); + await runToCompletion(await first); + await runToCompletion(await startHostUpdate()); + }); + + it("frees the lock when recording the update fails", async () => { + mocks.audit.mockRejectedValueOnce(new Error("audit store is down")); + + await expect(startHostUpdate()).rejects.toThrow("audit store is down"); + await runToCompletion(await startHostUpdate()); + }); + + it("keeps a panel update out while a host-stack update holds the lock", async () => { + const recorded = holdAudit(); + const hostUpdate = startHostUpdate(); + await vi.waitFor(() => expect(mocks.audit).toHaveBeenCalledTimes(1)); + + await expect(createCaller().updateServer()).rejects.toMatchObject({ + code: "CONFLICT", + }); + expect(mocks.startServerUpdate).not.toHaveBeenCalled(); + + recorded(); + await runToCompletion(await hostUpdate); + }); + + it("starts a panel update when no host-stack update is running", async () => { + await expect(createCaller().updateServer()).resolves.toBe(true); + expect(mocks.startServerUpdate).toHaveBeenCalledTimes(1); + }); +}); diff --git a/apps/dokploy/__test__/settings/host-stack-log.test.ts b/apps/dokploy/__test__/settings/host-stack-log.test.ts new file mode 100644 index 00000000000..be96a652efe --- /dev/null +++ b/apps/dokploy/__test__/settings/host-stack-log.test.ts @@ -0,0 +1,60 @@ +import { + COMPONENTS_UPDATE_DONE, + COMPONENTS_UPDATE_FAILED, +} from "@dokploy/server/setup/server-components"; +import { describe, expect, it } from "vitest"; +import { + UPDATE_DONE, + UPDATE_FAILED, + updateOutcome, +} from "@/components/dashboard/settings/servers/host-stack-log"; + +const outcomesFor = (chunks: string[]) => { + let log = ""; + const outcomes: string[] = []; + for (const chunk of chunks) { + const outcome = updateOutcome(log, `${log}${chunk}`); + log += chunk; + if (outcome !== null) { + outcomes.push(outcome); + } + } + return outcomes; +}; + +describe("updateOutcome", () => { + it("uses the same markers as the server", () => { + expect(UPDATE_DONE).toBe(COMPONENTS_UPDATE_DONE); + expect(UPDATE_FAILED).toBe(COMPONENTS_UPDATE_FAILED); + }); + + it("reports a success marker split across two chunks", () => { + expect( + outcomesFor([ + "Redis version 8.10.2 installed ✅\nComponents update fin", + "ished ✅\n", + ]), + ).toEqual(["done"]); + }); + + it("reports a failure marker split across two chunks", () => { + expect(outcomesFor(["Components update fa", "iled ❌\n"])).toEqual([ + "failed", + ]); + }); + + it("reports a marker once, even when more output follows it", () => { + expect( + outcomesFor([`${COMPONENTS_UPDATE_DONE}\n`, "trailing output\n"]), + ).toEqual(["done"]); + }); + + it("reports nothing while no marker has arrived", () => { + expect( + outcomesFor([ + "Pulling redis:8.10.2\n", + "Traefik version 3.6.1 installed ✅\n", + ]), + ).toEqual([]); + }); +}); diff --git a/apps/dokploy/__test__/settings/install-agenthits-update.test.ts b/apps/dokploy/__test__/settings/install-agenthits-update.test.ts index 321c30bc35a..b9195d8dc73 100644 --- a/apps/dokploy/__test__/settings/install-agenthits-update.test.ts +++ b/apps/dokploy/__test__/settings/install-agenthits-update.test.ts @@ -48,7 +48,14 @@ case "$1" in exit 0 ;; ps) - echo fake-task-container + case "$*" in + *"name=dokploy-postgres"*"ancestor="*) + case "$*" in + *"ancestor=$FAKE_POSTGRES_IMAGE") echo fake-task-container ;; + esac + ;; + *) echo fake-task-container ;; + esac exit 0 ;; exec) @@ -629,19 +636,42 @@ describe("install-agenthits.sh update", () => { SPAWN_TEST_TIMEOUT_MS, ); it( - "refuses a Postgres major version change before anything runs", + "keeps an installed Postgres whose major differs from the pin and runs the other steps", () => { - const { result, calls } = runInstaller({ + const { result, calls, backups } = runInstaller({ postgresImage: "postgres:17.5", + redisImage: "redis:8.10.1", + traefikImage: "traefik:v3.6.25", }); - expect(result.status).not.toBe(0); - expect(result.stderr).toContain("changes the major version"); - expect(result.stderr).toContain("Nothing was changed"); - expect(calls.some((call) => call.startsWith("pull "))).toBe(false); - expect(calls.some((call) => call.startsWith("service update"))).toBe( - false, + expect(result.status).toBe(0); + expect(result.stdout).toContain( + "Postgres 17 stays. A major upgrade is a separate migration; this update does not change it.", + ); + expect(result.stderr).not.toContain("changes the major version"); + expect(backups).toEqual([]); + expect(calls.some((call) => call.includes("pg_dumpall"))).toBe(false); + expect(calls).not.toContain("pull postgres:18.6"); + expect( + calls.some( + (call) => + call.startsWith("service update") && + call.includes("dokploy-postgres"), + ), + ).toBe(false); + + const redis = firstIndex( + calls, + "service update --detach --update-order stop-first --update-failure-action rollback --image redis:8.10.2 dokploy-redis", ); + const traefikStop = firstIndex(calls, "stop dokploy-traefik"); + const traefikStart = firstIndex(calls, "start dokploy-traefik"); + const panel = firstIndex(calls, "buildx imagetools inspect"); + expect(redis).toBeGreaterThanOrEqual(0); + expect(redis).toBeLessThan(traefikStop); + expect(traefikStop).toBeLessThan(traefikStart); + expect(traefikStart).toBeLessThan(panel); + expect(result.stdout).toContain("Panel host update finished."); }, SPAWN_TEST_TIMEOUT_MS, ); @@ -709,6 +739,29 @@ describe("install-agenthits.sh Docker Engine upgrade", () => { SPAWN_TEST_TIMEOUT_MS, ); + it( + "finishes the Docker Engine upgrade while Postgres keeps its major version", + () => { + const { result, calls } = runInstaller({ + postgresImage: "postgres:17.5", + upgradeEngine: true, + engineVersion: "28.3.0", + }); + + expect(result.status).toBe(0); + expect(result.stdout).toContain("Postgres 17 stays."); + expect(result.stdout).toContain("Docker Engine is 29.8.2"); + expect(result.stdout).toContain("Panel host update finished."); + expect(result.stderr).not.toContain("rolled back"); + expect( + calls.includes( + `apt-get install -y -qq --no-download docker-ce=${DOCKER_NEW} docker-ce-cli=${DOCKER_NEW} containerd.io docker-buildx-plugin docker-compose-plugin`, + ), + ).toBe(true); + }, + SPAWN_TEST_TIMEOUT_MS, + ); + it( "stops before any change when a current Docker package cannot be saved", () => { diff --git a/apps/dokploy/__test__/settings/request-log-authorization.test.ts b/apps/dokploy/__test__/settings/request-log-authorization.test.ts index 49b231f54d4..6def3f05dc4 100644 --- a/apps/dokploy/__test__/settings/request-log-authorization.test.ts +++ b/apps/dokploy/__test__/settings/request-log-authorization.test.ts @@ -99,6 +99,7 @@ vi.mock("@dokploy/server", async () => ({ getDokployImageKeepCount: vi.fn(), getDokployImages: vi.fn(), IS_CLOUD: false, + HOST_STACK_COMPONENTS: ["panel", "docker", "traefik", "postgres", "redis"], checkGPUStatus: mocks.checkGPUStatus, checkPortInUse: mocks.checkPortInUse, checkPostgresHealth: mocks.checkPostgresHealth, diff --git a/apps/dokploy/components/dashboard/settings/servers/host-stack-log.ts b/apps/dokploy/components/dashboard/settings/servers/host-stack-log.ts new file mode 100644 index 00000000000..a8c00413acc --- /dev/null +++ b/apps/dokploy/components/dashboard/settings/servers/host-stack-log.ts @@ -0,0 +1,20 @@ +export const UPDATE_DONE = "Components update finished ✅"; +export const UPDATE_FAILED = "Components update failed ❌"; + +// A chunk can end in the middle of a marker, so the outcome is read from the +// whole log. Only the chunk that completes a marker reports it, so a marker +// that is already in the log is not reported again. +export const updateOutcome = ( + previousLog: string, + log: string, +): "done" | "failed" | null => { + const completes = (marker: string) => + !previousLog.includes(marker) && log.includes(marker); + if (completes(UPDATE_DONE)) { + return "done"; + } + if (completes(UPDATE_FAILED)) { + return "failed"; + } + return null; +}; diff --git a/apps/dokploy/components/dashboard/settings/servers/panel-host-stack.tsx b/apps/dokploy/components/dashboard/settings/servers/panel-host-stack.tsx new file mode 100644 index 00000000000..89c72ec16f5 --- /dev/null +++ b/apps/dokploy/components/dashboard/settings/servers/panel-host-stack.tsx @@ -0,0 +1,387 @@ +import type { + HostStackComponent, + HostStackRow, +} from "@dokploy/server/setup/host-stack-rows"; +import { + AlertTriangle, + CheckCircle2, + Loader2, + RefreshCw, + Server, +} from "lucide-react"; +import { useRef, useState } from "react"; +import { toast } from "sonner"; +import { + AlertDialog, + AlertDialogAction, + AlertDialogCancel, + AlertDialogContent, + AlertDialogDescription, + AlertDialogFooter, + AlertDialogHeader, + AlertDialogTitle, +} from "@/components/ui/alert-dialog"; +import { Badge } from "@/components/ui/badge"; +import { Button } from "@/components/ui/button"; +import { + Card, + CardContent, + CardDescription, + CardHeader, + CardTitle, +} from "@/components/ui/card"; +import { Checkbox } from "@/components/ui/checkbox"; +import { + Dialog, + DialogContent, + DialogDescription, + DialogHeader, + DialogTitle, + DialogTrigger, +} from "@/components/ui/dialog"; +import { Label } from "@/components/ui/label"; +import { cn } from "@/lib/utils"; +import { api } from "@/utils/api"; +import { UpdateServer } from "../web-server/update-server"; +import { updateOutcome } from "./host-stack-log"; + +const TRAEFIK_CONNECTION_NOTE = + "If the connection to the panel dropped while Traefik was recreated, the update can still finish on the host. Reload this page to see the result."; + +const LABELS: Record = { + panel: "Panel", + docker: "Docker", + traefik: "Traefik", + postgres: "Postgres", + redis: "Redis", +}; + +const WARNINGS: Partial> = { + traefik: + "Traefik is recreated; the admin site is unreachable for a few seconds.", + redis: "Redis is restarted; it is unavailable for a few seconds.", +}; + +const StatusLine = ({ row }: { row: HostStackRow }) => { + if (row.status === "outdated") { + return ( + + + Outdated + + ); + } + if (row.status === "current") { + return ( + + + Up to date + + ); + } + return Unknown; +}; + +const HostStackRowItem = ({ + row, + checked, + onCheckedChange, + onOpenWebUpdate, +}: { + row: HostStackRow; + checked: boolean; + onCheckedChange: (checked: boolean) => void; + onOpenWebUpdate: () => void; +}) => { + const checkboxId = `host-stack-${row.name}`; + return ( +
+
+ {LABELS[row.name]} + + installed: {row.installed ?? "not found"} + + + target: {row.target ?? "not pinned"} + +
+
+ + {row.action === "panel" && ( + + )} + {row.action === "ui" && ( +
+ onCheckedChange(value === true)} + /> + +
+ )} + {row.reason && + (row.action === "manual" || row.status === "unknown") && ( +

{row.reason}

+ )} + {row.action === "manual" && row.command && ( + + {row.command} + + )} +
+
+ ); +}; + +const PanelHostCard = ({ + rows, + refetch, +}: { + rows: HostStackRow[]; + refetch: () => unknown; +}) => { + const [isOpen, setIsOpen] = useState(false); + const [isConfirmOpen, setIsConfirmOpen] = useState(false); + const [isWebUpdateOpen, setIsWebUpdateOpen] = useState(false); + const [chosen, setChosen] = useState< + Partial> + >({}); + const [isUpdating, setIsUpdating] = useState(false); + const [runComponents, setRunComponents] = useState([]); + const [logs, setLogs] = useState(""); + const logBuffer = useRef(""); + + const outdatedCount = rows.filter((row) => row.outdated).length; + const unknownCount = rows.filter((row) => row.status === "unknown").length; + const selected = rows + .filter( + (row) => + row.action === "ui" && row.outdated && (chosen[row.name] ?? true), + ) + .map((row) => row.name); + const summary = rows + .filter((row) => row.name !== "panel") + .map((row) => `${LABELS[row.name]} ${row.installed ?? "unknown"}`) + .join(" · "); + + api.settings.updateHostStack.useSubscription( + { components: runComponents }, + { + enabled: isUpdating, + onData(log) { + const previous = logBuffer.current; + logBuffer.current = `${previous}${log}`; + setLogs(logBuffer.current); + const outcome = updateOutcome(previous, logBuffer.current); + if (outcome === null) { + return; + } + setIsUpdating(false); + if (outcome === "done") { + toast.success("Panel host components updated"); + } else { + toast.error("Panel host update failed, see the log"); + } + void refetch(); + }, + onError(err) { + const note = runComponents.includes("traefik") + ? `\n${TRAEFIK_CONNECTION_NOTE}` + : ""; + logBuffer.current = `${logBuffer.current}${err.message}${note}\n`; + setLogs(logBuffer.current); + setIsUpdating(false); + toast.error("Panel host update did not finish"); + void refetch(); + }, + }, + ); + + const startUpdate = () => { + logBuffer.current = ""; + setLogs(""); + setRunComponents(selected); + setIsConfirmOpen(false); + setIsUpdating(true); + }; + + return ( + <> + + +
+ + + Panel host (this server) + + {outdatedCount > 0 ? ( + {outdatedCount} to update + ) : unknownCount > 0 ? ( + Some versions unknown + ) : ( + Up to date + )} +
+ + Docker, Traefik, Postgres and Redis on the server that runs this + panel. + +
+ +

{summary}

+ + + + + + + + Panel host components + + + Versions on the server that runs this panel, compared with the + versions pinned in Dokploy. + + + +
+ {rows.map((row) => ( + + setChosen((prev) => ({ ...prev, [row.name]: value })) + } + onOpenWebUpdate={() => { + setIsOpen(false); + setIsWebUpdateOpen(true); + }} + /> + ))} +
+ + {selected.map((name) => + WARNINGS[name] ? ( +

+ {WARNINGS[name]} +

+ ) : null, + )} + + + + {logs && ( +
+									{logs}
+								
+ )} +
+
+
+
+ + {/* Opened from the row above, so the dialog's own trigger stays hidden. */} + + + + + + + + Update {selected.map((name) => LABELS[name]).join(" and ")} on the + panel host? + + + {selected.map((name) => + WARNINGS[name] ? ( + + {WARNINGS[name]} + + ) : null, + )} + + + + Cancel + Update + + + + + ); +}; + +export const PanelHostStack = () => { + const { data, error, isPending, refetch } = api.settings.hostStack.useQuery( + undefined, + { + refetchOnWindowFocus: false, + staleTime: 60 * 1000, + retry: false, + }, + ); + + if (isPending) { + return ( + + + + Checking the panel host... + + + ); + } + + if (error) { + if ( + error.data?.code === "UNAUTHORIZED" || + error.data?.code === "FORBIDDEN" + ) { + return null; + } + return ( + + + The panel host check failed: {error.message} + + + ); + } + + if (!data) { + return null; + } + + return ; +}; diff --git a/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx b/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx index b6d4cc7fb17..89963993b07 100644 --- a/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx +++ b/apps/dokploy/components/dashboard/settings/servers/show-servers.tsx @@ -33,6 +33,7 @@ import { TerminalModal } from "../web-server/terminal-modal"; import { ShowServerActions } from "./actions/show-server-actions"; import { DeleteServerModal } from "./delete-server-modal"; import { HandleServers } from "./handle-servers"; +import { PanelHostStack } from "./panel-host-stack"; import { ServerComponentsUpdate } from "./server-components-update"; import { SetupServer } from "./setup-server"; import { ShowMonitoringModal } from "./show-monitoring-modal"; @@ -124,6 +125,7 @@ export const ShowServers = () => { )} + {isPending ? (
Loading... diff --git a/apps/dokploy/server/api/routers/settings.ts b/apps/dokploy/server/api/routers/settings.ts index 9c21426da19..e67b39c3333 100644 --- a/apps/dokploy/server/api/routers/settings.ts +++ b/apps/dokploy/server/api/routers/settings.ts @@ -1,5 +1,7 @@ import { + acquireHostStackUpdateLock, CLEANUP_CRON_JOB, + COMPONENTS_UPDATE_FAILED, checkGPUStatus, checkPortInUse, checkPostgresHealth, @@ -30,7 +32,11 @@ import { getUpdateData, getUpdateDiskSpace, getWebServerSettings, + HOST_STACK_COMPONENTS, + hostStackRefusals, IS_CLOUD, + isHostStackUiComponent, + isHostStackUpdateRunning, isServerUpdateRunning, parseRawConfig, paths, @@ -40,6 +46,7 @@ import { readConfigInPath, readDirectory, readEnvironmentVariables, + readHostStackRows, readMainConfig, readMonitoringConfig, readPorts, @@ -53,6 +60,7 @@ import { startLogCleanup, startServerUpdate, stopLogCleanup, + updateHostStackComponents, updateLetsEncryptEmail, updateServerById, updateServerTraefik, @@ -67,6 +75,7 @@ import { checkPermission } from "@dokploy/server/services/permission"; import { isProtectedTraefikPath } from "@dokploy/server/utils/traefik/application"; import { generateOpenApiDocument } from "@dokploy/trpc-openapi"; import { TRPCError } from "@trpc/server"; +import { observable } from "@trpc/server/observable"; import { eq, sql } from "drizzle-orm"; import { scheduledJobs, scheduleJob } from "node-schedule"; import { parse, stringify } from "yaml"; @@ -193,6 +202,79 @@ export const settingsRouter = createTRPCRouter({ }); return true; }), + hostStack: adminProcedure.query(async ({ ctx }) => { + await assertLocalHostAccess(ctx); + if (IS_CLOUD) { + return null; + } + return await readHostStackRows(packageInfo.version); + }), + updateHostStack: adminProcedure + .meta({ + openapi: { + path: "/settings/update-host-stack", + method: "POST", + override: true, + enabled: false, + }, + }) + .input( + z.object({ + components: z.array(z.enum(HOST_STACK_COMPONENTS)).min(1), + }), + ) + .subscription(async ({ input, ctx }) => { + await assertLocalHostAccess(ctx); + if (IS_CLOUD) { + throw new TRPCError({ + code: "FORBIDDEN", + message: "The panel host components are not managed here.", + }); + } + const refusals = hostStackRefusals(input.components); + if (refusals.length > 0) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: refusals.join(" "), + }); + } + const releaseHostStackLock = isServerUpdateRunning() + ? null + : acquireHostStackUpdateLock(); + if (!releaseHostStackLock) { + throw new TRPCError({ + code: "CONFLICT", + message: + "Another update of this panel host is running. Try again when it finishes.", + }); + } + const components = input.components.filter(isHostStackUiComponent); + try { + await audit(ctx, { + action: "update", + resourceType: "settings", + resourceName: "panel-host", + metadata: { components }, + }); + } catch (error) { + releaseHostStackLock(); + throw error; + } + return observable((emit) => { + updateHostStackComponents(components, (log) => { + emit.next(log); + }) + .catch((error) => { + emit.next( + `${error instanceof Error ? error.message : error}\n${COMPONENTS_UPDATE_FAILED}\n`, + ); + }) + .finally(() => { + releaseHostStackLock(); + emit.complete(); + }); + }); + }), toggleDashboard: adminProcedure .input(apiEnableDashboard) .mutation(async ({ input, ctx }) => { @@ -659,6 +741,14 @@ export const settingsRouter = createTRPCRouter({ const keepImages = input?.keepImages; const data = await getUpdateData(packageInfo.version); + // After the await: a host-stack update can start while the update check runs. + if (isHostStackUpdateRunning()) { + throw new TRPCError({ + code: "CONFLICT", + message: + "The panel host components are being updated. Try again when they finish.", + }); + } if (data.updateAvailable) { startServerUpdate( data.updateSource === "agenthits" diff --git a/apps/dokploy/server/api/utils/procedure-access.ts b/apps/dokploy/server/api/utils/procedure-access.ts index 7b755f4e555..4cc7778f49f 100644 --- a/apps/dokploy/server/api/utils/procedure-access.ts +++ b/apps/dokploy/server/api/utils/procedure-access.ts @@ -610,6 +610,7 @@ export const PROCEDURE_ACCESS: Record = { "settings.haveActivateRequests": "read", "settings.haveTraefikDashboardPortEnabled": "read", "settings.health": "read", + "settings.hostStack": "read", "settings.isCloud": "read", "settings.isUserSubscribed": "read", "settings.readDirectories": "read", @@ -629,6 +630,7 @@ export const PROCEDURE_ACCESS: Record = { "settings.updateBuildsConcurrency": "write", "settings.updateDockerCleanup": "write", "settings.updateEnforceSSO": "dangerous", + "settings.updateHostStack": "dangerous", "settings.updateLogCleanup": "write", "settings.updateMiddlewareTraefikConfig": "dangerous", "settings.updateRemoteServersOnly": "write", diff --git a/docs/agenthits-install.md b/docs/agenthits-install.md index 986927067c8..58516ee23bc 100644 --- a/docs/agenthits-install.md +++ b/docs/agenthits-install.md @@ -229,8 +229,9 @@ Windows (PowerShell от администратора): повторного `docker pull` и без перезапуска `dokploy`. `update` обновляет весь стек панели, а не только её образ. Сначала скачивается -всё, что понадобится: образы Postgres, Redis, Traefik и панели (закреплённые -теги) и `update.sh`. Если что-то не скачалось, команда завершается строкой +всё, что понадобится: образы Redis, Traefik и панели (закреплённые теги), образ +Postgres, если его major-версия совпадает с закреплённой, и `update.sh`. Если +что-то не скачалось, команда завершается строкой `Pre-download failed: ... Nothing was changed.`, и ни один сервис не трогается. Затем шаги идут в таком порядке: @@ -247,7 +248,8 @@ Windows (PowerShell от администратора): после этого запускается. В течение `DOKPLOY_TRAEFIK_SETTLE` секунд (10 по умолчанию) installer раз в секунду проверяет, что он запущен и без перезапусков. Как только это перестает выполняться, обновление откатывается, не дожидаясь конца окна. В конце проверка повторяется, и только после нее ему выставляется `--restart always`. Если на любом шаге после остановки старого контейнера что-то не сходится, старый образ создается заново с теми же сетями и запускается, а команда завершается ошибкой с названием шага. -4. Postgres: `docker service update` с остановкой старой задачи до запуска новой, +4. Postgres: только если его major-версия совпадает с закреплённой. Выполняется + `docker service update` с остановкой старой задачи до запуска новой, потому что две задачи не могут работать с одним volume. Ждёт `pg_isready`. 5. Панель: `update.sh`. Ждёт `healthy`-статуса контейнера; при откате swarm или нездоровом статусе команда завершается с ошибкой. @@ -276,9 +278,11 @@ Windows (PowerShell от администратора): Серверы, которые панель обновляет по SSH (раздел Components), ждут 30 секунд, пока Docker отвечает закрепленной версией. Если за это время Docker не проходит проверку, Docker откатывается сразу после истечения окна, а `dokploy-traefik` запускается снова с `--restart always`. -Смена major-версии Postgres (например, 17 → 18) этим обновлением не выполняется: -команда останавливается до любых изменений. Перенос данных между major-версиями -делается отдельно (dump и restore или `pg_upgrade`). +Смена major-версии Postgres (например, 17 → 18) этим обновлением не выполняется. +Если установленная major-версия отличается от закреплённой, installer печатает +`Postgres 17 stays. A major upgrade is a separate migration; this update does not change it.`, +не делает резервную копию, не трогает Postgres и выполняет остальные шаги. +Перенос данных между major-версиями делается отдельно (dump и restore или `pg_upgrade`). Простой на каждом шаге равен времени остановки старой задачи (или контейнера) и запуска новой: образы уже локальные, поэтому скачивание в простое не участвует. diff --git a/docs/server-maintenance.md b/docs/server-maintenance.md new file mode 100644 index 00000000000..7da63ad48cc --- /dev/null +++ b/docs/server-maintenance.md @@ -0,0 +1,81 @@ +# Обслуживание сервера панели + +Этот текст для того, кто отвечает за сервер панели, но не пишет код. Здесь написано, что обновляется кнопкой, что обновляется только в плановое окно, какие порты открыты и как безопасно запустить обновление. + +## 1. Что обновляет кнопка + +В карточке **Panel host (this server)** есть два способа обновления: + +- **Components** обновляет **Traefik** (веб-сервер, через который идёт весь HTTPS-трафик) и **Redis** (кэш панели). Во время замены Traefik на несколько секунд недоступны все сайты и админка. Во время замены Redis он на несколько секунд недоступен. +- **Web Server Update** обновляет саму **панель**. + +## 2. Что обновляется только в плановое окно + +Docker и Postgres кнопкой не обновляются. Карточка показывает для них команду, которую запускают в заранее выбранное время. + +**Docker Engine.** При обновлении перезапускается демон Docker. Вместе с ним останавливаются все контейнеры сервера, включая панель. Поэтому это делают один раз, в объявленное окно, и предупреждают пользователей о простое. Команда: `DOCKER_ENGINE_UPGRADE=1 bash install-agenthits.sh update`. + +**Postgres (база панели).** Смена major-версии (например, 17 → 18) требует переноса данных. Это отдельная работа, а не обновление. Поэтому сервер сохраняет текущую major-версию: установщик её не меняет. Сейчас с Postgres ничего делать не нужно. Обновление внутри одной major-версии (например, 18.5 → 18.6) выполняет та же команда обновления. Перед заменой она сохраняет резервную копию базы. + +## 3. Порты + +Для входа по домену открыт только порт **443** (TCP и UDP): через него работают сайты и панель по HTTPS. UDP нужен для HTTP/3. Порт **80** не открывается никогда. Сертификаты выпускаются и продлеваются только через порт 443 (TLS-challenge). + +- **3000** (TCP) — прямой вход в панель. Адрес с этим портом установщик печатает в конце установки. +- **SSH** — вход на сервер. + +Если включён UFW (команда `bash install-agenthits.sh harden --ufw`), входящие подключения разрешены только для SSH, 443 и 3000. Правила UFW не закрывают порты, которые публикует Docker, поэтому закрыть их таким правилом нельзя. + +## 4. Как безопасно запустить обновление + +1. Выберите тихое время. Пока заменяется Traefik, все сайты на сервере недоступны. Пока обновляется панель, недоступна её админка. При обновлении Docker останавливается весь сервер. +2. Проверьте свободное место: `df -h /var`. Резервная копия Postgres и копии пакетов Docker лежат в `/var/backups/dokploy`. +3. Запустите одну команду в папке, где лежит файл установщика `install-agenthits.sh`: + + ```bash + bash install-agenthits.sh update + ``` + + Если вы не root, скрипт сам перезапустится через sudo и спросит пароль. На Mac команда та же, но без sudo. + + Чтобы вывод сохранился в файл, запустите так: + + ```bash + bash install-agenthits.sh update 2>&1 | tee update-$(date +%F).log + ``` + +4. Смотрите вывод. Что значат строки: + + | Строка | Что значит | + |---|---| + | `Pre-download failed: <образ>. Nothing was changed.` | Образ не скачался. Сервер не тронут, команду можно повторить позже. | + | `Error: ... Nothing was changed.` | Остановка до любых изменений. Текст ошибки говорит, что именно не так. | + | `Postgres stays. A major upgrade is a separate migration; this update does not change it.` | Major-версия Postgres отличается от закреплённой. Postgres остаётся как есть, копия не делается. Это штатно. | + | `Postgres backup written to <файл>` | Резервная копия базы сохранена перед заменой Postgres. | + | `Traefik already runs <образ>` | Traefik уже нужной версии, менять нечего. | + | `Traefik runs <образ>` | Traefik заменён на новую версию. | + | `<сервис> runs <образ> (previous image kept: <образ>)` | Сервис заменён. Старый образ записан в скобках, он нужен для ручного отката. | + | Строка про панель с `already up to date` | Панель уже на нужной версии. | + | Строка про панель с `updated to` | Панель обновлена сейчас. | + | `Docker Engine is <версия>` | Docker обновлён и работает. Строка появляется только при `DOCKER_ENGINE_UPGRADE=1`. | + | `Panel host update finished.` | Обновление завершено без ошибок. | + +5. Проверьте результат: + + ```bash + docker service ls + ``` + + У `dokploy`, `dokploy-postgres` и `dokploy-redis` в колонке REPLICAS должно быть `1/1`. Затем откройте панель в браузере. + +6. Если обновление остановилось с ошибкой, не запускайте его снова вслепую. Сначала посмотрите логи (раздел 5). + +## 5. Где логи + +- Установщик на диск логи не пишет. Вывод виден в терминале, а если вы использовали `tee`, он сохранён в файл. +- Панель: `docker service logs --tail 120 dokploy` +- Postgres: `docker service logs --tail 120 dokploy-postgres` +- Redis: `docker service logs --tail 120 dokploy-redis` +- Traefik: `docker logs --tail 120 dokploy-traefik` +- Резервные копии Postgres: `/var/backups/dokploy/`, файлы вида `postgres-<дата>.sql.gz`. Доступны только root, сами не удаляются. +- Копии пакетов Docker, если обновляли Docker: `/var/backups/dokploy/docker-rollback-<дата>/`. diff --git a/install-agenthits.sh b/install-agenthits.sh index 456486a8c98..6ffe26fbecb 100755 --- a/install-agenthits.sh +++ b/install-agenthits.sh @@ -61,8 +61,10 @@ DOCKER_ENGINE_PREVIOUS_VERSION="" DOCKER_ROLLBACK_DIR="" DOCKER_ROLLBACK_DIR_CREATED=0 TRAEFIK_EXTRA_NETWORKS="" +TRAEFIK_HTTP_PUBLISH=1 TRAEFIK_OLD_IMAGE="" TRAEFIK_OLD_STOPPED=0 +POSTGRES_KEPT=0 command_exists() { command -v "$@" >/dev/null 2>&1 @@ -1168,7 +1170,11 @@ resolve_panel_update_script() { predownload_update_artifacts() { resolve_panel_update_script local image="" - for image in "$POSTGRES_IMAGE" "$REDIS_IMAGE" "$TRAEFIK_IMAGE" "$DOKPLOY_IMAGE"; do + local images=("$REDIS_IMAGE" "$TRAEFIK_IMAGE" "$DOKPLOY_IMAGE") + if [ "$POSTGRES_KEPT" = "0" ]; then + images=("$POSTGRES_IMAGE" "${images[@]}") + fi + for image in "${images[@]}"; do if ! download_with_retry docker pull "$image"; then echo "Pre-download failed: $image. Nothing was changed." >&2 exit 1 @@ -1207,9 +1213,10 @@ redis_ready() { [ -n "$container" ] && [ "$(docker exec "$container" redis-cli ping 2>/dev/null)" = "PONG" ] } +# A kept Postgres runs its installed major, so readiness follows the service's image, not the pinned one. postgres_ready() { local container="" - container="$(task_container dokploy-postgres "$POSTGRES_IMAGE")" + container="$(task_container dokploy-postgres "$(service_image dokploy-postgres)")" [ -n "$container" ] && docker exec "$container" pg_isready -U dokploy -d dokploy >/dev/null 2>&1 } @@ -1334,13 +1341,17 @@ traefik_extra_networks() { create_dokploy_traefik() { local image="$1" local restart="$2" + local http_publish=() + if [ "${TRAEFIK_HTTP_PUBLISH:-1}" = "1" ]; then + http_publish=(-p 80:80/tcp) + fi docker create \ --name dokploy-traefik \ --restart "$restart" \ -v /etc/dokploy/traefik/traefik.yml:/etc/traefik/traefik.yml \ -v /etc/dokploy/traefik/dynamic:/etc/dokploy/traefik/dynamic \ -v /var/run/docker.sock:/var/run/docker.sock:ro \ - -p 80:80/tcp \ + ${http_publish[@]+"${http_publish[@]}"} \ -p 443:443/tcp \ -p 443:443/udp \ "$image" >/dev/null || return 1 @@ -1407,6 +1418,10 @@ swap_dokploy_traefik() { fi TRAEFIK_OLD_IMAGE="$current" TRAEFIK_EXTRA_NETWORKS="$(traefik_extra_networks)" + TRAEFIK_HTTP_PUBLISH=1 + if [ "$existed" = "1" ] && ! docker inspect -f '{{json .HostConfig.PortBindings}}' dokploy-traefik | grep -q '"80/tcp"'; then + TRAEFIK_HTTP_PUBLISH=0 + fi TRAEFIK_OLD_STOPPED=0 trap 'traefik_restore "being interrupted"' HUP INT TERM docker rm -f dokploy-traefik-previous >/dev/null 2>&1 || true @@ -1527,16 +1542,18 @@ update_agenthits_dokploy() { require_update_services local current_postgres="" + local current_postgres_major="" current_postgres="$(image_without_digest "$(service_image dokploy-postgres)")" - if [ "$(get_postgres_major_version "$current_postgres" || true)" != "$(get_postgres_major_version || true)" ]; then - echo "Error: Postgres $current_postgres -> $POSTGRES_IMAGE changes the major version, which this update does not migrate. Nothing was changed." >&2 - exit 1 + current_postgres_major="$(get_postgres_major_version "$current_postgres" || true)" + if [ "$current_postgres_major" != "$(get_postgres_major_version || true)" ]; then + POSTGRES_KEPT=1 + echo "Postgres ${current_postgres_major:-$current_postgres} stays. A major upgrade is a separate migration; this update does not change it." fi predownload_update_artifacts local postgres_swap=0 - if [ "$(image_without_digest "$(service_image dokploy-postgres)")" != "$(image_without_digest "$POSTGRES_IMAGE")" ]; then + if [ "$POSTGRES_KEPT" = "0" ] && [ "$(image_without_digest "$(service_image dokploy-postgres)")" != "$(image_without_digest "$POSTGRES_IMAGE")" ]; then postgres_swap=1 backup_postgres fi diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index a9aca1d3f5a..c1c52b4d253 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -64,6 +64,9 @@ export * from "./services/web-server-settings"; export * from "./services/web-server-update"; export * from "./setup/config-paths"; export * from "./setup/forward-auth-setup"; +export * from "./setup/host-stack"; +export * from "./setup/host-stack-lock"; +export * from "./setup/host-stack-rows"; export * from "./setup/monitoring-setup"; export * from "./setup/postgres-setup"; export * from "./setup/server-audit"; diff --git a/packages/server/src/setup/host-stack-lock.ts b/packages/server/src/setup/host-stack-lock.ts new file mode 100644 index 00000000000..4d2e8da51f4 --- /dev/null +++ b/packages/server/src/setup/host-stack-lock.ts @@ -0,0 +1,19 @@ +let hostStackUpdating = false; + +export const isHostStackUpdateRunning = () => hostStackUpdating; + +// The check and the set run in one synchronous step, so no other request can +// pass the check while the caller is still awaiting something. +export const acquireHostStackUpdateLock = (): (() => void) | null => { + if (hostStackUpdating) { + return null; + } + hostStackUpdating = true; + let held = true; + return () => { + if (held) { + held = false; + hostStackUpdating = false; + } + }; +}; diff --git a/packages/server/src/setup/host-stack-rows.ts b/packages/server/src/setup/host-stack-rows.ts new file mode 100644 index 00000000000..14ecc22eb23 --- /dev/null +++ b/packages/server/src/setup/host-stack-rows.ts @@ -0,0 +1,250 @@ +import { PINNED_VERSIONS } from "./component-versions"; +import { TRAEFIK_VERSION } from "./traefik-setup"; + +export const HOST_STACK_COMPONENTS = [ + "panel", + "docker", + "traefik", + "postgres", + "redis", +] as const; + +export type HostStackComponent = (typeof HOST_STACK_COMPONENTS)[number]; + +export const HOST_STACK_UI_COMPONENTS = ["traefik", "redis"] as const; + +export type HostStackUiComponent = (typeof HOST_STACK_UI_COMPONENTS)[number]; + +export type HostStackAction = "panel" | "ui" | "manual"; + +export type HostStackStatus = "current" | "outdated" | "unknown"; + +export type HostStackRow = { + name: HostStackComponent; + installed: string | null; + target: string | null; + status: HostStackStatus; + outdated: boolean; + action: HostStackAction; + reason: string | null; + command: string | null; +}; + +export type HostStackReadings = { + docker: string | null; + traefikImage: string | null; + postgresImage: string | null; + redisImage: string | null; + panel: { + installed: string | null; + latest: string | null; + updateAvailable: boolean | null; + }; +}; + +export const isHostStackUiComponent = ( + name: HostStackComponent, +): name is HostStackUiComponent => + (HOST_STACK_UI_COMPONENTS as readonly string[]).includes(name); + +const INSTALLER_UPDATE = "bash install-agenthits.sh update"; +const DOCKER_UPGRADE_COMMAND = `DOCKER_ENGINE_UPGRADE=1 ${INSTALLER_UPDATE}`; +const UNKNOWN_REASON = + "The installed version could not be read on the panel host."; +const PANEL_UNKNOWN_REASON = + "The update check did not return a version for this panel."; +const DOCKER_REASON = + "Docker Engine upgrades restart the daemon, which stops this panel. Run them only in the maintenance window."; +const POSTGRES_REASON = + "Postgres changes only in the maintenance window, and keeps its major version."; + +const HOST_STACK_REFUSALS: Record< + Exclude, + string +> = { + panel: "The panel is updated from the Web Server Update dialog.", + docker: `Docker Engine is not upgraded from the panel because the daemon restart stops it. In the maintenance window run: ${DOCKER_UPGRADE_COMMAND}`, + postgres: + "Postgres is not updated from the panel, and its major version never changes here. In the maintenance window run the host update with POSTGRES_IMAGE set to the image Postgres runs now (see the Panel host components list).", +}; + +const withoutDigest = (image: string) => image.split("@")[0] ?? image; + +const imageTag = (image: string) => { + const name = withoutDigest(image); + const slash = name.lastIndexOf("/"); + const colon = name.lastIndexOf(":"); + return colon > slash ? name.slice(colon + 1) : null; +}; + +const VERSION_PATTERN = /\d+(?:\.\d+)*/; + +const versionOf = (value: string | null) => + value?.match(VERSION_PATTERN)?.[0] ?? null; + +const imageVersion = (image: string | null) => + versionOf(image ? imageTag(image) : null); + +const postgresVersion = (image: string | null) => + (image ? imageTag(image) : null)?.match(/^\d+(?:\.\d+)*/)?.[0] ?? null; + +const compareVersions = (left: string, right: string) => { + const leftParts = left.split(".").map(Number); + const rightParts = right.split(".").map(Number); + const length = Math.max(leftParts.length, rightParts.length); + for (let index = 0; index < length; index += 1) { + const difference = (leftParts[index] ?? 0) - (rightParts[index] ?? 0); + if (difference !== 0) { + return difference; + } + } + return 0; +}; + +const unknownRow = ( + name: HostStackComponent, + target: string | null, + action: HostStackAction, +): HostStackRow => ({ + name, + installed: null, + target, + status: "unknown", + outdated: false, + action, + reason: UNKNOWN_REASON, + command: null, +}); + +const panelRow = ({ + installed, + latest, + updateAvailable, +}: HostStackReadings["panel"]): HostStackRow => { + if (updateAvailable === null) { + return { + name: "panel", + installed, + target: latest, + status: "unknown", + outdated: false, + action: "panel", + reason: PANEL_UNKNOWN_REASON, + command: null, + }; + } + return { + name: "panel", + installed, + target: latest, + status: updateAvailable ? "outdated" : "current", + outdated: updateAvailable, + action: "panel", + reason: null, + command: null, + }; +}; + +const dockerRow = (readout: string | null): HostStackRow => { + const target = PINNED_VERSIONS.docker; + const installed = versionOf(readout); + if (installed === null) { + return { + ...unknownRow("docker", target, "manual"), + command: DOCKER_UPGRADE_COMMAND, + }; + } + const outdated = compareVersions(installed, target) < 0; + return { + name: "docker", + installed, + target, + status: outdated ? "outdated" : "current", + outdated, + action: "manual", + reason: DOCKER_REASON, + command: DOCKER_UPGRADE_COMMAND, + }; +}; + +const traefikRow = (image: string | null): HostStackRow => { + const target = TRAEFIK_VERSION; + const installed = imageVersion(image); + if (installed === null) { + return unknownRow("traefik", target, "ui"); + } + const outdated = compareVersions(installed, target) < 0; + return { + name: "traefik", + installed, + target, + status: outdated ? "outdated" : "current", + outdated, + action: "ui", + reason: null, + command: null, + }; +}; + +const postgresRow = (image: string | null): HostStackRow => { + const target = PINNED_VERSIONS.postgres; + const targetMajor = Number(target.split(".")[0]); + const installed = postgresVersion(image); + if (image === null || installed === null) { + return unknownRow("postgres", target, "manual"); + } + const installedMajor = Number(installed.split(".")[0]); + const majorChanged = installedMajor !== targetMajor; + const outdated = majorChanged + ? installedMajor < targetMajor + : compareVersions(installed, target) < 0; + return { + name: "postgres", + installed, + target, + status: outdated ? "outdated" : "current", + outdated, + action: "manual", + reason: + majorChanged && outdated + ? `Major version change (${installedMajor} to ${targetMajor}) needs a migration. The panel never changes the Postgres major version.` + : POSTGRES_REASON, + command: majorChanged + ? `POSTGRES_IMAGE=${withoutDigest(image)} ${INSTALLER_UPDATE}` + : null, + }; +}; + +const redisRow = (image: string | null): HostStackRow => { + const target = PINNED_VERSIONS.redis; + const installed = imageVersion(image); + if (installed === null) { + return unknownRow("redis", target, "ui"); + } + const outdated = compareVersions(installed, target) < 0; + return { + name: "redis", + installed, + target, + status: outdated ? "outdated" : "current", + outdated, + action: "ui", + reason: null, + command: null, + }; +}; + +export const buildHostStackRows = ( + readings: HostStackReadings, +): HostStackRow[] => [ + panelRow(readings.panel), + dockerRow(readings.docker), + traefikRow(readings.traefikImage), + postgresRow(readings.postgresImage), + redisRow(readings.redisImage), +]; + +export const hostStackRefusals = (components: HostStackComponent[]) => + components.flatMap((component) => + isHostStackUiComponent(component) ? [] : [HOST_STACK_REFUSALS[component]], + ); diff --git a/packages/server/src/setup/host-stack.ts b/packages/server/src/setup/host-stack.ts new file mode 100644 index 00000000000..0b40b4f46b1 --- /dev/null +++ b/packages/server/src/setup/host-stack.ts @@ -0,0 +1,186 @@ +import { getDokployVersionData, getUpdateData } from "../services/settings"; +import { execAsync } from "../utils/process/execAsync"; +import { spawnAsync } from "../utils/process/spawnAsync"; +import { PINNED_VERSIONS } from "./component-versions"; +import { + buildHostStackRows, + HOST_STACK_UI_COMPONENTS, + type HostStackRow, + type HostStackUiComponent, +} from "./host-stack-rows"; +import { + buildComponentUpdateSteps, + wrapComponentUpdateSteps, +} from "./server-components"; + +const REDIS_IMAGE = `redis:${PINNED_VERSIONS.redis}`; + +const UPDATE_CHECK_TIMEOUT_MS = 15_000; + +const readCommand = async (command: string) => { + try { + const { stdout } = await execAsync(command); + return stdout.trim() || null; + } catch { + return null; + } +}; + +const checkPanelUpdate = async (currentVersion: string) => { + let timer: ReturnType | undefined; + const timeout = new Promise((resolve) => { + timer = setTimeout(() => resolve(null), UPDATE_CHECK_TIMEOUT_MS); + }); + try { + return await Promise.race([ + getUpdateData(currentVersion).catch(() => null), + timeout, + ]); + } finally { + clearTimeout(timer); + } +}; + +export const readHostStackRows = async ( + currentVersion: string, +): Promise => { + const [docker, traefikImage, postgresImage, redisImage, update] = + await Promise.all([ + readCommand("docker version --format '{{.Server.Version}}'"), + readCommand("docker inspect -f '{{.Config.Image}}' dokploy-traefik"), + readCommand( + "docker service inspect dokploy-postgres --format '{{.Spec.TaskTemplate.ContainerSpec.Image}}'", + ), + readCommand( + "docker service inspect dokploy-redis --format '{{.Spec.TaskTemplate.ContainerSpec.Image}}'", + ), + checkPanelUpdate(currentVersion), + ]); + + return buildHostStackRows({ + docker, + traefikImage, + postgresImage, + redisImage, + panel: { + installed: getDokployVersionData(currentVersion).forkVersion, + latest: update?.latestVersion ?? null, + updateAvailable: update?.latestVersion ? update.updateAvailable : null, + }, + }); +}; + +const redisPullStep = () => ` +redis_image="${REDIS_IMAGE}" +redis_pull() { + local attempt=1 + while [ "$attempt" -le 3 ]; do + if $SUDO_CMD docker pull "$redis_image"; then + return 0 + fi + attempt=$((attempt + 1)) + sleep 5 + done + return 1 +} +echo "Downloading $redis_image before anything is changed" +if ! redis_pull; then + echo "Pre-download failed: $redis_image. Nothing was changed." >&2 + exit 1 +fi +`; + +const redisServiceStep = () => ` +echo "Updating Redis to ${PINNED_VERSIONS.redis}" +redis_timeout="\${DOKPLOY_REDIS_TIMEOUT:-240}" +redis_previous_image="$($SUDO_CMD docker service inspect dokploy-redis --format '{{.Spec.TaskTemplate.ContainerSpec.Image}}' 2>/dev/null || true)" +redis_previous_index="$($SUDO_CMD docker service inspect dokploy-redis --format '{{.Version.Index}}' 2>/dev/null || true)" +redis_update_needed=1 +if [ "\${redis_previous_image%%@*}" = "$redis_image" ]; then + redis_update_needed=0 + echo "Redis already runs $redis_image" +elif ! $SUDO_CMD docker service update --detach --update-order stop-first --update-failure-action rollback --image "$redis_image" dokploy-redis >/dev/null; then + echo "Error: docker service update was refused for dokploy-redis. Redis was not changed." >&2 + exit 1 +fi +redis_ready() { + local container="" + container="$($SUDO_CMD docker ps -q --no-trunc --filter label=com.docker.swarm.service.name=dokploy-redis --filter status=running --filter "ancestor=$redis_image" | head -n1)" + [ -n "$container" ] && [ "$($SUDO_CMD docker exec "$container" redis-cli ping 2>/dev/null)" = "PONG" ] +} +redis_settled() { + if [ "$redis_update_needed" = 1 ]; then + [ "\${redis_index:-0}" -gt "\${redis_previous_index:-0}" ] && [ "$redis_state" = "completed" ] || return 1 + fi + redis_ready +} +redis_deadline=$(( $(date +%s) + $redis_timeout )) +while :; do + redis_index="$($SUDO_CMD docker service inspect dokploy-redis --format '{{.Version.Index}}' 2>/dev/null || true)" + redis_state="$($SUDO_CMD docker service inspect dokploy-redis --format '{{.UpdateStatus.State}}' 2>/dev/null || true)" + if [ "$redis_update_needed" = 1 ] && [ "\${redis_index:-0}" -gt "\${redis_previous_index:-0}" ]; then + case "$redis_state" in + rollback_*) + echo "Error: dokploy-redis did not start on $redis_image (swarm state: $redis_state). Previous image: $redis_previous_image." >&2 + exit 1 + ;; + paused) + echo "Error: dokploy-redis is paused after its update to $redis_image. Previous image: $redis_previous_image." >&2 + exit 1 + ;; + esac + fi + if redis_settled; then + break + fi + if [ "$(date +%s)" -ge "$redis_deadline" ]; then + echo "Error: dokploy-redis was not ready on $redis_image within $redis_timeout seconds: the update did not finish or redis-cli ping did not return PONG. Previous image: $redis_previous_image. Check with: docker service ps dokploy-redis --no-trunc" >&2 + exit 1 + fi + sleep 2 +done +echo "Redis version ${PINNED_VERSIONS.redis} installed ✅" +`; + +const hostStackSteps = (components: HostStackUiComponent[]) => { + const selected = HOST_STACK_UI_COMPONENTS.filter((component) => + components.includes(component), + ); + const steps: string[] = []; + if (selected.includes("redis")) { + steps.push(redisPullStep()); + } + if (selected.includes("traefik")) { + steps.push(buildComponentUpdateSteps(["traefik"])); + } + if (selected.includes("redis")) { + steps.push(redisServiceStep()); + } + return steps.join("\n"); +}; + +export const buildHostStackUpdateScript = ( + components: HostStackUiComponent[], +) => wrapComponentUpdateSteps(hostStackSteps(components)); + +// Callers hold the lock from acquireHostStackUpdateLock for the whole run. +export const updateHostStackComponents = async ( + components: HostStackUiComponent[], + onData?: (data: string) => void, +) => { + try { + await spawnAsync( + "bash", + ["-c", buildHostStackUpdateScript(components)], + onData, + { stdio: ["ignore", "pipe", "pipe"] }, + ); + } catch (error) { + const code = (error as { code?: unknown }).code; + throw new Error( + typeof code === "number" + ? `Panel host update failed with exit code ${code}` + : `Panel host update could not start: ${error instanceof Error ? error.message : String(error)}`, + ); + } +}; diff --git a/packages/server/src/setup/server-components.ts b/packages/server/src/setup/server-components.ts index c4f01d65fee..9eb7cd9f047 100644 --- a/packages/server/src/setup/server-components.ts +++ b/packages/server/src/setup/server-components.ts @@ -6,7 +6,11 @@ import { } from "../utils/servers/destination"; import { PINNED_VERSIONS } from "./component-versions"; import { rcloneInstallCommand } from "./server-setup"; -import { buildTraefikCreateWithImage, TRAEFIK_VERSION } from "./traefik-setup"; +import { + buildTraefikCreateWithImage, + TRAEFIK_PORT, + TRAEFIK_VERSION, +} from "./traefik-setup"; export const COMPONENT_NAMES = [ "docker", @@ -458,8 +462,16 @@ if $SUDO_CMD docker inspect dokploy-traefik >/dev/null 2>&1; then traefik_old_image="$($SUDO_CMD docker inspect -f '{{.Config.Image}}' dokploy-traefik 2>/dev/null || true)" traefik_extra_networks="$($SUDO_CMD docker inspect -f '{{range $k,$v := .NetworkSettings.Networks}}{{$k}} {{end}}' dokploy-traefik 2>/dev/null || true)" fi +traefik_keep_bridge=1 +if [ "$traefik_existed" = 1 ] && ! echo " $traefik_extra_networks " | grep -q " bridge "; then + traefik_keep_bridge=0 +fi +traefik_http_publish="-p ${TRAEFIK_PORT}:${TRAEFIK_PORT}" +if [ "$traefik_existed" = 1 ] && ! $SUDO_CMD docker inspect -f '{{json .HostConfig.PortBindings}}' dokploy-traefik | grep -q '"${TRAEFIK_PORT}/tcp"'; then + traefik_http_publish="" +fi traefik_create() { -${buildTraefikCreateWithImage('"$1"')} +${buildTraefikCreateWithImage('"$1"', "$traefik_http_publish")} } traefik_connect() { $SUDO_CMD docker network connect dokploy-network dokploy-traefik || return 1 @@ -502,34 +514,41 @@ traefik_restore() { fi exit 1 } -trap 'traefik_restore "being interrupted"' HUP INT TERM -$SUDO_CMD docker rm -f dokploy-traefik-previous >/dev/null 2>&1 || true -if [ "$traefik_existed" = 1 ]; then - if ! $SUDO_CMD docker stop dokploy-traefik >/dev/null; then - traefik_restore "stopping the running Traefik container" +if [ "$traefik_existed" = 1 ] && [ "\${traefik_old_image%%@*}" = "traefik:v${TRAEFIK_VERSION}" ] && traefik_check_running; then + echo "Traefik already runs traefik:v${TRAEFIK_VERSION}" +else + trap 'traefik_restore "being interrupted"' HUP INT TERM + $SUDO_CMD docker rm -f dokploy-traefik-previous >/dev/null 2>&1 || true + if [ "$traefik_existed" = 1 ]; then + if ! $SUDO_CMD docker stop dokploy-traefik >/dev/null; then + traefik_restore "stopping the running Traefik container" + fi + traefik_old_stopped=1 + if ! $SUDO_CMD docker rm dokploy-traefik >/dev/null; then + traefik_restore "removing the stopped Traefik container" + fi fi - traefik_old_stopped=1 - if ! $SUDO_CMD docker rm dokploy-traefik >/dev/null; then - traefik_restore "removing the stopped Traefik container" + if ! traefik_create "traefik:v${TRAEFIK_VERSION}"; then + traefik_restore "creating the new Traefik container" fi + if ! traefik_connect "$traefik_extra_networks"; then + traefik_restore "connecting the new Traefik container to its networks" + fi + if ! $SUDO_CMD docker start dokploy-traefik >/dev/null; then + traefik_restore "starting the new Traefik container" + fi + if ! traefik_wait_settled; then + traefik_restore "checking the new Traefik container (running, with no restarts)" + fi + if [ "$traefik_keep_bridge" = 0 ]; then + $SUDO_CMD docker network disconnect bridge dokploy-traefik >/dev/null 2>&1 || true + fi + if ! $SUDO_CMD docker update --restart always dokploy-traefik >/dev/null; then + traefik_restore "setting the restart policy of the new Traefik container" + fi + trap - HUP INT TERM + echo "Traefik version ${TRAEFIK_VERSION} installed ✅" fi -if ! traefik_create "traefik:v${TRAEFIK_VERSION}"; then - traefik_restore "creating the new Traefik container" -fi -if ! traefik_connect "$traefik_extra_networks"; then - traefik_restore "connecting the new Traefik container to its networks" -fi -if ! $SUDO_CMD docker start dokploy-traefik >/dev/null; then - traefik_restore "starting the new Traefik container" -fi -if ! traefik_wait_settled; then - traefik_restore "checking the new Traefik container (running, with no restarts)" -fi -if ! $SUDO_CMD docker update --restart always dokploy-traefik >/dev/null; then - traefik_restore "setting the restart policy of the new Traefik container" -fi -trap - HUP INT TERM -echo "Traefik version ${TRAEFIK_VERSION} installed ✅" `; case "nixpacks": return ` @@ -579,16 +598,21 @@ echo "Buildpacks version ${PINNED_VERSIONS.buildpacks} installed ✅" } }; -export const buildComponentUpdateScript = ( - components: UpdatableComponent[], -) => ` -set -e -if [ "$(id -u)" -eq 0 ]; then SUDO_CMD=""; else SUDO_CMD="sudo"; fi +export const buildComponentUpdateSteps = (components: UpdatableComponent[]) => ` ${predownloadGate(components)} ${components.map(updateStepFor).join("\n")} +`; + +export const wrapComponentUpdateSteps = (steps: string) => ` +set -e +if [ "$(id -u)" -eq 0 ]; then SUDO_CMD=""; else SUDO_CMD="sudo"; fi +${steps} echo "${COMPONENTS_UPDATE_DONE}" `; +export const buildComponentUpdateScript = (components: UpdatableComponent[]) => + wrapComponentUpdateSteps(buildComponentUpdateSteps(components)); + const runSshCommand = async ( serverId: string, command: string, diff --git a/packages/server/src/setup/traefik-setup.ts b/packages/server/src/setup/traefik-setup.ts index 4bfe45c08a6..e2e6aa4af81 100644 --- a/packages/server/src/setup/traefik-setup.ts +++ b/packages/server/src/setup/traefik-setup.ts @@ -27,13 +27,14 @@ export const TRAEFIK_VERSION = const traefikContainerOptions = ( image: string, restart: string, + httpPublish = `-p ${TRAEFIK_PORT}:${TRAEFIK_PORT}`, ) => `--name dokploy-traefik \ --restart ${restart} \ -v /etc/dokploy/traefik/traefik.yml:/etc/traefik/traefik.yml \ -v /etc/dokploy/traefik/dynamic:/etc/dokploy/traefik/dynamic \ - -v /var/run/docker.sock:/var/run/docker.sock \ + -v /var/run/docker.sock:/var/run/docker.sock:ro \ -p ${TRAEFIK_SSL_PORT}:${TRAEFIK_SSL_PORT} \ - -p ${TRAEFIK_PORT}:${TRAEFIK_PORT} \ + ${httpPublish} \ -p ${TRAEFIK_HTTP3_PORT}:${TRAEFIK_HTTP3_PORT}/udp \ ${image}`; @@ -44,8 +45,11 @@ export const buildTraefikRunCommand = (version: string) => ` // Created with no restart policy and not started: networks are attached before the // ports are taken, and the policy is raised only after the new container runs. -export const buildTraefikCreateWithImage = (image: string) => ` - $SUDO_CMD docker create ${traefikContainerOptions(image, "no")} +export const buildTraefikCreateWithImage = ( + image: string, + httpPublish?: string, +) => ` + $SUDO_CMD docker create ${traefikContainerOptions(image, "no", httpPublish)} `; export const buildTraefikCreateCommand = (version: string) =>