From 0ed4bd6269688b75b9a278e0160089aa9b273969 Mon Sep 17 00:00:00 2001 From: merge-check <140916359+agentHits@users.noreply.github.com> Date: Wed, 7 Oct 2026 07:38:21 -0400 Subject: [PATCH 1/2] fix: pull the new image before restarting Dokploy on update The dokploy service updates stop-first, so the swarm task pulled the new 4.4 GB image only after the old container had stopped. On dev that left the panel down for 2 min 12 s (old container stopped 11:31:42, new one started 11:33:54) and Traefik answered Bad Gateway. Both update commands (AgentHits and official) now run `docker pull` first and stop if it fails, so the downtime is only the restart. The official update moves into getOfficialUpdateCommand to be testable. The update dialog used to reload after a fixed 8 s once /api/health answered. With the pull in front, the old server still answers then, so the dialog now waits until the server has gone down once and is healthy again (giving up after 20 minutes with an error). The image list and the cleanup also missed images that swarm pulls by digest: they have no tag, and `docker image ls` hides untagged images unless `-a` is passed. The running image after a UI update was one of them. Co-Authored-By: Claude Opus 5.5 --- .../settings/docker-server-boundary.test.ts | 33 +++++----- .../settings/dokploy-image-cleanup.test.ts | 47 ++++++++++++++ .../request-log-authorization.test.ts | 3 +- .../settings/web-server/update-webserver.tsx | 63 ++++++++++++------- apps/dokploy/server/api/routers/settings.ts | 15 ++--- packages/server/src/services/settings.ts | 23 ++++++- 6 files changed, 131 insertions(+), 53 deletions(-) diff --git a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts index b6c6089118f..a015493d9d1 100644 --- a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts +++ b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts @@ -24,6 +24,7 @@ const mocks = vi.hoisted(() => ({ generateOpenApiDocument: vi.fn(), getAccessibleServerIds: vi.fn(), getAgentHitsUpdateCommand: vi.fn(), + getOfficialUpdateCommand: vi.fn(), getDockerDiskUsage: vi.fn(), getDokployImageTag: vi.fn(), getDokployVersionData: vi.fn(), @@ -67,7 +68,6 @@ const redactWebServerSettings = (settings: T) => settings; vi.mock("@dokploy/server", () => ({ CLEANUP_CRON_JOB: "0 0 * * *", DEFAULT_UPDATE_DATA: {}, - DOKPLOY_KEEP_IMAGES_ENV: "DOKPLOY_KEEP_IMAGES", DOKPLOY_KEEP_IMAGES_MAX: 5, DOKPLOY_KEEP_IMAGES_MIN: 3, getDokployImageKeepCount: vi.fn(), @@ -90,6 +90,7 @@ vi.mock("@dokploy/server", () => ({ findServerById: mocks.findServerById, getAccessibleServerIds: mocks.getAccessibleServerIds, getAgentHitsUpdateCommand: mocks.getAgentHitsUpdateCommand, + getOfficialUpdateCommand: mocks.getOfficialUpdateCommand, getDockerDiskUsage: mocks.getDockerDiskUsage, getDokployImageTag: mocks.getDokployImageTag, getDokployVersionData: mocks.getDokployVersionData, @@ -470,15 +471,17 @@ describe("settings Docker server boundary", () => { updateSource: "official", }); + mocks.getOfficialUpdateCommand.mockReturnValue("official update command"); + await expect(createCaller().updateServer()).resolves.toBe(true); - expect(mocks.spawnAsync).toHaveBeenCalledWith("docker", [ - "service", - "update", - "--force", - "--image", - "dokploy/dokploy:v0.30.0", - "dokploy", + expect(mocks.getOfficialUpdateCommand).toHaveBeenCalledWith( + "v0.30.0", + undefined, + ); + expect(mocks.spawnAsync).toHaveBeenCalledWith("sh", [ + "-c", + "official update command", ]); expect(mocks.getAgentHitsUpdateCommand).not.toHaveBeenCalled(); }); @@ -494,16 +497,10 @@ describe("settings Docker server boundary", () => { createCaller().updateServer({ keepImages: null }), ).resolves.toBe(true); - expect(mocks.spawnAsync).toHaveBeenCalledWith("docker", [ - "service", - "update", - "--force", - "--image", - "dokploy/dokploy:v0.30.0", - "--env-add", - "DOKPLOY_KEEP_IMAGES=0", - "dokploy", - ]); + expect(mocks.getOfficialUpdateCommand).toHaveBeenCalledWith( + "v0.30.0", + null, + ); }); it("requires api.read before generating the OpenAPI document", async () => { diff --git a/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts b/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts index f7045a5123d..ff87e8d47d5 100644 --- a/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts +++ b/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts @@ -18,6 +18,7 @@ const { getDokployImageKeepCount, getDokployImageRepositories, getDokployImages, + getOfficialUpdateCommand, } = await import("@dokploy/server/services/settings"); describe("Dokploy image cleanup", () => { @@ -199,3 +200,49 @@ describe("Dokploy image list", () => { expect(execAsync).toHaveBeenCalledTimes(1); }); }); + +describe("Dokploy update commands", () => { + beforeEach(() => { + vi.stubEnv("DOKPLOY_AGENTHITS_UPDATE_IMAGE", undefined); + vi.stubEnv("DOKPLOY_AGENTHITS_UPDATE_TAG", undefined); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + const expectPullBeforeUpdate = (command: string, pull: string) => { + expect(command).toContain(`${pull} || exit 1`); + expect(command.indexOf(pull)).toBeLessThan( + command.indexOf("docker service update"), + ); + }; + + it("pulls the AgentHits image while the old container still runs", () => { + expectPullBeforeUpdate( + getAgentHitsUpdateCommand("v0.30.6"), + "docker pull ghcr.io/agenthits/dokploy\\:agenthits-dev", + ); + }); + + it("pulls the official image before updating the service", () => { + const command = getOfficialUpdateCommand("v0.30.7", 3); + + expectPullBeforeUpdate(command, "docker pull dokploy/dokploy\\:v0.30.7"); + expect(command).toContain( + "--image dokploy/dokploy\\:v0.30.7 --env-add DOKPLOY_KEEP_IMAGES\\=3 dokploy", + ); + expect(getOfficialUpdateCommand("v0.30.7")).not.toContain( + "DOKPLOY_KEEP_IMAGES", + ); + }); + + it("includes untagged images that swarm pulled by digest", async () => { + execAsync.mockResolvedValue({ stdout: "", stderr: "" }); + + await getDokployImages(); + + expect(execAsync.mock.calls[0]?.[0]).toMatch(/^docker image ls -a -q /); + expect(getDokployImageCleanupCommand(3)).toContain("docker image ls -a -q"); + }); +}); diff --git a/apps/dokploy/__test__/settings/request-log-authorization.test.ts b/apps/dokploy/__test__/settings/request-log-authorization.test.ts index f9066263f0a..85d505e754c 100644 --- a/apps/dokploy/__test__/settings/request-log-authorization.test.ts +++ b/apps/dokploy/__test__/settings/request-log-authorization.test.ts @@ -50,6 +50,7 @@ const mocks = vi.hoisted(() => ({ generateOpenApiDocument: vi.fn(), getAccessibleServerIds: vi.fn(), getAgentHitsUpdateCommand: vi.fn(), + getOfficialUpdateCommand: vi.fn(), getDockerDiskUsage: vi.fn(), getDokployImageTag: vi.fn(), getDokployVersionData: vi.fn(), @@ -90,7 +91,6 @@ const mocks = vi.hoisted(() => ({ vi.mock("@dokploy/server", () => ({ CLEANUP_CRON_JOB: "0 0 * * *", DEFAULT_UPDATE_DATA: {}, - DOKPLOY_KEEP_IMAGES_ENV: "DOKPLOY_KEEP_IMAGES", DOKPLOY_KEEP_IMAGES_MAX: 5, DOKPLOY_KEEP_IMAGES_MIN: 3, getDokployImageKeepCount: vi.fn(), @@ -112,6 +112,7 @@ vi.mock("@dokploy/server", () => ({ findServerById: mocks.findServerById, getAccessibleServerIds: mocks.getAccessibleServerIds, getAgentHitsUpdateCommand: mocks.getAgentHitsUpdateCommand, + getOfficialUpdateCommand: mocks.getOfficialUpdateCommand, getDockerDiskUsage: mocks.getDockerDiskUsage, getDokployImageTag: mocks.getDokployImageTag, getDokployVersionData: mocks.getDokployVersionData, diff --git a/apps/dokploy/components/dashboard/settings/web-server/update-webserver.tsx b/apps/dokploy/components/dashboard/settings/web-server/update-webserver.tsx index 8fbc3a1e8e3..980bbde976b 100644 --- a/apps/dokploy/components/dashboard/settings/web-server/update-webserver.tsx +++ b/apps/dokploy/components/dashboard/settings/web-server/update-webserver.tsx @@ -33,6 +33,11 @@ type HealthResult = { traefik: ServiceStatus; }; +const HEALTH_POLL_MS = 2000; +const HEALTH_TIMEOUT_MS = 5000; +// Pulling a 4+ GB image on a slow link can take several minutes. +const RESTART_WAIT_MS = 20 * 60 * 1000; + type ModalState = "idle" | "checking" | "results" | "updating"; const ServiceStatusItem = ({ @@ -92,23 +97,33 @@ export const UpdateWebServer = ({ healthResult.postgres.status === "healthy" && healthResult.traefik.status === "healthy"; - const checkIsUpdateFinished = async () => { + const isServerUp = async () => { try { - const response = await fetch("/api/health"); - if (!response.ok) { - throw new Error("Health check failed"); - } + const response = await fetch("/api/health", { + cache: "no-store", + signal: AbortSignal.timeout(HEALTH_TIMEOUT_MS), + }); + return response.ok; + } catch { + return false; + } + }; - toast.success( - "The server has been updated. The page will be reloaded to reflect the changes...", - ); + const sleep = (ms: number) => + new Promise((resolve) => setTimeout(resolve, ms)); - setTimeout(() => { - window.location.reload(); - }, 2000); - } catch { - await new Promise((resolve) => setTimeout(resolve, 2000)); - void checkIsUpdateFinished(); + // The old server keeps answering while the new image is pulled, so wait + // until it goes down once before treating a healthy answer as the new one. + const waitForRestart = async () => { + const startedAt = Date.now(); + while (await isServerUp()) { + if (Date.now() - startedAt > RESTART_WAIT_MS) { + throw new Error("The server did not restart"); + } + await sleep(HEALTH_POLL_MS); + } + while (!(await isServerUp())) { + await sleep(HEALTH_POLL_MS); } }; @@ -116,15 +131,19 @@ export const UpdateWebServer = ({ try { setModalState("updating"); await updateServer(keepImages === undefined ? undefined : { keepImages }); + await waitForRestart(); - await new Promise((resolve) => setTimeout(resolve, 8000)); - - await checkIsUpdateFinished(); + toast.success( + "The server has been updated. The page will be reloaded to reflect the changes...", + ); + setTimeout(() => { + window.location.reload(); + }, 2000); } catch (error) { setModalState("results"); console.error("Error updating server:", error); toast.error( - "An error occurred while updating the server, please try again.", + "The server did not restart with the new version. Check the Dokploy service logs and try again.", ); } }; @@ -168,8 +187,9 @@ export const UpdateWebServer = ({ This will update the web server to the selected latest build. AgentHits fork installs update from the AgentHits GHCR image. - You will not be able to use the panel during the update - process. The page will be reloaded once the update is + The new image is downloaded first, so the panel is + unavailable only for about 20 seconds while Dokploy + restarts. The page will be reloaded once the update is finished.

@@ -229,7 +249,8 @@ export const UpdateWebServer = ({ {modalState === "updating" && ( - The server is being updated, please wait... + Downloading the new image. The panel keeps working meanwhile; + it is unavailable for about 20 seconds while Dokploy restarts. )} diff --git a/apps/dokploy/server/api/routers/settings.ts b/apps/dokploy/server/api/routers/settings.ts index b7f957e5768..e5af5dd28fe 100644 --- a/apps/dokploy/server/api/routers/settings.ts +++ b/apps/dokploy/server/api/routers/settings.ts @@ -12,7 +12,6 @@ import { cleanupSystem, cleanupVolumes, DEFAULT_UPDATE_DATA, - DOKPLOY_KEEP_IMAGES_ENV, DOKPLOY_KEEP_IMAGES_MAX, DOKPLOY_KEEP_IMAGES_MIN, filterProtectedTraefikEntries, @@ -25,6 +24,7 @@ import { getDokployImageTag, getDokployVersionData, getLogCleanupStatus, + getOfficialUpdateCommand, getUpdateData, getWebServerSettings, IS_CLOUD, @@ -669,16 +669,9 @@ export const settingsRouter = createTRPCRouter({ ), ]); } else { - void spawnAsync("docker", [ - "service", - "update", - "--force", - "--image", - `dokploy/dokploy:${data.latestVersion}`, - ...(keepImages === undefined - ? [] - : ["--env-add", `${DOKPLOY_KEEP_IMAGES_ENV}=${keepImages ?? 0}`]), - "dokploy", + void spawnAsync("sh", [ + "-c", + getOfficialUpdateCommand(data.latestVersion ?? "", keepImages), ]); } await audit(ctx, { diff --git a/packages/server/src/services/settings.ts b/packages/server/src/services/settings.ts index d90df5e1870..20541750d4b 100644 --- a/packages/server/src/services/settings.ts +++ b/packages/server/src/services/settings.ts @@ -301,6 +301,8 @@ export const getDokployImageRepositories = () => [ * Removes Dokploy web server images except the `keep` newest ones. Swarm keeps * exited task containers of the dokploy service, and they pin old images, so * those are removed first. Images used by running containers are never forced. + * `-a` is needed because images swarm pulls by digest have no tag, and + * `docker image ls` hides untagged images without it. */ export const getDokployImageCleanupCommand = (keep: number) => { const filters = getDokployImageRepositories() @@ -308,7 +310,7 @@ export const getDokployImageCleanupCommand = (keep: number) => { .join(" "); return ` -ids=$(docker image ls -q --no-trunc ${filters} | sort -u) +ids=$(docker image ls -a -q --no-trunc ${filters} | sort -u) if [ -z "$ids" ]; then exit 0 fi @@ -404,7 +406,7 @@ export const getDokployImages = async (): Promise => { .map((repository) => `--filter ${quoteShellArg(`reference=${repository}`)}`) .join(" "); const { stdout: idsOutput } = await execAsync( - `docker image ls -q --no-trunc ${filters} | sort -u`, + `docker image ls -a -q --no-trunc ${filters} | sort -u`, ); const ids = idsOutput.split("\n").filter(Boolean); if (ids.length === 0) { @@ -490,7 +492,10 @@ export const getAgentHitsUpdateCommand = ( const keepImagesArg = keepImages === undefined ? "" : getDokployKeepImagesEnvArg(keepImages); + // Pull while the old container still serves the panel: the service is + // stop-first, so a pull inside the update would happen with Dokploy down. return ` +docker pull ${quoteShellArg(getAgentHitsUpdateImage())} || exit 1 fork_version_env_arg="" if docker service inspect dokploy --format '{{range .Spec.TaskTemplate.ContainerSpec.Env}}{{println .}}{{end}}' 2>/dev/null | grep -q '^DOKPLOY_FORK_VERSION='; then fork_version_env_arg="--env-rm DOKPLOY_FORK_VERSION" @@ -505,6 +510,20 @@ docker service update --force \\ `; }; +export const getOfficialUpdateCommand = ( + version: string, + keepImages?: number | null, +) => { + const image = quoteShellArg(`dokploy/dokploy:${version}`); + const keepImagesArg = + keepImages === undefined ? "" : getDokployKeepImagesEnvArg(keepImages); + + return ` +docker pull ${image} || exit 1 +docker service update --force --image ${image} ${keepImagesArg} dokploy +`; +}; + /** Returns Dokploy docker service image digest */ export const getServiceImageDigest = async () => { const { stdout } = await execAsync( From 41abd85516b4500251252e12afb54265eeb2eeb7 Mon Sep 17 00:00:00 2001 From: merge-check <140916359+agentHits@users.noreply.github.com> Date: Wed, 7 Oct 2026 07:47:13 -0400 Subject: [PATCH 2/2] feat: show a status panel while the web server updates After "Update Server" -> "Confirm" the dialog only showed a spinner and one sentence, and during the restart the admin saw nothing useful. The server now tracks the running update job in memory: the update scripts print a marker once `docker pull` succeeded, and the new startServerUpdate runs the script, follows its output and records the phase (pulling, updating, done, failed), timestamps, docker pull layer counts, the last few output lines and a short error. The admin-only settings.getServerUpdateStatus query (classified as "read") exposes it. The state is lost when Dokploy restarts, which is expected: a fresh "idle" status is also how the UI recognises the new process. A second update request while one is running does not start another script. The dialog now renders UpdateStatusPanel, a props-driven view of four steps (downloading the image, restarting Dokploy, starting the new version, done) with done/active/pending/failed states, elapsed times and a live downtime counter. The phase logic lives in update-progress.ts as pure functions: the dialog polls the status query while the old server answers, switches to /api/health while Dokploy is down, and fails when the pull or the service update fails, when the server does not go down within 20 minutes or does not come back within 10 minutes after it stopped. A failed update keeps the dialog closable. Co-Authored-By: Claude Opus 5.5 --- .../settings/docker-server-boundary.test.ts | 47 +++- .../settings/dokploy-image-cleanup.test.ts | 13 + .../request-log-authorization.test.ts | 4 + .../__test__/settings/update-progress.test.ts | 249 ++++++++++++++++++ .../settings/web-server-update-status.test.ts | 209 +++++++++++++++ .../settings/web-server/update-progress.ts | 205 ++++++++++++++ .../web-server/update-status-panel.tsx | 217 +++++++++++++++ .../settings/web-server/update-webserver.tsx | 143 +++++++--- apps/dokploy/server/api/routers/settings.ts | 33 ++- .../server/api/utils/procedure-access.ts | 1 + packages/server/src/index.ts | 1 + packages/server/src/services/settings.ts | 3 + .../server/src/services/web-server-update.ts | 158 +++++++++++ 13 files changed, 1226 insertions(+), 57 deletions(-) create mode 100644 apps/dokploy/__test__/settings/update-progress.test.ts create mode 100644 apps/dokploy/__test__/settings/web-server-update-status.test.ts create mode 100644 apps/dokploy/components/dashboard/settings/web-server/update-progress.ts create mode 100644 apps/dokploy/components/dashboard/settings/web-server/update-status-panel.tsx create mode 100644 packages/server/src/services/web-server-update.ts diff --git a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts index a015493d9d1..6bc2ddcdb61 100644 --- a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts +++ b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts @@ -51,6 +51,8 @@ const mocks = vi.hoisted(() => ({ sendDockerCleanupNotifications: vi.fn(), setupGPUSupport: vi.fn(), spawnAsync: vi.fn(), + startServerUpdate: vi.fn(), + getServerUpdateStatus: vi.fn(), startLogCleanup: vi.fn(), stopLogCleanup: vi.fn(), updateLetsEncryptEmail: vi.fn(), @@ -114,6 +116,8 @@ vi.mock("@dokploy/server", () => ({ sendDockerCleanupNotifications: mocks.sendDockerCleanupNotifications, setupGPUSupport: mocks.setupGPUSupport, spawnAsync: mocks.spawnAsync, + startServerUpdate: mocks.startServerUpdate, + getServerUpdateStatus: mocks.getServerUpdateStatus, startLogCleanup: mocks.startLogCleanup, stopLogCleanup: mocks.stopLogCleanup, updateLetsEncryptEmail: mocks.updateLetsEncryptEmail, @@ -427,10 +431,9 @@ describe("settings Docker server boundary", () => { "v0.30.0", undefined, ); - expect(mocks.spawnAsync).toHaveBeenCalledWith("sh", [ - "-c", + expect(mocks.startServerUpdate).toHaveBeenCalledWith( "agenthits update command", - ]); + ); }); it("passes the image keep count to the AgentHits update command", async () => { @@ -461,7 +464,7 @@ describe("settings Docker server boundary", () => { await expect( createCaller().updateServer({ keepImages: 10 }), ).rejects.toThrow(); - expect(mocks.spawnAsync).not.toHaveBeenCalled(); + expect(mocks.startServerUpdate).not.toHaveBeenCalled(); }); it("keeps official updates on the official Dokploy image path", async () => { @@ -479,10 +482,9 @@ describe("settings Docker server boundary", () => { "v0.30.0", undefined, ); - expect(mocks.spawnAsync).toHaveBeenCalledWith("sh", [ - "-c", + expect(mocks.startServerUpdate).toHaveBeenCalledWith( "official update command", - ]); + ); expect(mocks.getAgentHitsUpdateCommand).not.toHaveBeenCalled(); }); @@ -503,6 +505,37 @@ describe("settings Docker server boundary", () => { ); }); + it("does not start an update when none is available", async () => { + mocks.getUpdateData.mockResolvedValue({ + latestVersion: null, + updateAvailable: false, + updateSource: "official", + }); + + await expect(createCaller().updateServer()).resolves.toBe(true); + + expect(mocks.startServerUpdate).not.toHaveBeenCalled(); + }); + + it("returns the web server update status", async () => { + const status = { + phase: "pulling", + startedAt: 1, + pulledAt: null, + finishedAt: null, + error: null, + layersTotal: 3, + layersDownloaded: 1, + layersExtracted: 0, + output: [], + }; + mocks.getServerUpdateStatus.mockReturnValue(status); + + await expect(createCaller().getServerUpdateStatus()).resolves.toEqual( + status, + ); + }); + it("requires api.read before generating the OpenAPI document", async () => { const result = await createCaller().getOpenApiDocument(); diff --git a/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts b/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts index ff87e8d47d5..e11518b6ce5 100644 --- a/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts +++ b/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts @@ -1,3 +1,4 @@ +import { execFileSync } from "node:child_process"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const { execAsync } = vi.hoisted(() => ({ execAsync: vi.fn() })); @@ -20,6 +21,9 @@ const { getDokployImages, getOfficialUpdateCommand, } = await import("@dokploy/server/services/settings"); +const { UPDATE_IMAGE_PULLED_MARKER } = await import( + "@dokploy/server/services/web-server-update" +); describe("Dokploy image cleanup", () => { beforeEach(() => { @@ -216,6 +220,15 @@ describe("Dokploy update commands", () => { expect(command.indexOf(pull)).toBeLessThan( command.indexOf("docker service update"), ); + + const lines = command.split("\n"); + const markerLine = lines.findIndex((line) => line.startsWith("echo ")); + expect(lines[markerLine - 1]).toBe(`${pull} || exit 1`); + expect( + execFileSync("sh", ["-c", lines[markerLine] as string], { + encoding: "utf8", + }).trim(), + ).toBe(UPDATE_IMAGE_PULLED_MARKER); }; it("pulls the AgentHits image while the old container still runs", () => { diff --git a/apps/dokploy/__test__/settings/request-log-authorization.test.ts b/apps/dokploy/__test__/settings/request-log-authorization.test.ts index 85d505e754c..5cf7578a881 100644 --- a/apps/dokploy/__test__/settings/request-log-authorization.test.ts +++ b/apps/dokploy/__test__/settings/request-log-authorization.test.ts @@ -76,6 +76,8 @@ const mocks = vi.hoisted(() => ({ sendDockerCleanupNotifications: vi.fn(), setupGPUSupport: vi.fn(), spawnAsync: vi.fn(), + startServerUpdate: vi.fn(), + getServerUpdateStatus: vi.fn(), startLogCleanup: vi.fn(), stopLogCleanup: vi.fn(), updateLetsEncryptEmail: vi.fn(), @@ -136,6 +138,8 @@ vi.mock("@dokploy/server", () => ({ sendDockerCleanupNotifications: mocks.sendDockerCleanupNotifications, setupGPUSupport: mocks.setupGPUSupport, spawnAsync: mocks.spawnAsync, + startServerUpdate: mocks.startServerUpdate, + getServerUpdateStatus: mocks.getServerUpdateStatus, startLogCleanup: mocks.startLogCleanup, stopLogCleanup: mocks.stopLogCleanup, updateLetsEncryptEmail: mocks.updateLetsEncryptEmail, diff --git a/apps/dokploy/__test__/settings/update-progress.test.ts b/apps/dokploy/__test__/settings/update-progress.test.ts new file mode 100644 index 00000000000..35786b0e818 --- /dev/null +++ b/apps/dokploy/__test__/settings/update-progress.test.ts @@ -0,0 +1,249 @@ +import type { ServerUpdateStatus } from "@dokploy/server/services/web-server-update"; +import { describe, expect, it } from "vitest"; +import { + API_WAIT_MS, + advanceUpdateProgress, + createUpdateProgress, + DOWNLOAD_LIMIT_MS, + failUpdateProgress, + formatElapsed, + getUpdateStepState, + RESTART_LIMIT_MS, + UPDATE_STEPS, + type UpdateProgress, + type UpdateProgressEvent, +} from "@/components/dashboard/settings/web-server/update-progress"; + +const serverStatus = ( + overrides: Partial = {}, +): ServerUpdateStatus => ({ + phase: "pulling", + startedAt: 1_000, + pulledAt: null, + finishedAt: null, + error: null, + layersTotal: 0, + layersDownloaded: 0, + layersExtracted: 0, + output: [], + ...overrides, +}); + +const run = (events: UpdateProgressEvent[], start = createUpdateProgress(0)) => + events.reduce(advanceUpdateProgress, start); + +const stepStates = (progress: UpdateProgress) => + UPDATE_STEPS.map((step) => getUpdateStepState(progress, step)); + +describe("update progress", () => { + it("walks through download, restart and start to done", () => { + const pulling = run([ + { type: "status", status: serverStatus(), at: 2_000 }, + ]); + expect(pulling.phase).toBe("downloading"); + expect(stepStates(pulling)).toEqual([ + "active", + "pending", + "pending", + "pending", + ]); + + const down = run( + [ + { + type: "status", + status: serverStatus({ phase: "updating", pulledAt: 50_000 }), + at: 60_000, + }, + { type: "down", at: 62_000 }, + ], + pulling, + ); + expect(down).toMatchObject({ phase: "restarting", downAt: 62_000 }); + expect(stepStates(down)).toEqual(["done", "active", "pending", "pending"]); + + const up = advanceUpdateProgress(down, { type: "up", at: 80_000 }); + expect(up).toMatchObject({ phase: "starting", upAt: 80_000 }); + + const done = advanceUpdateProgress(up, { + type: "status", + status: serverStatus({ phase: "idle", startedAt: null }), + at: 82_000, + }); + expect(done).toMatchObject({ phase: "done", finishedAt: 82_000 }); + expect(stepStates(done)).toEqual(["done", "done", "done", "done"]); + expect(done.server?.phase).toBe("updating"); + }); + + it("keeps the first down time when the new server flaps", () => { + const progress = run([ + { type: "down", at: 10_000 }, + { type: "up", at: 20_000 }, + { type: "down", at: 22_000 }, + ]); + + expect(progress).toMatchObject({ + phase: "restarting", + downAt: 10_000, + upAt: null, + }); + }); + + it("ignores health answers while the old server is still pulling", () => { + const progress = run([{ type: "up", at: 2_000 }]); + + expect(progress.phase).toBe("downloading"); + }); + + it("fails on the download step when the pull failed", () => { + const progress = run([ + { + type: "status", + status: serverStatus({ + phase: "failed", + error: "Downloading the new image failed.", + output: ["manifest unknown"], + }), + at: 5_000, + }, + ]); + + expect(progress).toMatchObject({ + phase: "failed", + failedStep: "downloading", + error: "Downloading the new image failed.", + }); + expect(progress.server?.output).toEqual(["manifest unknown"]); + expect(stepStates(progress)).toEqual([ + "failed", + "pending", + "pending", + "pending", + ]); + }); + + it("fails on the restart step when the service update failed", () => { + const progress = run([ + { + type: "status", + status: serverStatus({ + phase: "failed", + pulledAt: 3_000, + error: "docker service update failed.", + }), + at: 5_000, + }, + ]); + + expect(progress.failedStep).toBe("restarting"); + expect(stepStates(progress)).toEqual([ + "done", + "failed", + "pending", + "pending", + ]); + }); + + it("fails when the server never started an update job", () => { + const progress = run([ + { + type: "status", + status: serverStatus({ phase: "idle", startedAt: null }), + at: 2_000, + }, + ]); + + expect(progress).toMatchObject({ + phase: "failed", + failedStep: "downloading", + }); + expect(progress.error).toContain("did not start an update"); + }); + + it("treats an idle status after a seen job as a restart that was missed", () => { + const progress = run([ + { type: "status", status: serverStatus(), at: 2_000 }, + { + type: "status", + status: serverStatus({ phase: "idle", startedAt: null }), + at: 400_000, + }, + ]); + + expect(progress.phase).toBe("done"); + }); + + it("fails when the server does not go down within the download limit", () => { + const waiting = run([ + { type: "status", status: serverStatus(), at: 2_000 }, + { type: "tick", at: DOWNLOAD_LIMIT_MS }, + ]); + expect(waiting.phase).toBe("downloading"); + + const progress = advanceUpdateProgress(waiting, { + type: "tick", + at: DOWNLOAD_LIMIT_MS + 1, + }); + expect(progress).toMatchObject({ + phase: "failed", + failedStep: "downloading", + error: "Dokploy did not restart within 20 minutes.", + }); + }); + + it("fails when the server does not come back after going down", () => { + const progress = run([ + { type: "down", at: 10_000 }, + { type: "tick", at: 10_000 + RESTART_LIMIT_MS + 1 }, + ]); + + expect(progress).toMatchObject({ + phase: "failed", + failedStep: "restarting", + }); + expect(stepStates(progress)).toEqual([ + "done", + "failed", + "pending", + "pending", + ]); + }); + + it("finishes when health is fine but the API keeps failing", () => { + const starting = run([ + { type: "down", at: 10_000 }, + { type: "up", at: 30_000 }, + { type: "tick", at: 30_000 + API_WAIT_MS }, + ]); + expect(starting.phase).toBe("starting"); + + expect( + advanceUpdateProgress(starting, { + type: "tick", + at: 30_000 + API_WAIT_MS + 1, + }).phase, + ).toBe("done"); + }); + + it("does not change once finished", () => { + const failed = failUpdateProgress( + createUpdateProgress(0), + "downloading", + "Could not start the update.", + 1_000, + ); + + expect(advanceUpdateProgress(failed, { type: "down", at: 2_000 })).toBe( + failed, + ); + }); +}); + +describe("formatElapsed", () => { + it("formats seconds and minutes", () => { + expect(formatElapsed(-5)).toBe("0 s"); + expect(formatElapsed(19_900)).toBe("19 s"); + expect(formatElapsed(65_000)).toBe("1 min 05 s"); + expect(formatElapsed(20 * 60 * 1000)).toBe("20 min 00 s"); + }); +}); diff --git a/apps/dokploy/__test__/settings/web-server-update-status.test.ts b/apps/dokploy/__test__/settings/web-server-update-status.test.ts new file mode 100644 index 00000000000..43fb810201e --- /dev/null +++ b/apps/dokploy/__test__/settings/web-server-update-status.test.ts @@ -0,0 +1,209 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { spawnAsync } = vi.hoisted(() => ({ spawnAsync: vi.fn() })); + +vi.mock("@dokploy/server/utils/process/spawnAsync", () => ({ spawnAsync })); + +const { + UPDATE_IMAGE_PULLED_MARKER, + getServerUpdateStatus, + resetServerUpdateStatus, + startServerUpdate, +} = await import("@dokploy/server/services/web-server-update"); + +const startFakeScript = () => { + let onData: (data: string) => void = () => {}; + let resolve: () => void = () => {}; + let reject: (error: unknown) => void = () => {}; + spawnAsync.mockImplementation( + (_command: string, _args: string[], callback: (data: string) => void) => { + onData = callback; + return new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + }, + ); + expect(startServerUpdate("update script")).toBe(true); + return { + write: (data: string) => onData(data), + exit: async (code = 0) => { + if (code === 0) { + resolve(); + } else { + reject(Object.assign(new Error("failed"), { code })); + } + await new Promise((r) => setTimeout(r, 0)); + }, + failToSpawn: async () => { + reject(Object.assign(new Error("spawn sh ENOENT"), { code: "ENOENT" })); + await new Promise((r) => setTimeout(r, 0)); + }, + }; +}; + +describe("web server update status", () => { + beforeEach(() => { + vi.clearAllMocks(); + resetServerUpdateStatus(); + }); + + it("is idle before any update", () => { + expect(getServerUpdateStatus()).toMatchObject({ + phase: "idle", + startedAt: null, + error: null, + output: [], + }); + }); + + it("runs the script through sh and starts in the pulling phase", () => { + startFakeScript(); + + expect(spawnAsync).toHaveBeenCalledWith( + "sh", + ["-c", "update script"], + expect.any(Function), + ); + const status = getServerUpdateStatus(); + expect(status.phase).toBe("pulling"); + expect(status.startedAt).toEqual(expect.any(Number)); + }); + + it("counts docker pull layers, even across split chunks", () => { + const script = startFakeScript(); + + script.write("v1: Pulling from dokploy/dokploy\n"); + script.write( + "aaaaaaaaaaaa: Already exists\nbbbbbbbbbbbb: Pulling fs layer\n", + ); + script.write("cccccccccccc: Pulling fs layer\nbbbbbbbbbbbb: Downlo"); + script.write("ad complete\n"); + + expect(getServerUpdateStatus()).toMatchObject({ + layersTotal: 3, + layersDownloaded: 2, + layersExtracted: 1, + }); + + script.write("bbbbbbbbbbbb: Pull complete\ncccccccccccc: Pull complete\n"); + expect(getServerUpdateStatus()).toMatchObject({ + layersTotal: 3, + layersDownloaded: 3, + layersExtracted: 3, + }); + }); + + it("does not move a finished layer back to downloading", () => { + const script = startFakeScript(); + + script.write("aaaaaaaaaaaa: Pull complete\naaaaaaaaaaaa: Downloading\n"); + + expect(getServerUpdateStatus()).toMatchObject({ + layersTotal: 1, + layersExtracted: 1, + }); + }); + + it("keeps only the last few output lines", () => { + const script = startFakeScript(); + + script.write( + "line 1\r\nline 2\nline 3\n\nline 4\nline 5\nline 6\nline 7\n", + ); + + expect(getServerUpdateStatus().output).toEqual([ + "line 3", + "line 4", + "line 5", + "line 6", + "line 7", + ]); + }); + + it("moves to updating once the pulled marker is printed", () => { + const script = startFakeScript(); + + script.write( + `Status: Downloaded newer image\n${UPDATE_IMAGE_PULLED_MARKER}\n`, + ); + + const status = getServerUpdateStatus(); + expect(status.phase).toBe("updating"); + expect(status.pulledAt).toEqual(expect.any(Number)); + expect(status.output).not.toContain(UPDATE_IMAGE_PULLED_MARKER); + }); + + it("reports a failed pull", async () => { + const script = startFakeScript(); + + script.write("Error response from daemon: manifest unknown"); + await script.exit(1); + + expect(getServerUpdateStatus()).toMatchObject({ + phase: "failed", + error: "Downloading the new image failed.", + output: ["Error response from daemon: manifest unknown"], + finishedAt: expect.any(Number), + }); + }); + + it("reports a failed service update after the pull", async () => { + const script = startFakeScript(); + + script.write(`${UPDATE_IMAGE_PULLED_MARKER}\n`); + await script.exit(1); + + expect(getServerUpdateStatus()).toMatchObject({ + phase: "failed", + error: "docker service update failed.", + }); + }); + + it("reports a script that could not be started", async () => { + const script = startFakeScript(); + + await script.failToSpawn(); + + expect(getServerUpdateStatus()).toMatchObject({ + phase: "failed", + error: "Could not start the update script.", + }); + }); + + it("is done when the script exits cleanly", async () => { + const script = startFakeScript(); + + script.write(`${UPDATE_IMAGE_PULLED_MARKER}\n`); + await script.exit(0); + + expect(getServerUpdateStatus()).toMatchObject({ + phase: "done", + error: null, + }); + }); + + it("does not start a second update while one is running", async () => { + const script = startFakeScript(); + + expect(startServerUpdate("another script")).toBe(false); + expect(spawnAsync).toHaveBeenCalledTimes(1); + + await script.exit(1); + expect(startServerUpdate("retry script")).toBe(true); + expect(getServerUpdateStatus()).toMatchObject({ + phase: "pulling", + error: null, + output: [], + }); + }); + + it("returns a copy of the status", () => { + const script = startFakeScript(); + script.write("line\n"); + + getServerUpdateStatus().output.push("changed"); + + expect(getServerUpdateStatus().output).toEqual(["line"]); + }); +}); diff --git a/apps/dokploy/components/dashboard/settings/web-server/update-progress.ts b/apps/dokploy/components/dashboard/settings/web-server/update-progress.ts new file mode 100644 index 00000000000..24738c341f3 --- /dev/null +++ b/apps/dokploy/components/dashboard/settings/web-server/update-progress.ts @@ -0,0 +1,205 @@ +import type { ServerUpdateStatus } from "@dokploy/server/services/web-server-update"; + +export const UPDATE_POLL_MS = 2000; +// Pulling a 4+ GB image on a slow link can take several minutes. +export const DOWNLOAD_LIMIT_MS = 20 * 60 * 1000; +export const RESTART_LIMIT_MS = 10 * 60 * 1000; +// /api/health does not touch the database or the session, so a healthy +// server can still fail API calls for a while (migrations, cold caches). Past +// this, reloading shows the real state better than waiting longer. +export const API_WAIT_MS = 60 * 1000; + +export type UpdateStep = "downloading" | "restarting" | "starting"; +export type UpdateProgressPhase = UpdateStep | "done" | "failed"; +export type UpdateStepState = "done" | "active" | "pending" | "failed"; + +export interface UpdateProgress { + phase: UpdateProgressPhase; + failedStep: UpdateStep | null; + error: string | null; + startedAt: number; + /** When the old server stopped answering. */ + downAt: number | null; + /** When the new server answered /api/health. */ + upAt: number | null; + finishedAt: number | null; + /** Last update job status reported by the old server. */ + server: ServerUpdateStatus | null; +} + +export type UpdateProgressEvent = + | { type: "status"; status: ServerUpdateStatus; at: number } + | { type: "down"; at: number } + | { type: "up"; at: number } + | { type: "tick"; at: number }; + +export const createUpdateProgress = (at: number): UpdateProgress => ({ + phase: "downloading", + failedStep: null, + error: null, + startedAt: at, + downAt: null, + upAt: null, + finishedAt: null, + server: null, +}); + +export const isUpdateFinished = (progress: UpdateProgress) => + progress.phase === "done" || progress.phase === "failed"; + +export const failUpdateProgress = ( + progress: UpdateProgress, + step: UpdateStep, + error: string, + at: number, +): UpdateProgress => ({ + ...progress, + phase: "failed", + failedStep: step, + error, + finishedAt: at, +}); + +const finishUpdateProgress = ( + progress: UpdateProgress, + at: number, +): UpdateProgress => ({ ...progress, phase: "done", finishedAt: at }); + +const handleStatus = ( + progress: UpdateProgress, + status: ServerUpdateStatus, + at: number, +): UpdateProgress => { + if (progress.phase !== "downloading") { + // The API answers again, so the new version is up. + return finishUpdateProgress({ ...progress, upAt: progress.upAt ?? at }, at); + } + + if (status.phase === "failed") { + return failUpdateProgress( + { ...progress, server: status }, + status.pulledAt ? "restarting" : "downloading", + status.error ?? "The update failed.", + at, + ); + } + + if (status.phase === "idle") { + // A fresh process reports "idle": the restart happened between two polls, + // e.g. while the browser throttled timers in a background tab. + if (progress.server) { + return finishUpdateProgress(progress, at); + } + return failUpdateProgress( + progress, + "downloading", + "Dokploy did not start an update. The server may already run the latest version.", + at, + ); + } + + return { ...progress, server: status }; +}; + +export const advanceUpdateProgress = ( + progress: UpdateProgress, + event: UpdateProgressEvent, +): UpdateProgress => { + if (isUpdateFinished(progress)) { + return progress; + } + + switch (event.type) { + case "status": + return handleStatus(progress, event.status, event.at); + case "down": + if (progress.phase === "restarting") { + return progress; + } + return { + ...progress, + phase: "restarting", + downAt: progress.downAt ?? event.at, + upAt: null, + }; + case "up": + if (progress.phase !== "restarting") { + return progress; + } + return { ...progress, phase: "starting", upAt: event.at }; + case "tick": + if ( + progress.phase === "downloading" && + event.at - progress.startedAt > DOWNLOAD_LIMIT_MS + ) { + return failUpdateProgress( + progress, + "downloading", + "Dokploy did not restart within 20 minutes.", + event.at, + ); + } + if ( + progress.downAt !== null && + event.at - progress.downAt > RESTART_LIMIT_MS + ) { + return failUpdateProgress( + progress, + progress.phase as UpdateStep, + "Dokploy did not come back within 10 minutes after it stopped.", + event.at, + ); + } + if ( + progress.phase === "starting" && + progress.upAt !== null && + event.at - progress.upAt > API_WAIT_MS + ) { + return finishUpdateProgress(progress, event.at); + } + return progress; + } +}; + +export const UPDATE_STEPS = [ + "downloading", + "restarting", + "starting", + "done", +] as const; + +export type UpdatePanelStep = (typeof UPDATE_STEPS)[number]; + +export const getUpdateStepState = ( + progress: UpdateProgress, + step: UpdatePanelStep, +): UpdateStepState => { + const index = UPDATE_STEPS.indexOf(step); + if (progress.phase === "done") { + return "done"; + } + if (progress.phase === "failed") { + const failedIndex = UPDATE_STEPS.indexOf( + progress.failedStep ?? "downloading", + ); + if (index < failedIndex) { + return "done"; + } + return index === failedIndex ? "failed" : "pending"; + } + const activeIndex = UPDATE_STEPS.indexOf(progress.phase); + if (index < activeIndex) { + return "done"; + } + return index === activeIndex ? "active" : "pending"; +}; + +export const formatElapsed = (ms: number) => { + const totalSeconds = Math.max(0, Math.floor(ms / 1000)); + if (totalSeconds < 60) { + return `${totalSeconds} s`; + } + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return `${minutes} min ${String(seconds).padStart(2, "0")} s`; +}; diff --git a/apps/dokploy/components/dashboard/settings/web-server/update-status-panel.tsx b/apps/dokploy/components/dashboard/settings/web-server/update-status-panel.tsx new file mode 100644 index 00000000000..dc42d5f458b --- /dev/null +++ b/apps/dokploy/components/dashboard/settings/web-server/update-status-panel.tsx @@ -0,0 +1,217 @@ +import { + AlertTriangle, + CheckCircle2, + Circle, + Download, + Loader2, + type LucideIcon, + PartyPopper, + Power, + Rocket, + XCircle, +} from "lucide-react"; +import type { ReactNode } from "react"; +import { cn } from "@/lib/utils"; +import { + formatElapsed, + getUpdateStepState, + type UpdatePanelStep, + type UpdateProgress, + type UpdateStepState, +} from "./update-progress"; + +export interface UpdateStatusPanelProps { + progress: UpdateProgress; + /** Current time in ms; the parent ticks it so the counters move. */ + now: number; +} + +const STEP_LABELS: Record< + UpdatePanelStep, + { title: string; icon: LucideIcon } +> = { + downloading: { title: "Downloading the new image", icon: Download }, + restarting: { title: "Restarting Dokploy", icon: Power }, + starting: { title: "Starting the new version", icon: Rocket }, + done: { title: "Done", icon: PartyPopper }, +}; + +const StateIcon = ({ state }: { state: UpdateStepState }) => { + switch (state) { + case "done": + return ; + case "active": + return ; + case "failed": + return ; + default: + return ; + } +}; + +const getDownloadDetails = ( + progress: UpdateProgress, + state: UpdateStepState, + now: number, +): ReactNode[] => { + const { server } = progress; + if (state === "done") { + // Server timestamps are compared only with each other: the browser + // clock may be off. + if (server?.pulledAt && server.startedAt) { + return [ + `Finished in ${formatElapsed(server.pulledAt - server.startedAt)}`, + ]; + } + const end = progress.downAt ?? progress.finishedAt; + return end + ? [`Finished in ${formatElapsed(end - progress.startedAt)}`] + : []; + } + + const end = state === "failed" ? (progress.finishedAt ?? now) : now; + const details: ReactNode[] = [ + `Elapsed ${formatElapsed(end - progress.startedAt)}`, + ]; + if (server?.phase === "pulling" && server.layersTotal > 0) { + details.push( + `${server.layersDownloaded} of ${server.layersTotal} layers downloaded, ${server.layersExtracted} extracted`, + ); + } + if (server?.phase === "updating" || server?.phase === "done") { + details.push("Image downloaded, waiting for Dokploy to stop"); + } + if (state === "active") { + details.push("The panel keeps working while the image downloads."); + } + return details; +}; + +const getStepDetails = ( + progress: UpdateProgress, + step: UpdatePanelStep, + state: UpdateStepState, + now: number, +): ReactNode[] => { + if (state === "pending") { + return []; + } + + switch (step) { + case "downloading": + return getDownloadDetails(progress, state, now); + case "restarting": { + if (progress.downAt === null) { + return []; + } + if (state === "done") { + const end = progress.upAt ?? progress.finishedAt; + return end + ? [`Unavailable for ${formatElapsed(end - progress.downAt)}`] + : []; + } + const end = state === "failed" ? (progress.finishedAt ?? now) : now; + return [ + + Unavailable for {formatElapsed(end - progress.downAt)} + , + "The panel is unavailable for about 20 seconds.", + ]; + } + case "starting": { + if (progress.upAt === null) { + return []; + } + if (state === "done") { + return progress.finishedAt + ? [`Ready in ${formatElapsed(progress.finishedAt - progress.upAt)}`] + : []; + } + return [ + "Dokploy answers again, waiting for the API", + `Elapsed ${formatElapsed(now - progress.upAt)}`, + ]; + } + case "done": + return progress.finishedAt + ? [ + `Updated in ${formatElapsed(progress.finishedAt - progress.startedAt)}`, + "Reloading the page...", + ] + : []; + } +}; + +export const UpdateStatusPanel = ({ + progress, + now, +}: UpdateStatusPanelProps) => { + const totalEnd = progress.finishedAt ?? now; + const output = progress.server?.output ?? []; + + return ( +
+
    + {(Object.keys(STEP_LABELS) as UpdatePanelStep[]).map((step) => { + const state = getUpdateStepState(progress, step); + const { title, icon: Icon } = STEP_LABELS[step]; + const details = getStepDetails(progress, step, state, now); + return ( +
  1. + +
    + + + {title} + + {details.map((detail, index) => ( + + {detail} + + ))} +
    +
    2. + ); + })} +
+ + {progress.phase === "failed" && ( +
+
+ +
+ + {progress.error ?? "The update failed."} + + + Check the Dokploy service logs, then close this dialog and try + again. + +
+
+ {output.length > 0 && ( + 
+							{output.join("\n")}
+
+ )} +
+ )} + + + Total time {formatElapsed(totalEnd - progress.startedAt)} + +
+ ); +}; diff --git a/apps/dokploy/components/dashboard/settings/web-server/update-webserver.tsx b/apps/dokploy/components/dashboard/settings/web-server/update-webserver.tsx index 980bbde976b..8411836249d 100644 --- a/apps/dokploy/components/dashboard/settings/web-server/update-webserver.tsx +++ b/apps/dokploy/components/dashboard/settings/web-server/update-webserver.tsx @@ -6,7 +6,7 @@ import { RefreshCw, XCircle, } from "lucide-react"; -import { useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { toast } from "sonner"; import { AlertDialog, @@ -22,6 +22,16 @@ import { import { Button } from "@/components/ui/button"; import { cn } from "@/lib/utils"; import { api } from "@/utils/api"; +import { + advanceUpdateProgress, + createUpdateProgress, + failUpdateProgress, + isUpdateFinished, + UPDATE_POLL_MS, + type UpdateProgress, + type UpdateProgressEvent, +} from "./update-progress"; +import { UpdateStatusPanel } from "./update-status-panel"; type ServiceStatus = { status: "healthy" | "unhealthy"; @@ -33,10 +43,7 @@ type HealthResult = { traefik: ServiceStatus; }; -const HEALTH_POLL_MS = 2000; const HEALTH_TIMEOUT_MS = 5000; -// Pulling a 4+ GB image on a slow link can take several minutes. -const RESTART_WAIT_MS = 20 * 60 * 1000; type ModalState = "idle" | "checking" | "results" | "updating"; @@ -70,7 +77,11 @@ export const UpdateWebServer = ({ const [modalState, setModalState] = useState("idle"); const [open, setOpen] = useState(false); const [healthResult, setHealthResult] = useState(null); + const [progress, setProgress] = useState(null); + const [now, setNow] = useState(() => Date.now()); + const unmountedRef = useRef(false); + const utils = api.useUtils(); const { mutateAsync: updateServer } = api.settings.updateServer.useMutation(); const { refetch: checkHealth } = api.settings.checkInfrastructureHealth.useQuery(undefined, { @@ -112,47 +123,108 @@ export const UpdateWebServer = ({ const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - // The old server keeps answering while the new image is pulled, so wait - // until it goes down once before treating a healthy answer as the new one. - const waitForRestart = async () => { - const startedAt = Date.now(); - while (await isServerUp()) { - if (Date.now() - startedAt > RESTART_WAIT_MS) { - throw new Error("The server did not restart"); - } - await sleep(HEALTH_POLL_MS); + useEffect(() => { + unmountedRef.current = false; + return () => { + unmountedRef.current = true; + }; + }, []); + + const isTracking = + modalState === "updating" && + progress !== null && + !isUpdateFinished(progress); + + useEffect(() => { + if (!isTracking) { + return; } - while (!(await isServerUp())) { - await sleep(HEALTH_POLL_MS); + const interval = setInterval(() => setNow(Date.now()), 1000); + return () => clearInterval(interval); + }, [isTracking]); + + // The old server answers the status query while it pulls the image; while + // Dokploy restarts only /api/health tells when it is back. + const probe = async ( + current: UpdateProgress, + ): Promise => { + if (current.phase === "restarting") { + return (await isServerUp()) + ? { type: "up", at: Date.now() } + : { type: "tick", at: Date.now() }; + } + try { + const status = await utils.client.settings.getServerUpdateStatus.query( + undefined, + { signal: AbortSignal.timeout(HEALTH_TIMEOUT_MS) }, + ); + return { type: "status", status, at: Date.now() }; + } catch { + return (await isServerUp()) + ? { type: "tick", at: Date.now() } + : { type: "down", at: Date.now() }; } }; - const handleConfirm = async () => { + const trackUpdate = async () => { + let current = createUpdateProgress(Date.now()); + setNow(current.startedAt); + setProgress(current); + try { - setModalState("updating"); await updateServer(keepImages === undefined ? undefined : { keepImages }); - await waitForRestart(); + } catch (error) { + console.error("Error updating server:", error); + current = failUpdateProgress( + current, + "downloading", + error instanceof Error && error.message + ? error.message + : "Could not start the update.", + Date.now(), + ); + } + while (!isUpdateFinished(current) && !unmountedRef.current) { + await sleep(UPDATE_POLL_MS); + const event = await probe(current); + current = advanceUpdateProgress(advanceUpdateProgress(current, event), { + type: "tick", + at: Date.now(), + }); + setProgress(current); + } + setProgress(current); + setNow(Date.now()); + return current; + }; + + const handleConfirm = async () => { + setModalState("updating"); + const result = await trackUpdate(); + + if (result.phase === "done") { toast.success( "The server has been updated. The page will be reloaded to reflect the changes...", ); setTimeout(() => { window.location.reload(); }, 2000); - } catch (error) { - setModalState("results"); - console.error("Error updating server:", error); + } else if (result.phase === "failed") { toast.error( "The server did not restart with the new version. Check the Dokploy service logs and try again.", ); } }; + const updateFailed = progress?.phase === "failed"; + const handleClose = () => { - if (modalState !== "updating") { + if (modalState !== "updating" || updateFailed) { setOpen(false); setModalState("idle"); setHealthResult(null); + setProgress(null); } }; @@ -179,7 +251,12 @@ export const UpdateWebServer = ({ {modalState === "checking" && "Verifying Services..."} {modalState === "results" && (allHealthy ? "Ready to Update" : "Service Issues Detected")} - {modalState === "updating" && "Server update in progress"} + {modalState === "updating" && + (updateFailed + ? "Server update failed" + : progress?.phase === "done" + ? "Server updated" + : "Server update in progress")}
@@ -187,10 +264,9 @@ export const UpdateWebServer = ({ This will update the web server to the selected latest build. AgentHits fork installs update from the AgentHits GHCR image. - The new image is downloaded first, so the panel is - unavailable only for about 20 seconds while Dokploy - restarts. The page will be reloaded once the update is - finished. + The new image is downloaded first, so the panel is unavailable + only for about 20 seconds while Dokploy restarts. The page + will be reloaded once the update is finished.

We recommend verifying that all services are running before @@ -246,12 +322,8 @@ export const UpdateWebServer = ({
)} - {modalState === "updating" && ( - - - Downloading the new image. The panel keeps working meanwhile; - it is unavailable for about 20 seconds while Dokploy restarts. - + {modalState === "updating" && progress && ( + )} @@ -268,6 +340,11 @@ export const UpdateWebServer = ({ )} + {modalState === "updating" && updateFailed && ( + + Close + + )} {modalState === "results" && ( Cancel diff --git a/apps/dokploy/server/api/routers/settings.ts b/apps/dokploy/server/api/routers/settings.ts index e5af5dd28fe..e3f21147953 100644 --- a/apps/dokploy/server/api/routers/settings.ts +++ b/apps/dokploy/server/api/routers/settings.ts @@ -25,6 +25,7 @@ import { getDokployVersionData, getLogCleanupStatus, getOfficialUpdateCommand, + getServerUpdateStatus, getUpdateData, getWebServerSettings, IS_CLOUD, @@ -45,8 +46,8 @@ import { resolveDockerDiskUsageDetailLimit, sendDockerCleanupNotifications, setupGPUSupport, - spawnAsync, startLogCleanup, + startServerUpdate, stopLogCleanup, updateLetsEncryptEmail, updateServerById, @@ -658,22 +659,16 @@ export const settingsRouter = createTRPCRouter({ const keepImages = input?.keepImages; const data = await getUpdateData(packageInfo.version); if (data.updateAvailable) { - if (data.updateSource === "agenthits") { - void spawnAsync("sh", [ - "-c", - getAgentHitsUpdateCommand( - packageInfo.version, - data.latestVersion, - data.latestOfficialVersion, - keepImages, - ), - ]); - } else { - void spawnAsync("sh", [ - "-c", - getOfficialUpdateCommand(data.latestVersion ?? "", keepImages), - ]); - } + startServerUpdate( + data.updateSource === "agenthits" + ? getAgentHitsUpdateCommand( + packageInfo.version, + data.latestVersion, + data.latestOfficialVersion, + keepImages, + ) + : getOfficialUpdateCommand(data.latestVersion ?? "", keepImages), + ); await audit(ctx, { action: "update", resourceType: "settings", @@ -684,6 +679,10 @@ export const settingsRouter = createTRPCRouter({ return true; }), + getServerUpdateStatus: adminProcedure.query(() => { + return getServerUpdateStatus(); + }), + getDokployVersion: protectedProcedure.query(() => { return packageInfo.version; }), diff --git a/apps/dokploy/server/api/utils/procedure-access.ts b/apps/dokploy/server/api/utils/procedure-access.ts index 8a945508333..320a298f03d 100644 --- a/apps/dokploy/server/api/utils/procedure-access.ts +++ b/apps/dokploy/server/api/utils/procedure-access.ts @@ -599,6 +599,7 @@ export const PROCEDURE_ACCESS: Record = { "settings.getLogCleanupStatus": "read", "settings.getOpenApiDocument": "read", "settings.getReleaseTag": "read", + "settings.getServerUpdateStatus": "read", "settings.getTraefikPorts": "read", "settings.getUpdateData": "write", "settings.getWebServerSettings": "read", diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 25645ecc55d..55470a0f2bb 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -61,6 +61,7 @@ export * from "./services/user"; export * from "./services/vault-provider"; export * from "./services/volume-backups"; export * from "./services/web-server-settings"; +export * from "./services/web-server-update"; export * from "./setup/config-paths"; export * from "./setup/forward-auth-setup"; export * from "./setup/monitoring-setup"; diff --git a/packages/server/src/services/settings.ts b/packages/server/src/services/settings.ts index 20541750d4b..ea4029a792b 100644 --- a/packages/server/src/services/settings.ts +++ b/packages/server/src/services/settings.ts @@ -15,6 +15,7 @@ import { initializeTraefikService, type TraefikOptions, } from "../setup/traefik-setup"; +import { UPDATE_IMAGE_PULLED_MARKER } from "./web-server-update"; export interface IUpdateData { latestVersion: string | null; updateAvailable: boolean; @@ -496,6 +497,7 @@ export const getAgentHitsUpdateCommand = ( // stop-first, so a pull inside the update would happen with Dokploy down. return ` docker pull ${quoteShellArg(getAgentHitsUpdateImage())} || exit 1 +echo ${quoteShellArg(UPDATE_IMAGE_PULLED_MARKER)} fork_version_env_arg="" if docker service inspect dokploy --format '{{range .Spec.TaskTemplate.ContainerSpec.Env}}{{println .}}{{end}}' 2>/dev/null | grep -q '^DOKPLOY_FORK_VERSION='; then fork_version_env_arg="--env-rm DOKPLOY_FORK_VERSION" @@ -520,6 +522,7 @@ export const getOfficialUpdateCommand = ( return ` docker pull ${image} || exit 1 +echo ${quoteShellArg(UPDATE_IMAGE_PULLED_MARKER)} docker service update --force --image ${image} ${keepImagesArg} dokploy `; }; diff --git a/packages/server/src/services/web-server-update.ts b/packages/server/src/services/web-server-update.ts new file mode 100644 index 00000000000..4d82028348d --- /dev/null +++ b/packages/server/src/services/web-server-update.ts @@ -0,0 +1,158 @@ +import { spawnAsync } from "../utils/process/spawnAsync"; + +/** Printed by the update scripts once `docker pull` succeeded. */ +export const UPDATE_IMAGE_PULLED_MARKER = "[dokploy-update] image pulled"; + +export type ServerUpdateJobPhase = + | "idle" + | "pulling" + | "updating" + | "done" + | "failed"; + +export interface ServerUpdateStatus { + phase: ServerUpdateJobPhase; + startedAt: number | null; + pulledAt: number | null; + finishedAt: number | null; + error: string | null; + layersTotal: number; + layersDownloaded: number; + layersExtracted: number; + /** Last lines of the update script output: image refs and layer ids only. */ + output: string[]; +} + +const OUTPUT_LINES = 5; +const OUTPUT_LINE_MAX_LENGTH = 300; + +const createIdleStatus = (): ServerUpdateStatus => ({ + phase: "idle", + startedAt: null, + pulledAt: null, + finishedAt: null, + error: null, + layersTotal: 0, + layersDownloaded: 0, + layersExtracted: 0, + output: [], +}); + +// Lives only in this process: the update restarts Dokploy, and the new +// process starting from "idle" is how the UI knows the restart happened. +let status = createIdleStatus(); +let layers = new Map(); +let pendingOutput = ""; + +export const getServerUpdateStatus = (): ServerUpdateStatus => ({ + ...status, + output: [...status.output], +}); + +export const resetServerUpdateStatus = () => { + status = createIdleStatus(); + layers = new Map(); + pendingOutput = ""; +}; + +export const isServerUpdateRunning = () => + status.phase === "pulling" || status.phase === "updating"; + +const LAYER_LINE = /^([0-9a-f]{12,64}): (.+)$/; +const DOWNLOADED_LAYER_STATES = new Set([ + "Download complete", + "Pull complete", + "Already exists", +]); +const EXTRACTED_LAYER_STATES = new Set(["Pull complete", "Already exists"]); + +const countLayers = (states: Set) => + [...layers.values()].filter((state) => states.has(state)).length; + +const handleOutputLine = (rawLine: string) => { + const line = rawLine.trim(); + if (!line) { + return; + } + + if (line === UPDATE_IMAGE_PULLED_MARKER) { + if (status.phase === "pulling") { + status.phase = "updating"; + status.pulledAt = Date.now(); + } + return; + } + + const layer = line.match(LAYER_LINE); + if (layer && status.phase === "pulling") { + const [, id, state] = layer as unknown as [string, string, string]; + // Docker repeats a layer's earlier states while it is retried, so a + // layer that already finished must not go back to "downloading". + if (!EXTRACTED_LAYER_STATES.has(layers.get(id) ?? "")) { + layers.set(id, state); + } + status.layersTotal = layers.size; + status.layersDownloaded = countLayers(DOWNLOADED_LAYER_STATES); + status.layersExtracted = countLayers(EXTRACTED_LAYER_STATES); + } + + status.output = [ + ...status.output, + line.slice(0, OUTPUT_LINE_MAX_LENGTH), + ].slice(-OUTPUT_LINES); +}; + +const handleOutput = (chunk: string) => { + // Chunks split lines anywhere; docker also separates some updates with \r. + const parts = `${pendingOutput}${chunk}`.split(/\r?\n|\r/); + pendingOutput = parts.pop() ?? ""; + for (const part of parts) { + handleOutputLine(part); + } +}; + +const finish = (error: string | null) => { + if (pendingOutput) { + handleOutputLine(pendingOutput); + pendingOutput = ""; + } + status.finishedAt = Date.now(); + if (error) { + status.phase = "failed"; + status.error = error; + } else { + status.phase = "done"; + } +}; + +/** + * Runs an update script from `getAgentHitsUpdateCommand` or + * `getOfficialUpdateCommand` in the background and tracks it in + * `getServerUpdateStatus`. Returns false when an update is already running. + */ +export const startServerUpdate = (command: string) => { + if (isServerUpdateRunning()) { + return false; + } + + resetServerUpdateStatus(); + status.phase = "pulling"; + status.startedAt = Date.now(); + + spawnAsync("sh", ["-c", command], handleOutput).then( + () => finish(null), + (error: unknown) => { + const spawnFailed = + typeof (error as { code?: unknown })?.code !== "number"; + if (spawnFailed) { + finish("Could not start the update script."); + } else if (status.phase === "pulling") { + finish("Downloading the new image failed."); + } else { + finish("docker service update failed."); + } + }, + ); + + return true; +};