From 0ed4bd6269688b75b9a278e0160089aa9b273969 Mon Sep 17 00:00:00 2001 From: merge-check <140916359+agentHits@users.noreply.github.com> Date: Wed, 7 Oct 2026 07:38:21 -0400 Subject: [PATCH] fix: pull the new image before restarting Dokploy on update The dokploy service updates stop-first, so the swarm task pulled the new 4.4 GB image only after the old container had stopped. On dev that left the panel down for 2 min 12 s (old container stopped 11:31:42, new one started 11:33:54) and Traefik answered Bad Gateway. Both update commands (AgentHits and official) now run `docker pull` first and stop if it fails, so the downtime is only the restart. The official update moves into getOfficialUpdateCommand to be testable. The update dialog used to reload after a fixed 8 s once /api/health answered. With the pull in front, the old server still answers then, so the dialog now waits until the server has gone down once and is healthy again (giving up after 20 minutes with an error). The image list and the cleanup also missed images that swarm pulls by digest: they have no tag, and `docker image ls` hides untagged images unless `-a` is passed. The running image after a UI update was one of them. Co-Authored-By: Claude Opus 5.5 --- .../settings/docker-server-boundary.test.ts | 33 +++++----- .../settings/dokploy-image-cleanup.test.ts | 47 ++++++++++++++ .../request-log-authorization.test.ts | 3 +- .../settings/web-server/update-webserver.tsx | 63 ++++++++++++------- apps/dokploy/server/api/routers/settings.ts | 15 ++--- packages/server/src/services/settings.ts | 23 ++++++- 6 files changed, 131 insertions(+), 53 deletions(-) diff --git a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts index b6c6089118f..a015493d9d1 100644 --- a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts +++ b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts @@ -24,6 +24,7 @@ const mocks = vi.hoisted(() => ({ generateOpenApiDocument: vi.fn(), getAccessibleServerIds: vi.fn(), getAgentHitsUpdateCommand: vi.fn(), + getOfficialUpdateCommand: vi.fn(), getDockerDiskUsage: vi.fn(), getDokployImageTag: vi.fn(), getDokployVersionData: vi.fn(), @@ -67,7 +68,6 @@ const redactWebServerSettings = (settings: T) => settings; vi.mock("@dokploy/server", () => ({ CLEANUP_CRON_JOB: "0 0 * * *", DEFAULT_UPDATE_DATA: {}, - DOKPLOY_KEEP_IMAGES_ENV: "DOKPLOY_KEEP_IMAGES", DOKPLOY_KEEP_IMAGES_MAX: 5, DOKPLOY_KEEP_IMAGES_MIN: 3, getDokployImageKeepCount: vi.fn(), @@ -90,6 +90,7 @@ vi.mock("@dokploy/server", () => ({ findServerById: mocks.findServerById, getAccessibleServerIds: mocks.getAccessibleServerIds, getAgentHitsUpdateCommand: mocks.getAgentHitsUpdateCommand, + getOfficialUpdateCommand: mocks.getOfficialUpdateCommand, getDockerDiskUsage: mocks.getDockerDiskUsage, getDokployImageTag: mocks.getDokployImageTag, getDokployVersionData: mocks.getDokployVersionData, @@ -470,15 +471,17 @@ describe("settings Docker server boundary", () => { updateSource: "official", }); + mocks.getOfficialUpdateCommand.mockReturnValue("official update command"); + await expect(createCaller().updateServer()).resolves.toBe(true); - expect(mocks.spawnAsync).toHaveBeenCalledWith("docker", [ - "service", - "update", - "--force", - "--image", - "dokploy/dokploy:v0.30.0", - "dokploy", + expect(mocks.getOfficialUpdateCommand).toHaveBeenCalledWith( + "v0.30.0", + undefined, + ); + expect(mocks.spawnAsync).toHaveBeenCalledWith("sh", [ + "-c", + "official update command", ]); expect(mocks.getAgentHitsUpdateCommand).not.toHaveBeenCalled(); }); @@ -494,16 +497,10 @@ describe("settings Docker server boundary", () => { createCaller().updateServer({ keepImages: null }), ).resolves.toBe(true); - expect(mocks.spawnAsync).toHaveBeenCalledWith("docker", [ - "service", - "update", - "--force", - "--image", - "dokploy/dokploy:v0.30.0", - "--env-add", - "DOKPLOY_KEEP_IMAGES=0", - "dokploy", - ]); + expect(mocks.getOfficialUpdateCommand).toHaveBeenCalledWith( + "v0.30.0", + null, + ); }); it("requires api.read before generating the OpenAPI document", async () => { diff --git a/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts b/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts index f7045a5123d..ff87e8d47d5 100644 --- a/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts +++ b/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts @@ -18,6 +18,7 @@ const { getDokployImageKeepCount, getDokployImageRepositories, getDokployImages, + getOfficialUpdateCommand, } = await import("@dokploy/server/services/settings"); describe("Dokploy image cleanup", () => { @@ -199,3 +200,49 @@ describe("Dokploy image list", () => { expect(execAsync).toHaveBeenCalledTimes(1); }); }); + +describe("Dokploy update commands", () => { + beforeEach(() => { + vi.stubEnv("DOKPLOY_AGENTHITS_UPDATE_IMAGE", undefined); + vi.stubEnv("DOKPLOY_AGENTHITS_UPDATE_TAG", undefined); + }); + + afterEach(() => { + vi.unstubAllEnvs(); + }); + + const expectPullBeforeUpdate = (command: string, pull: string) => { + expect(command).toContain(`${pull} || exit 1`); + expect(command.indexOf(pull)).toBeLessThan( + command.indexOf("docker service update"), + ); + }; + + it("pulls the AgentHits image while the old container still runs", () => { + expectPullBeforeUpdate( + getAgentHitsUpdateCommand("v0.30.6"), + "docker pull ghcr.io/agenthits/dokploy\\:agenthits-dev", + ); + }); + + it("pulls the official image before updating the service", () => { + const command = getOfficialUpdateCommand("v0.30.7", 3); + + expectPullBeforeUpdate(command, "docker pull dokploy/dokploy\\:v0.30.7"); + expect(command).toContain( + "--image dokploy/dokploy\\:v0.30.7 --env-add DOKPLOY_KEEP_IMAGES\\=3 dokploy", + ); + expect(getOfficialUpdateCommand("v0.30.7")).not.toContain( + "DOKPLOY_KEEP_IMAGES", + ); + }); + + it("includes untagged images that swarm pulled by digest", async () => { + execAsync.mockResolvedValue({ stdout: "", stderr: "" }); + + await getDokployImages(); + + expect(execAsync.mock.calls[0]?.[0]).toMatch(/^docker image ls -a -q /); + expect(getDokployImageCleanupCommand(3)).toContain("docker image ls -a -q"); + }); +}); diff --git a/apps/dokploy/__test__/settings/request-log-authorization.test.ts b/apps/dokploy/__test__/settings/request-log-authorization.test.ts index f9066263f0a..85d505e754c 100644 --- a/apps/dokploy/__test__/settings/request-log-authorization.test.ts +++ b/apps/dokploy/__test__/settings/request-log-authorization.test.ts @@ -50,6 +50,7 @@ const mocks = vi.hoisted(() => ({ generateOpenApiDocument: vi.fn(), getAccessibleServerIds: vi.fn(), getAgentHitsUpdateCommand: vi.fn(), + getOfficialUpdateCommand: vi.fn(), getDockerDiskUsage: vi.fn(), getDokployImageTag: vi.fn(), getDokployVersionData: vi.fn(), @@ -90,7 +91,6 @@ const mocks = vi.hoisted(() => ({ vi.mock("@dokploy/server", () => ({ CLEANUP_CRON_JOB: "0 0 * * *", DEFAULT_UPDATE_DATA: {}, - DOKPLOY_KEEP_IMAGES_ENV: "DOKPLOY_KEEP_IMAGES", DOKPLOY_KEEP_IMAGES_MAX: 5, DOKPLOY_KEEP_IMAGES_MIN: 3, getDokployImageKeepCount: vi.fn(), @@ -112,6 +112,7 @@ vi.mock("@dokploy/server", () => ({ findServerById: mocks.findServerById, getAccessibleServerIds: mocks.getAccessibleServerIds, getAgentHitsUpdateCommand: mocks.getAgentHitsUpdateCommand, + getOfficialUpdateCommand: mocks.getOfficialUpdateCommand, getDockerDiskUsage: mocks.getDockerDiskUsage, getDokployImageTag: mocks.getDokployImageTag, getDokployVersionData: mocks.getDokployVersionData, diff --git a/apps/dokploy/components/dashboard/settings/web-server/update-webserver.tsx b/apps/dokploy/components/dashboard/settings/web-server/update-webserver.tsx index 8fbc3a1e8e3..980bbde976b 100644 --- a/apps/dokploy/components/dashboard/settings/web-server/update-webserver.tsx +++ b/apps/dokploy/components/dashboard/settings/web-server/update-webserver.tsx @@ -33,6 +33,11 @@ type HealthResult = { traefik: ServiceStatus; }; +const HEALTH_POLL_MS = 2000; +const HEALTH_TIMEOUT_MS = 5000; +// Pulling a 4+ GB image on a slow link can take several minutes. +const RESTART_WAIT_MS = 20 * 60 * 1000; + type ModalState = "idle" | "checking" | "results" | "updating"; const ServiceStatusItem = ({ @@ -92,23 +97,33 @@ export const UpdateWebServer = ({ healthResult.postgres.status === "healthy" && healthResult.traefik.status === "healthy"; - const checkIsUpdateFinished = async () => { + const isServerUp = async () => { try { - const response = await fetch("/api/health"); - if (!response.ok) { - throw new Error("Health check failed"); - } + const response = await fetch("/api/health", { + cache: "no-store", + signal: AbortSignal.timeout(HEALTH_TIMEOUT_MS), + }); + return response.ok; + } catch { + return false; + } + }; - toast.success( - "The server has been updated. The page will be reloaded to reflect the changes...", - ); + const sleep = (ms: number) => + new Promise((resolve) => setTimeout(resolve, ms)); - setTimeout(() => { - window.location.reload(); - }, 2000); - } catch { - await new Promise((resolve) => setTimeout(resolve, 2000)); - void checkIsUpdateFinished(); + // The old server keeps answering while the new image is pulled, so wait + // until it goes down once before treating a healthy answer as the new one. + const waitForRestart = async () => { + const startedAt = Date.now(); + while (await isServerUp()) { + if (Date.now() - startedAt > RESTART_WAIT_MS) { + throw new Error("The server did not restart"); + } + await sleep(HEALTH_POLL_MS); + } + while (!(await isServerUp())) { + await sleep(HEALTH_POLL_MS); } }; @@ -116,15 +131,19 @@ export const UpdateWebServer = ({ try { setModalState("updating"); await updateServer(keepImages === undefined ? undefined : { keepImages }); + await waitForRestart(); - await new Promise((resolve) => setTimeout(resolve, 8000)); - - await checkIsUpdateFinished(); + toast.success( + "The server has been updated. The page will be reloaded to reflect the changes...", + ); + setTimeout(() => { + window.location.reload(); + }, 2000); } catch (error) { setModalState("results"); console.error("Error updating server:", error); toast.error( - "An error occurred while updating the server, please try again.", + "The server did not restart with the new version. Check the Dokploy service logs and try again.", ); } }; @@ -168,8 +187,9 @@ export const UpdateWebServer = ({ This will update the web server to the selected latest build. AgentHits fork installs update from the AgentHits GHCR image. - You will not be able to use the panel during the update - process. The page will be reloaded once the update is + The new image is downloaded first, so the panel is + unavailable only for about 20 seconds while Dokploy + restarts. The page will be reloaded once the update is finished.

@@ -229,7 +249,8 @@ export const UpdateWebServer = ({ {modalState === "updating" && ( - The server is being updated, please wait... + Downloading the new image. The panel keeps working meanwhile; + it is unavailable for about 20 seconds while Dokploy restarts. )} diff --git a/apps/dokploy/server/api/routers/settings.ts b/apps/dokploy/server/api/routers/settings.ts index b7f957e5768..e5af5dd28fe 100644 --- a/apps/dokploy/server/api/routers/settings.ts +++ b/apps/dokploy/server/api/routers/settings.ts @@ -12,7 +12,6 @@ import { cleanupSystem, cleanupVolumes, DEFAULT_UPDATE_DATA, - DOKPLOY_KEEP_IMAGES_ENV, DOKPLOY_KEEP_IMAGES_MAX, DOKPLOY_KEEP_IMAGES_MIN, filterProtectedTraefikEntries, @@ -25,6 +24,7 @@ import { getDokployImageTag, getDokployVersionData, getLogCleanupStatus, + getOfficialUpdateCommand, getUpdateData, getWebServerSettings, IS_CLOUD, @@ -669,16 +669,9 @@ export const settingsRouter = createTRPCRouter({ ), ]); } else { - void spawnAsync("docker", [ - "service", - "update", - "--force", - "--image", - `dokploy/dokploy:${data.latestVersion}`, - ...(keepImages === undefined - ? [] - : ["--env-add", `${DOKPLOY_KEEP_IMAGES_ENV}=${keepImages ?? 0}`]), - "dokploy", + void spawnAsync("sh", [ + "-c", + getOfficialUpdateCommand(data.latestVersion ?? "", keepImages), ]); } await audit(ctx, { diff --git a/packages/server/src/services/settings.ts b/packages/server/src/services/settings.ts index d90df5e1870..20541750d4b 100644 --- a/packages/server/src/services/settings.ts +++ b/packages/server/src/services/settings.ts @@ -301,6 +301,8 @@ export const getDokployImageRepositories = () => [ * Removes Dokploy web server images except the `keep` newest ones. Swarm keeps * exited task containers of the dokploy service, and they pin old images, so * those are removed first. Images used by running containers are never forced. + * `-a` is needed because images swarm pulls by digest have no tag, and + * `docker image ls` hides untagged images without it. */ export const getDokployImageCleanupCommand = (keep: number) => { const filters = getDokployImageRepositories() @@ -308,7 +310,7 @@ export const getDokployImageCleanupCommand = (keep: number) => { .join(" "); return ` -ids=$(docker image ls -q --no-trunc ${filters} | sort -u) +ids=$(docker image ls -a -q --no-trunc ${filters} | sort -u) if [ -z "$ids" ]; then exit 0 fi @@ -404,7 +406,7 @@ export const getDokployImages = async (): Promise => { .map((repository) => `--filter ${quoteShellArg(`reference=${repository}`)}`) .join(" "); const { stdout: idsOutput } = await execAsync( - `docker image ls -q --no-trunc ${filters} | sort -u`, + `docker image ls -a -q --no-trunc ${filters} | sort -u`, ); const ids = idsOutput.split("\n").filter(Boolean); if (ids.length === 0) { @@ -490,7 +492,10 @@ export const getAgentHitsUpdateCommand = ( const keepImagesArg = keepImages === undefined ? "" : getDokployKeepImagesEnvArg(keepImages); + // Pull while the old container still serves the panel: the service is + // stop-first, so a pull inside the update would happen with Dokploy down. return ` +docker pull ${quoteShellArg(getAgentHitsUpdateImage())} || exit 1 fork_version_env_arg="" if docker service inspect dokploy --format '{{range .Spec.TaskTemplate.ContainerSpec.Env}}{{println .}}{{end}}' 2>/dev/null | grep -q '^DOKPLOY_FORK_VERSION='; then fork_version_env_arg="--env-rm DOKPLOY_FORK_VERSION" @@ -505,6 +510,20 @@ docker service update --force \\ `; }; +export const getOfficialUpdateCommand = ( + version: string, + keepImages?: number | null, +) => { + const image = quoteShellArg(`dokploy/dokploy:${version}`); + const keepImagesArg = + keepImages === undefined ? "" : getDokployKeepImagesEnvArg(keepImages); + + return ` +docker pull ${image} || exit 1 +docker service update --force --image ${image} ${keepImagesArg} dokploy +`; +}; + /** Returns Dokploy docker service image digest */ export const getServiceImageDigest = async () => { const { stdout } = await execAsync(