fix: pull the new image before restarting Dokploy on update

apps/dokploy/__test__/settings/docker-server-boundary.test.ts

@@ -24,6 +24,7 @@ const mocks = vi.hoisted(() => ({
 	generateOpenApiDocument: vi.fn(),
 	getAccessibleServerIds: vi.fn(),
 	getAgentHitsUpdateCommand: vi.fn(),
+	getOfficialUpdateCommand: vi.fn(),
 	getDockerDiskUsage: vi.fn(),
 	getDokployImageTag: vi.fn(),
 	getDokployVersionData: vi.fn(),
@@ -67,7 +68,6 @@ const redactWebServerSettings = <T>(settings: T) => settings;
 vi.mock("@dokploy/server", () => ({
 	CLEANUP_CRON_JOB: "0 0 * * *",
 	DEFAULT_UPDATE_DATA: {},
-	DOKPLOY_KEEP_IMAGES_ENV: "DOKPLOY_KEEP_IMAGES",
 	DOKPLOY_KEEP_IMAGES_MAX: 5,
 	DOKPLOY_KEEP_IMAGES_MIN: 3,
 	getDokployImageKeepCount: vi.fn(),
@@ -90,6 +90,7 @@ vi.mock("@dokploy/server", () => ({
 	findServerById: mocks.findServerById,
 	getAccessibleServerIds: mocks.getAccessibleServerIds,
 	getAgentHitsUpdateCommand: mocks.getAgentHitsUpdateCommand,
+	getOfficialUpdateCommand: mocks.getOfficialUpdateCommand,
 	getDockerDiskUsage: mocks.getDockerDiskUsage,
 	getDokployImageTag: mocks.getDokployImageTag,
 	getDokployVersionData: mocks.getDokployVersionData,
@@ -470,15 +471,17 @@ describe("settings Docker server boundary", () => {
 			updateSource: "official",
 		});
 
+		mocks.getOfficialUpdateCommand.mockReturnValue("official update command");
+
 		await expect(createCaller().updateServer()).resolves.toBe(true);
 
-		expect(mocks.spawnAsync).toHaveBeenCalledWith("docker", [
-			"service",
-			"update",
-			"--force",
-			"--image",
-			"dokploy/dokploy:v0.30.0",
-			"dokploy",
+		expect(mocks.getOfficialUpdateCommand).toHaveBeenCalledWith(
+			"v0.30.0",
+			undefined,
+		);
+		expect(mocks.spawnAsync).toHaveBeenCalledWith("sh", [
+			"-c",
+			"official update command",
 		]);
 		expect(mocks.getAgentHitsUpdateCommand).not.toHaveBeenCalled();
 	});
@@ -494,16 +497,10 @@ describe("settings Docker server boundary", () => {
 			createCaller().updateServer({ keepImages: null }),
 		).resolves.toBe(true);
 
-		expect(mocks.spawnAsync).toHaveBeenCalledWith("docker", [
-			"service",
-			"update",
-			"--force",
-			"--image",
-			"dokploy/dokploy:v0.30.0",
-			"--env-add",
-			"DOKPLOY_KEEP_IMAGES=0",
-			"dokploy",
-		]);
+		expect(mocks.getOfficialUpdateCommand).toHaveBeenCalledWith(
+			"v0.30.0",
+			null,
+		);
 	});
 
 	it("requires api.read before generating the OpenAPI document", async () => {

apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts

@@ -18,6 +18,7 @@ const {
 	getDokployImageKeepCount,
 	getDokployImageRepositories,
 	getDokployImages,
+	getOfficialUpdateCommand,
 } = await import("@dokploy/server/services/settings");
 
 describe("Dokploy image cleanup", () => {
@@ -199,3 +200,49 @@ describe("Dokploy image list", () => {
 		expect(execAsync).toHaveBeenCalledTimes(1);
 	});
 });
+
+describe("Dokploy update commands", () => {
+	beforeEach(() => {
+		vi.stubEnv("DOKPLOY_AGENTHITS_UPDATE_IMAGE", undefined);
+		vi.stubEnv("DOKPLOY_AGENTHITS_UPDATE_TAG", undefined);
+	});
+
+	afterEach(() => {
+		vi.unstubAllEnvs();
+	});
+
+	const expectPullBeforeUpdate = (command: string, pull: string) => {
+		expect(command).toContain(`${pull} || exit 1`);
+		expect(command.indexOf(pull)).toBeLessThan(
+			command.indexOf("docker service update"),
+		);
+	};
+
+	it("pulls the AgentHits image while the old container still runs", () => {
+		expectPullBeforeUpdate(
+			getAgentHitsUpdateCommand("v0.30.6"),
+			"docker pull ghcr.io/agenthits/dokploy\\:agenthits-dev",
+		);
+	});
+
+	it("pulls the official image before updating the service", () => {
+		const command = getOfficialUpdateCommand("v0.30.7", 3);
+
+		expectPullBeforeUpdate(command, "docker pull dokploy/dokploy\\:v0.30.7");
+		expect(command).toContain(
+			"--image dokploy/dokploy\\:v0.30.7 --env-add DOKPLOY_KEEP_IMAGES\\=3 dokploy",
+		);
+		expect(getOfficialUpdateCommand("v0.30.7")).not.toContain(
+			"DOKPLOY_KEEP_IMAGES",
+		);
+	});
+
+	it("includes untagged images that swarm pulled by digest", async () => {
+		execAsync.mockResolvedValue({ stdout: "", stderr: "" });
+
+		await getDokployImages();
+
+		expect(execAsync.mock.calls[0]?.[0]).toMatch(/^docker image ls -a -q /);
+		expect(getDokployImageCleanupCommand(3)).toContain("docker image ls -a -q");
+	});
+});

apps/dokploy/__test__/settings/request-log-authorization.test.ts

@@ -50,6 +50,7 @@ const mocks = vi.hoisted(() => ({
 	generateOpenApiDocument: vi.fn(),
 	getAccessibleServerIds: vi.fn(),
 	getAgentHitsUpdateCommand: vi.fn(),
+	getOfficialUpdateCommand: vi.fn(),
 	getDockerDiskUsage: vi.fn(),
 	getDokployImageTag: vi.fn(),
 	getDokployVersionData: vi.fn(),
@@ -90,7 +91,6 @@ const mocks = vi.hoisted(() => ({
 vi.mock("@dokploy/server", () => ({
 	CLEANUP_CRON_JOB: "0 0 * * *",
 	DEFAULT_UPDATE_DATA: {},
-	DOKPLOY_KEEP_IMAGES_ENV: "DOKPLOY_KEEP_IMAGES",
 	DOKPLOY_KEEP_IMAGES_MAX: 5,
 	DOKPLOY_KEEP_IMAGES_MIN: 3,
 	getDokployImageKeepCount: vi.fn(),
@@ -112,6 +112,7 @@ vi.mock("@dokploy/server", () => ({
 	findServerById: mocks.findServerById,
 	getAccessibleServerIds: mocks.getAccessibleServerIds,
 	getAgentHitsUpdateCommand: mocks.getAgentHitsUpdateCommand,
+	getOfficialUpdateCommand: mocks.getOfficialUpdateCommand,
 	getDockerDiskUsage: mocks.getDockerDiskUsage,
 	getDokployImageTag: mocks.getDokployImageTag,
 	getDokployVersionData: mocks.getDokployVersionData,

apps/dokploy/components/dashboard/settings/web-server/update-webserver.tsx

@@ -33,6 +33,11 @@ type HealthResult = {
 	traefik: ServiceStatus;
 };
 
+const HEALTH_POLL_MS = 2000;
+const HEALTH_TIMEOUT_MS = 5000;
+// Pulling a 4+ GB image on a slow link can take several minutes.
+const RESTART_WAIT_MS = 20 * 60 * 1000;
+
 type ModalState = "idle" | "checking" | "results" | "updating";
 
 const ServiceStatusItem = ({
@@ -92,39 +97,53 @@ export const UpdateWebServer = ({
 		healthResult.postgres.status === "healthy" &&
 		healthResult.traefik.status === "healthy";
 
-	const checkIsUpdateFinished = async () => {
+	const isServerUp = async () => {
 		try {
-			const response = await fetch("/api/health");
-			if (!response.ok) {
-				throw new Error("Health check failed");
-			}
+			const response = await fetch("/api/health", {
+				cache: "no-store",
+				signal: AbortSignal.timeout(HEALTH_TIMEOUT_MS),
+			});
+			return response.ok;
+		} catch {
+			return false;
+		}
+	};
 
-			toast.success(
-				"The server has been updated. The page will be reloaded to reflect the changes...",
-			);
+	const sleep = (ms: number) =>
+		new Promise((resolve) => setTimeout(resolve, ms));
 
-			setTimeout(() => {
-				window.location.reload();
-			}, 2000);
-		} catch {
-			await new Promise((resolve) => setTimeout(resolve, 2000));
-			void checkIsUpdateFinished();
+	// The old server keeps answering while the new image is pulled, so wait
+	// until it goes down once before treating a healthy answer as the new one.
+	const waitForRestart = async () => {
+		const startedAt = Date.now();
+		while (await isServerUp()) {
+			if (Date.now() - startedAt > RESTART_WAIT_MS) {
+				throw new Error("The server did not restart");
+			}
+			await sleep(HEALTH_POLL_MS);
+		}
+		while (!(await isServerUp())) {
+			await sleep(HEALTH_POLL_MS);
 		}
 	};
 
 	const handleConfirm = async () => {
 		try {
 			setModalState("updating");
 			await updateServer(keepImages === undefined ? undefined : { keepImages });
+			await waitForRestart();
 
-			await new Promise((resolve) => setTimeout(resolve, 8000));
-
-			await checkIsUpdateFinished();
+			toast.success(
+				"The server has been updated. The page will be reloaded to reflect the changes...",
+			);
+			setTimeout(() => {
+				window.location.reload();
+			}, 2000);
 		} catch (error) {
 			setModalState("results");
 			console.error("Error updating server:", error);
 			toast.error(
-				"An error occurred while updating the server, please try again.",
+				"The server did not restart with the new version. Check the Dokploy service logs and try again.",
 			);
 		}
 	};
@@ -168,8 +187,9 @@ export const UpdateWebServer = ({
 								<span>
 									This will update the web server to the selected latest build.
 									AgentHits fork installs update from the AgentHits GHCR image.
-									You will not be able to use the panel during the update
-									process. The page will be reloaded once the update is
+									The new image is downloaded first, so the panel is
+									unavailable only for about 20 seconds while Dokploy
+									restarts. The page will be reloaded once the update is
 									finished.
 									<br />
 									<br />
@@ -229,7 +249,8 @@ export const UpdateWebServer = ({
 							{modalState === "updating" && (
 								<span className="flex items-center gap-2">
 									<Loader2 className="animate-spin h-4 w-4" />
-									The server is being updated, please wait...
+									Downloading the new image. The panel keeps working meanwhile;
+									it is unavailable for about 20 seconds while Dokploy restarts.
 								</span>
 							)}
 						</div>

apps/dokploy/server/api/routers/settings.ts

@@ -12,7 +12,6 @@ import {
 	cleanupSystem,
 	cleanupVolumes,
 	DEFAULT_UPDATE_DATA,
-	DOKPLOY_KEEP_IMAGES_ENV,
 	DOKPLOY_KEEP_IMAGES_MAX,
 	DOKPLOY_KEEP_IMAGES_MIN,
 	filterProtectedTraefikEntries,
@@ -25,6 +24,7 @@ import {
 	getDokployImageTag,
 	getDokployVersionData,
 	getLogCleanupStatus,
+	getOfficialUpdateCommand,
 	getUpdateData,
 	getWebServerSettings,
 	IS_CLOUD,
@@ -669,16 +669,9 @@ export const settingsRouter = createTRPCRouter({
 						),
 					]);
 				} else {
-					void spawnAsync("docker", [
-						"service",
-						"update",
-						"--force",
-						"--image",
-						`dokploy/dokploy:${data.latestVersion}`,
-						...(keepImages === undefined
-							? []
-							: ["--env-add", `${DOKPLOY_KEEP_IMAGES_ENV}=${keepImages ?? 0}`]),
-						"dokploy",
+					void spawnAsync("sh", [
+						"-c",
+						getOfficialUpdateCommand(data.latestVersion ?? "", keepImages),
 					]);
 				}
 				await audit(ctx, {

packages/server/src/services/settings.ts

@@ -301,14 +301,16 @@ export const getDokployImageRepositories = () => [
  * Removes Dokploy web server images except the `keep` newest ones. Swarm keeps
  * exited task containers of the dokploy service, and they pin old images, so
  * those are removed first. Images used by running containers are never forced.
+ * `-a` is needed because images swarm pulls by digest have no tag, and
+ * `docker image ls` hides untagged images without it.
  */
 export const getDokployImageCleanupCommand = (keep: number) => {
 	const filters = getDokployImageRepositories()
 		.map((repository) => `--filter ${quoteShellArg(`reference=${repository}`)}`)
 		.join(" ");
 
 	return `
-ids=$(docker image ls -q --no-trunc ${filters} | sort -u)
+ids=$(docker image ls -a -q --no-trunc ${filters} | sort -u)
 if [ -z "$ids" ]; then
 	exit 0
 fi
@@ -404,7 +406,7 @@ export const getDokployImages = async (): Promise<DokployImageInfo[]> => {
 		.map((repository) => `--filter ${quoteShellArg(`reference=${repository}`)}`)
 		.join(" ");
 	const { stdout: idsOutput } = await execAsync(
-		`docker image ls -q --no-trunc ${filters} | sort -u`,
+		`docker image ls -a -q --no-trunc ${filters} | sort -u`,
 	);
 	const ids = idsOutput.split("\n").filter(Boolean);
 	if (ids.length === 0) {
@@ -490,7 +492,10 @@ export const getAgentHitsUpdateCommand = (
 	const keepImagesArg =
 		keepImages === undefined ? "" : getDokployKeepImagesEnvArg(keepImages);
 
+	// Pull while the old container still serves the panel: the service is
+	// stop-first, so a pull inside the update would happen with Dokploy down.
 	return `
+docker pull ${quoteShellArg(getAgentHitsUpdateImage())} || exit 1
 fork_version_env_arg=""
 if docker service inspect dokploy --format '{{range .Spec.TaskTemplate.ContainerSpec.Env}}{{println .}}{{end}}' 2>/dev/null | grep -q '^DOKPLOY_FORK_VERSION='; then
 	fork_version_env_arg="--env-rm DOKPLOY_FORK_VERSION"
@@ -505,6 +510,20 @@ docker service update --force \\
 `;
 };
 
+export const getOfficialUpdateCommand = (
+	version: string,
+	keepImages?: number | null,
+) => {
+	const image = quoteShellArg(`dokploy/dokploy:${version}`);
+	const keepImagesArg =
+		keepImages === undefined ? "" : getDokployKeepImagesEnvArg(keepImages);
+
+	return `
+docker pull ${image} || exit 1
+docker service update --force --image ${image} ${keepImagesArg} dokploy
+`;
+};
+
 /** Returns Dokploy docker service image digest */
 export const getServiceImageDigest = async () => {
 	const { stdout } = await execAsync(