diff --git a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts index a015493d9d1..6bc2ddcdb61 100644 --- a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts +++ b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts @@ -51,6 +51,8 @@ const mocks = vi.hoisted(() => ({ sendDockerCleanupNotifications: vi.fn(), setupGPUSupport: vi.fn(), spawnAsync: vi.fn(), + startServerUpdate: vi.fn(), + getServerUpdateStatus: vi.fn(), startLogCleanup: vi.fn(), stopLogCleanup: vi.fn(), updateLetsEncryptEmail: vi.fn(), @@ -114,6 +116,8 @@ vi.mock("@dokploy/server", () => ({ sendDockerCleanupNotifications: mocks.sendDockerCleanupNotifications, setupGPUSupport: mocks.setupGPUSupport, spawnAsync: mocks.spawnAsync, + startServerUpdate: mocks.startServerUpdate, + getServerUpdateStatus: mocks.getServerUpdateStatus, startLogCleanup: mocks.startLogCleanup, stopLogCleanup: mocks.stopLogCleanup, updateLetsEncryptEmail: mocks.updateLetsEncryptEmail, @@ -427,10 +431,9 @@ describe("settings Docker server boundary", () => { "v0.30.0", undefined, ); - expect(mocks.spawnAsync).toHaveBeenCalledWith("sh", [ - "-c", + expect(mocks.startServerUpdate).toHaveBeenCalledWith( "agenthits update command", - ]); + ); }); it("passes the image keep count to the AgentHits update command", async () => { @@ -461,7 +464,7 @@ describe("settings Docker server boundary", () => { await expect( createCaller().updateServer({ keepImages: 10 }), ).rejects.toThrow(); - expect(mocks.spawnAsync).not.toHaveBeenCalled(); + expect(mocks.startServerUpdate).not.toHaveBeenCalled(); }); it("keeps official updates on the official Dokploy image path", async () => { @@ -479,10 +482,9 @@ describe("settings Docker server boundary", () => { "v0.30.0", undefined, ); - expect(mocks.spawnAsync).toHaveBeenCalledWith("sh", [ - "-c", + expect(mocks.startServerUpdate).toHaveBeenCalledWith( "official update command", - ]); + ); expect(mocks.getAgentHitsUpdateCommand).not.toHaveBeenCalled(); }); @@ -503,6 +505,37 @@ describe("settings Docker server boundary", () => { ); }); + it("does not start an update when none is available", async () => { + mocks.getUpdateData.mockResolvedValue({ + latestVersion: null, + updateAvailable: false, + updateSource: "official", + }); + + await expect(createCaller().updateServer()).resolves.toBe(true); + + expect(mocks.startServerUpdate).not.toHaveBeenCalled(); + }); + + it("returns the web server update status", async () => { + const status = { + phase: "pulling", + startedAt: 1, + pulledAt: null, + finishedAt: null, + error: null, + layersTotal: 3, + layersDownloaded: 1, + layersExtracted: 0, + output: [], + }; + mocks.getServerUpdateStatus.mockReturnValue(status); + + await expect(createCaller().getServerUpdateStatus()).resolves.toEqual( + status, + ); + }); + it("requires api.read before generating the OpenAPI document", async () => { const result = await createCaller().getOpenApiDocument(); diff --git a/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts b/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts index ff87e8d47d5..e11518b6ce5 100644 --- a/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts +++ b/apps/dokploy/__test__/settings/dokploy-image-cleanup.test.ts @@ -1,3 +1,4 @@ +import { execFileSync } from "node:child_process"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; const { execAsync } = vi.hoisted(() => ({ execAsync: vi.fn() })); @@ -20,6 +21,9 @@ const { getDokployImages, getOfficialUpdateCommand, } = await import("@dokploy/server/services/settings"); +const { UPDATE_IMAGE_PULLED_MARKER } = await import( + "@dokploy/server/services/web-server-update" +); describe("Dokploy image cleanup", () => { beforeEach(() => { @@ -216,6 +220,15 @@ describe("Dokploy update commands", () => { expect(command.indexOf(pull)).toBeLessThan( command.indexOf("docker service update"), ); + + const lines = command.split("\n"); + const markerLine = lines.findIndex((line) => line.startsWith("echo ")); + expect(lines[markerLine - 1]).toBe(`${pull} || exit 1`); + expect( + execFileSync("sh", ["-c", lines[markerLine] as string], { + encoding: "utf8", + }).trim(), + ).toBe(UPDATE_IMAGE_PULLED_MARKER); }; it("pulls the AgentHits image while the old container still runs", () => { diff --git a/apps/dokploy/__test__/settings/request-log-authorization.test.ts b/apps/dokploy/__test__/settings/request-log-authorization.test.ts index 85d505e754c..5cf7578a881 100644 --- a/apps/dokploy/__test__/settings/request-log-authorization.test.ts +++ b/apps/dokploy/__test__/settings/request-log-authorization.test.ts @@ -76,6 +76,8 @@ const mocks = vi.hoisted(() => ({ sendDockerCleanupNotifications: vi.fn(), setupGPUSupport: vi.fn(), spawnAsync: vi.fn(), + startServerUpdate: vi.fn(), + getServerUpdateStatus: vi.fn(), startLogCleanup: vi.fn(), stopLogCleanup: vi.fn(), updateLetsEncryptEmail: vi.fn(), @@ -136,6 +138,8 @@ vi.mock("@dokploy/server", () => ({ sendDockerCleanupNotifications: mocks.sendDockerCleanupNotifications, setupGPUSupport: mocks.setupGPUSupport, spawnAsync: mocks.spawnAsync, + startServerUpdate: mocks.startServerUpdate, + getServerUpdateStatus: mocks.getServerUpdateStatus, startLogCleanup: mocks.startLogCleanup, stopLogCleanup: mocks.stopLogCleanup, updateLetsEncryptEmail: mocks.updateLetsEncryptEmail, diff --git a/apps/dokploy/__test__/settings/update-progress.test.ts b/apps/dokploy/__test__/settings/update-progress.test.ts new file mode 100644 index 00000000000..35786b0e818 --- /dev/null +++ b/apps/dokploy/__test__/settings/update-progress.test.ts @@ -0,0 +1,249 @@ +import type { ServerUpdateStatus } from "@dokploy/server/services/web-server-update"; +import { describe, expect, it } from "vitest"; +import { + API_WAIT_MS, + advanceUpdateProgress, + createUpdateProgress, + DOWNLOAD_LIMIT_MS, + failUpdateProgress, + formatElapsed, + getUpdateStepState, + RESTART_LIMIT_MS, + UPDATE_STEPS, + type UpdateProgress, + type UpdateProgressEvent, +} from "@/components/dashboard/settings/web-server/update-progress"; + +const serverStatus = ( + overrides: Partial = {}, +): ServerUpdateStatus => ({ + phase: "pulling", + startedAt: 1_000, + pulledAt: null, + finishedAt: null, + error: null, + layersTotal: 0, + layersDownloaded: 0, + layersExtracted: 0, + output: [], + ...overrides, +}); + +const run = (events: UpdateProgressEvent[], start = createUpdateProgress(0)) => + events.reduce(advanceUpdateProgress, start); + +const stepStates = (progress: UpdateProgress) => + UPDATE_STEPS.map((step) => getUpdateStepState(progress, step)); + +describe("update progress", () => { + it("walks through download, restart and start to done", () => { + const pulling = run([ + { type: "status", status: serverStatus(), at: 2_000 }, + ]); + expect(pulling.phase).toBe("downloading"); + expect(stepStates(pulling)).toEqual([ + "active", + "pending", + "pending", + "pending", + ]); + + const down = run( + [ + { + type: "status", + status: serverStatus({ phase: "updating", pulledAt: 50_000 }), + at: 60_000, + }, + { type: "down", at: 62_000 }, + ], + pulling, + ); + expect(down).toMatchObject({ phase: "restarting", downAt: 62_000 }); + expect(stepStates(down)).toEqual(["done", "active", "pending", "pending"]); + + const up = advanceUpdateProgress(down, { type: "up", at: 80_000 }); + expect(up).toMatchObject({ phase: "starting", upAt: 80_000 }); + + const done = advanceUpdateProgress(up, { + type: "status", + status: serverStatus({ phase: "idle", startedAt: null }), + at: 82_000, + }); + expect(done).toMatchObject({ phase: "done", finishedAt: 82_000 }); + expect(stepStates(done)).toEqual(["done", "done", "done", "done"]); + expect(done.server?.phase).toBe("updating"); + }); + + it("keeps the first down time when the new server flaps", () => { + const progress = run([ + { type: "down", at: 10_000 }, + { type: "up", at: 20_000 }, + { type: "down", at: 22_000 }, + ]); + + expect(progress).toMatchObject({ + phase: "restarting", + downAt: 10_000, + upAt: null, + }); + }); + + it("ignores health answers while the old server is still pulling", () => { + const progress = run([{ type: "up", at: 2_000 }]); + + expect(progress.phase).toBe("downloading"); + }); + + it("fails on the download step when the pull failed", () => { + const progress = run([ + { + type: "status", + status: serverStatus({ + phase: "failed", + error: "Downloading the new image failed.", + output: ["manifest unknown"], + }), + at: 5_000, + }, + ]); + + expect(progress).toMatchObject({ + phase: "failed", + failedStep: "downloading", + error: "Downloading the new image failed.", + }); + expect(progress.server?.output).toEqual(["manifest unknown"]); + expect(stepStates(progress)).toEqual([ + "failed", + "pending", + "pending", + "pending", + ]); + }); + + it("fails on the restart step when the service update failed", () => { + const progress = run([ + { + type: "status", + status: serverStatus({ + phase: "failed", + pulledAt: 3_000, + error: "docker service update failed.", + }), + at: 5_000, + }, + ]); + + expect(progress.failedStep).toBe("restarting"); + expect(stepStates(progress)).toEqual([ + "done", + "failed", + "pending", + "pending", + ]); + }); + + it("fails when the server never started an update job", () => { + const progress = run([ + { + type: "status", + status: serverStatus({ phase: "idle", startedAt: null }), + at: 2_000, + }, + ]); + + expect(progress).toMatchObject({ + phase: "failed", + failedStep: "downloading", + }); + expect(progress.error).toContain("did not start an update"); + }); + + it("treats an idle status after a seen job as a restart that was missed", () => { + const progress = run([ + { type: "status", status: serverStatus(), at: 2_000 }, + { + type: "status", + status: serverStatus({ phase: "idle", startedAt: null }), + at: 400_000, + }, + ]); + + expect(progress.phase).toBe("done"); + }); + + it("fails when the server does not go down within the download limit", () => { + const waiting = run([ + { type: "status", status: serverStatus(), at: 2_000 }, + { type: "tick", at: DOWNLOAD_LIMIT_MS }, + ]); + expect(waiting.phase).toBe("downloading"); + + const progress = advanceUpdateProgress(waiting, { + type: "tick", + at: DOWNLOAD_LIMIT_MS + 1, + }); + expect(progress).toMatchObject({ + phase: "failed", + failedStep: "downloading", + error: "Dokploy did not restart within 20 minutes.", + }); + }); + + it("fails when the server does not come back after going down", () => { + const progress = run([ + { type: "down", at: 10_000 }, + { type: "tick", at: 10_000 + RESTART_LIMIT_MS + 1 }, + ]); + + expect(progress).toMatchObject({ + phase: "failed", + failedStep: "restarting", + }); + expect(stepStates(progress)).toEqual([ + "done", + "failed", + "pending", + "pending", + ]); + }); + + it("finishes when health is fine but the API keeps failing", () => { + const starting = run([ + { type: "down", at: 10_000 }, + { type: "up", at: 30_000 }, + { type: "tick", at: 30_000 + API_WAIT_MS }, + ]); + expect(starting.phase).toBe("starting"); + + expect( + advanceUpdateProgress(starting, { + type: "tick", + at: 30_000 + API_WAIT_MS + 1, + }).phase, + ).toBe("done"); + }); + + it("does not change once finished", () => { + const failed = failUpdateProgress( + createUpdateProgress(0), + "downloading", + "Could not start the update.", + 1_000, + ); + + expect(advanceUpdateProgress(failed, { type: "down", at: 2_000 })).toBe( + failed, + ); + }); +}); + +describe("formatElapsed", () => { + it("formats seconds and minutes", () => { + expect(formatElapsed(-5)).toBe("0 s"); + expect(formatElapsed(19_900)).toBe("19 s"); + expect(formatElapsed(65_000)).toBe("1 min 05 s"); + expect(formatElapsed(20 * 60 * 1000)).toBe("20 min 00 s"); + }); +}); diff --git a/apps/dokploy/__test__/settings/web-server-update-status.test.ts b/apps/dokploy/__test__/settings/web-server-update-status.test.ts new file mode 100644 index 00000000000..43fb810201e --- /dev/null +++ b/apps/dokploy/__test__/settings/web-server-update-status.test.ts @@ -0,0 +1,209 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const { spawnAsync } = vi.hoisted(() => ({ spawnAsync: vi.fn() })); + +vi.mock("@dokploy/server/utils/process/spawnAsync", () => ({ spawnAsync })); + +const { + UPDATE_IMAGE_PULLED_MARKER, + getServerUpdateStatus, + resetServerUpdateStatus, + startServerUpdate, +} = await import("@dokploy/server/services/web-server-update"); + +const startFakeScript = () => { + let onData: (data: string) => void = () => {}; + let resolve: () => void = () => {}; + let reject: (error: unknown) => void = () => {}; + spawnAsync.mockImplementation( + (_command: string, _args: string[], callback: (data: string) => void) => { + onData = callback; + return new Promise((res, rej) => { + resolve = res; + reject = rej; + }); + }, + ); + expect(startServerUpdate("update script")).toBe(true); + return { + write: (data: string) => onData(data), + exit: async (code = 0) => { + if (code === 0) { + resolve(); + } else { + reject(Object.assign(new Error("failed"), { code })); + } + await new Promise((r) => setTimeout(r, 0)); + }, + failToSpawn: async () => { + reject(Object.assign(new Error("spawn sh ENOENT"), { code: "ENOENT" })); + await new Promise((r) => setTimeout(r, 0)); + }, + }; +}; + +describe("web server update status", () => { + beforeEach(() => { + vi.clearAllMocks(); + resetServerUpdateStatus(); + }); + + it("is idle before any update", () => { + expect(getServerUpdateStatus()).toMatchObject({ + phase: "idle", + startedAt: null, + error: null, + output: [], + }); + }); + + it("runs the script through sh and starts in the pulling phase", () => { + startFakeScript(); + + expect(spawnAsync).toHaveBeenCalledWith( + "sh", + ["-c", "update script"], + expect.any(Function), + ); + const status = getServerUpdateStatus(); + expect(status.phase).toBe("pulling"); + expect(status.startedAt).toEqual(expect.any(Number)); + }); + + it("counts docker pull layers, even across split chunks", () => { + const script = startFakeScript(); + + script.write("v1: Pulling from dokploy/dokploy\n"); + script.write( + "aaaaaaaaaaaa: Already exists\nbbbbbbbbbbbb: Pulling fs layer\n", + ); + script.write("cccccccccccc: Pulling fs layer\nbbbbbbbbbbbb: Downlo"); + script.write("ad complete\n"); + + expect(getServerUpdateStatus()).toMatchObject({ + layersTotal: 3, + layersDownloaded: 2, + layersExtracted: 1, + }); + + script.write("bbbbbbbbbbbb: Pull complete\ncccccccccccc: Pull complete\n"); + expect(getServerUpdateStatus()).toMatchObject({ + layersTotal: 3, + layersDownloaded: 3, + layersExtracted: 3, + }); + }); + + it("does not move a finished layer back to downloading", () => { + const script = startFakeScript(); + + script.write("aaaaaaaaaaaa: Pull complete\naaaaaaaaaaaa: Downloading\n"); + + expect(getServerUpdateStatus()).toMatchObject({ + layersTotal: 1, + layersExtracted: 1, + }); + }); + + it("keeps only the last few output lines", () => { + const script = startFakeScript(); + + script.write( + "line 1\r\nline 2\nline 3\n\nline 4\nline 5\nline 6\nline 7\n", + ); + + expect(getServerUpdateStatus().output).toEqual([ + "line 3", + "line 4", + "line 5", + "line 6", + "line 7", + ]); + }); + + it("moves to updating once the pulled marker is printed", () => { + const script = startFakeScript(); + + script.write( + `Status: Downloaded newer image\n${UPDATE_IMAGE_PULLED_MARKER}\n`, + ); + + const status = getServerUpdateStatus(); + expect(status.phase).toBe("updating"); + expect(status.pulledAt).toEqual(expect.any(Number)); + expect(status.output).not.toContain(UPDATE_IMAGE_PULLED_MARKER); + }); + + it("reports a failed pull", async () => { + const script = startFakeScript(); + + script.write("Error response from daemon: manifest unknown"); + await script.exit(1); + + expect(getServerUpdateStatus()).toMatchObject({ + phase: "failed", + error: "Downloading the new image failed.", + output: ["Error response from daemon: manifest unknown"], + finishedAt: expect.any(Number), + }); + }); + + it("reports a failed service update after the pull", async () => { + const script = startFakeScript(); + + script.write(`${UPDATE_IMAGE_PULLED_MARKER}\n`); + await script.exit(1); + + expect(getServerUpdateStatus()).toMatchObject({ + phase: "failed", + error: "docker service update failed.", + }); + }); + + it("reports a script that could not be started", async () => { + const script = startFakeScript(); + + await script.failToSpawn(); + + expect(getServerUpdateStatus()).toMatchObject({ + phase: "failed", + error: "Could not start the update script.", + }); + }); + + it("is done when the script exits cleanly", async () => { + const script = startFakeScript(); + + script.write(`${UPDATE_IMAGE_PULLED_MARKER}\n`); + await script.exit(0); + + expect(getServerUpdateStatus()).toMatchObject({ + phase: "done", + error: null, + }); + }); + + it("does not start a second update while one is running", async () => { + const script = startFakeScript(); + + expect(startServerUpdate("another script")).toBe(false); + expect(spawnAsync).toHaveBeenCalledTimes(1); + + await script.exit(1); + expect(startServerUpdate("retry script")).toBe(true); + expect(getServerUpdateStatus()).toMatchObject({ + phase: "pulling", + error: null, + output: [], + }); + }); + + it("returns a copy of the status", () => { + const script = startFakeScript(); + script.write("line\n"); + + getServerUpdateStatus().output.push("changed"); + + expect(getServerUpdateStatus().output).toEqual(["line"]); + }); +}); diff --git a/apps/dokploy/components/dashboard/settings/web-server/update-progress.ts b/apps/dokploy/components/dashboard/settings/web-server/update-progress.ts new file mode 100644 index 00000000000..24738c341f3 --- /dev/null +++ b/apps/dokploy/components/dashboard/settings/web-server/update-progress.ts @@ -0,0 +1,205 @@ +import type { ServerUpdateStatus } from "@dokploy/server/services/web-server-update"; + +export const UPDATE_POLL_MS = 2000; +// Pulling a 4+ GB image on a slow link can take several minutes. +export const DOWNLOAD_LIMIT_MS = 20 * 60 * 1000; +export const RESTART_LIMIT_MS = 10 * 60 * 1000; +// /api/health does not touch the database or the session, so a healthy +// server can still fail API calls for a while (migrations, cold caches). Past +// this, reloading shows the real state better than waiting longer. +export const API_WAIT_MS = 60 * 1000; + +export type UpdateStep = "downloading" | "restarting" | "starting"; +export type UpdateProgressPhase = UpdateStep | "done" | "failed"; +export type UpdateStepState = "done" | "active" | "pending" | "failed"; + +export interface UpdateProgress { + phase: UpdateProgressPhase; + failedStep: UpdateStep | null; + error: string | null; + startedAt: number; + /** When the old server stopped answering. */ + downAt: number | null; + /** When the new server answered /api/health. */ + upAt: number | null; + finishedAt: number | null; + /** Last update job status reported by the old server. */ + server: ServerUpdateStatus | null; +} + +export type UpdateProgressEvent = + | { type: "status"; status: ServerUpdateStatus; at: number } + | { type: "down"; at: number } + | { type: "up"; at: number } + | { type: "tick"; at: number }; + +export const createUpdateProgress = (at: number): UpdateProgress => ({ + phase: "downloading", + failedStep: null, + error: null, + startedAt: at, + downAt: null, + upAt: null, + finishedAt: null, + server: null, +}); + +export const isUpdateFinished = (progress: UpdateProgress) => + progress.phase === "done" || progress.phase === "failed"; + +export const failUpdateProgress = ( + progress: UpdateProgress, + step: UpdateStep, + error: string, + at: number, +): UpdateProgress => ({ + ...progress, + phase: "failed", + failedStep: step, + error, + finishedAt: at, +}); + +const finishUpdateProgress = ( + progress: UpdateProgress, + at: number, +): UpdateProgress => ({ ...progress, phase: "done", finishedAt: at }); + +const handleStatus = ( + progress: UpdateProgress, + status: ServerUpdateStatus, + at: number, +): UpdateProgress => { + if (progress.phase !== "downloading") { + // The API answers again, so the new version is up. + return finishUpdateProgress({ ...progress, upAt: progress.upAt ?? at }, at); + } + + if (status.phase === "failed") { + return failUpdateProgress( + { ...progress, server: status }, + status.pulledAt ? "restarting" : "downloading", + status.error ?? "The update failed.", + at, + ); + } + + if (status.phase === "idle") { + // A fresh process reports "idle": the restart happened between two polls, + // e.g. while the browser throttled timers in a background tab. + if (progress.server) { + return finishUpdateProgress(progress, at); + } + return failUpdateProgress( + progress, + "downloading", + "Dokploy did not start an update. The server may already run the latest version.", + at, + ); + } + + return { ...progress, server: status }; +}; + +export const advanceUpdateProgress = ( + progress: UpdateProgress, + event: UpdateProgressEvent, +): UpdateProgress => { + if (isUpdateFinished(progress)) { + return progress; + } + + switch (event.type) { + case "status": + return handleStatus(progress, event.status, event.at); + case "down": + if (progress.phase === "restarting") { + return progress; + } + return { + ...progress, + phase: "restarting", + downAt: progress.downAt ?? event.at, + upAt: null, + }; + case "up": + if (progress.phase !== "restarting") { + return progress; + } + return { ...progress, phase: "starting", upAt: event.at }; + case "tick": + if ( + progress.phase === "downloading" && + event.at - progress.startedAt > DOWNLOAD_LIMIT_MS + ) { + return failUpdateProgress( + progress, + "downloading", + "Dokploy did not restart within 20 minutes.", + event.at, + ); + } + if ( + progress.downAt !== null && + event.at - progress.downAt > RESTART_LIMIT_MS + ) { + return failUpdateProgress( + progress, + progress.phase as UpdateStep, + "Dokploy did not come back within 10 minutes after it stopped.", + event.at, + ); + } + if ( + progress.phase === "starting" && + progress.upAt !== null && + event.at - progress.upAt > API_WAIT_MS + ) { + return finishUpdateProgress(progress, event.at); + } + return progress; + } +}; + +export const UPDATE_STEPS = [ + "downloading", + "restarting", + "starting", + "done", +] as const; + +export type UpdatePanelStep = (typeof UPDATE_STEPS)[number]; + +export const getUpdateStepState = ( + progress: UpdateProgress, + step: UpdatePanelStep, +): UpdateStepState => { + const index = UPDATE_STEPS.indexOf(step); + if (progress.phase === "done") { + return "done"; + } + if (progress.phase === "failed") { + const failedIndex = UPDATE_STEPS.indexOf( + progress.failedStep ?? "downloading", + ); + if (index < failedIndex) { + return "done"; + } + return index === failedIndex ? "failed" : "pending"; + } + const activeIndex = UPDATE_STEPS.indexOf(progress.phase); + if (index < activeIndex) { + return "done"; + } + return index === activeIndex ? "active" : "pending"; +}; + +export const formatElapsed = (ms: number) => { + const totalSeconds = Math.max(0, Math.floor(ms / 1000)); + if (totalSeconds < 60) { + return `${totalSeconds} s`; + } + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return `${minutes} min ${String(seconds).padStart(2, "0")} s`; +}; diff --git a/apps/dokploy/components/dashboard/settings/web-server/update-status-panel.tsx b/apps/dokploy/components/dashboard/settings/web-server/update-status-panel.tsx new file mode 100644 index 00000000000..dc42d5f458b --- /dev/null +++ b/apps/dokploy/components/dashboard/settings/web-server/update-status-panel.tsx @@ -0,0 +1,217 @@ +import { + AlertTriangle, + CheckCircle2, + Circle, + Download, + Loader2, + type LucideIcon, + PartyPopper, + Power, + Rocket, + XCircle, +} from "lucide-react"; +import type { ReactNode } from "react"; +import { cn } from "@/lib/utils"; +import { + formatElapsed, + getUpdateStepState, + type UpdatePanelStep, + type UpdateProgress, + type UpdateStepState, +} from "./update-progress"; + +export interface UpdateStatusPanelProps { + progress: UpdateProgress; + /** Current time in ms; the parent ticks it so the counters move. */ + now: number; +} + +const STEP_LABELS: Record< + UpdatePanelStep, + { title: string; icon: LucideIcon } +> = { + downloading: { title: "Downloading the new image", icon: Download }, + restarting: { title: "Restarting Dokploy", icon: Power }, + starting: { title: "Starting the new version", icon: Rocket }, + done: { title: "Done", icon: PartyPopper }, +}; + +const StateIcon = ({ state }: { state: UpdateStepState }) => { + switch (state) { + case "done": + return ; + case "active": + return ; + case "failed": + return ; + default: + return ; + } +}; + +const getDownloadDetails = ( + progress: UpdateProgress, + state: UpdateStepState, + now: number, +): ReactNode[] => { + const { server } = progress; + if (state === "done") { + // Server timestamps are compared only with each other: the browser + // clock may be off. + if (server?.pulledAt && server.startedAt) { + return [ + `Finished in ${formatElapsed(server.pulledAt - server.startedAt)}`, + ]; + } + const end = progress.downAt ?? progress.finishedAt; + return end + ? [`Finished in ${formatElapsed(end - progress.startedAt)}`] + : []; + } + + const end = state === "failed" ? (progress.finishedAt ?? now) : now; + const details: ReactNode[] = [ + `Elapsed ${formatElapsed(end - progress.startedAt)}`, + ]; + if (server?.phase === "pulling" && server.layersTotal > 0) { + details.push( + `${server.layersDownloaded} of ${server.layersTotal} layers downloaded, ${server.layersExtracted} extracted`, + ); + } + if (server?.phase === "updating" || server?.phase === "done") { + details.push("Image downloaded, waiting for Dokploy to stop"); + } + if (state === "active") { + details.push("The panel keeps working while the image downloads."); + } + return details; +}; + +const getStepDetails = ( + progress: UpdateProgress, + step: UpdatePanelStep, + state: UpdateStepState, + now: number, +): ReactNode[] => { + if (state === "pending") { + return []; + } + + switch (step) { + case "downloading": + return getDownloadDetails(progress, state, now); + case "restarting": { + if (progress.downAt === null) { + return []; + } + if (state === "done") { + const end = progress.upAt ?? progress.finishedAt; + return end + ? [`Unavailable for ${formatElapsed(end - progress.downAt)}`] + : []; + } + const end = state === "failed" ? (progress.finishedAt ?? now) : now; + return [ + + Unavailable for {formatElapsed(end - progress.downAt)} + , + "The panel is unavailable for about 20 seconds.", + ]; + } + case "starting": { + if (progress.upAt === null) { + return []; + } + if (state === "done") { + return progress.finishedAt + ? [`Ready in ${formatElapsed(progress.finishedAt - progress.upAt)}`] + : []; + } + return [ + "Dokploy answers again, waiting for the API", + `Elapsed ${formatElapsed(now - progress.upAt)}`, + ]; + } + case "done": + return progress.finishedAt + ? [ + `Updated in ${formatElapsed(progress.finishedAt - progress.startedAt)}`, + "Reloading the page...", + ] + : []; + } +}; + +export const UpdateStatusPanel = ({ + progress, + now, +}: UpdateStatusPanelProps) => { + const totalEnd = progress.finishedAt ?? now; + const output = progress.server?.output ?? []; + + return ( +
+
    + {(Object.keys(STEP_LABELS) as UpdatePanelStep[]).map((step) => { + const state = getUpdateStepState(progress, step); + const { title, icon: Icon } = STEP_LABELS[step]; + const details = getStepDetails(progress, step, state, now); + return ( +
  1. + +
    + + + {title} + + {details.map((detail, index) => ( + + {detail} + + ))} +
    +
    2. + ); + })} +
+ + {progress.phase === "failed" && ( +
+
+ +
+ + {progress.error ?? "The update failed."} + + + Check the Dokploy service logs, then close this dialog and try + again. + +
+
+ {output.length > 0 && ( + 
+							{output.join("\n")}
+
+ )} +
+ )} + + + Total time {formatElapsed(totalEnd - progress.startedAt)} + +
+ ); +}; diff --git a/apps/dokploy/components/dashboard/settings/web-server/update-webserver.tsx b/apps/dokploy/components/dashboard/settings/web-server/update-webserver.tsx index 980bbde976b..8411836249d 100644 --- a/apps/dokploy/components/dashboard/settings/web-server/update-webserver.tsx +++ b/apps/dokploy/components/dashboard/settings/web-server/update-webserver.tsx @@ -6,7 +6,7 @@ import { RefreshCw, XCircle, } from "lucide-react"; -import { useState } from "react"; +import { useEffect, useRef, useState } from "react"; import { toast } from "sonner"; import { AlertDialog, @@ -22,6 +22,16 @@ import { import { Button } from "@/components/ui/button"; import { cn } from "@/lib/utils"; import { api } from "@/utils/api"; +import { + advanceUpdateProgress, + createUpdateProgress, + failUpdateProgress, + isUpdateFinished, + UPDATE_POLL_MS, + type UpdateProgress, + type UpdateProgressEvent, +} from "./update-progress"; +import { UpdateStatusPanel } from "./update-status-panel"; type ServiceStatus = { status: "healthy" | "unhealthy"; @@ -33,10 +43,7 @@ type HealthResult = { traefik: ServiceStatus; }; -const HEALTH_POLL_MS = 2000; const HEALTH_TIMEOUT_MS = 5000; -// Pulling a 4+ GB image on a slow link can take several minutes. -const RESTART_WAIT_MS = 20 * 60 * 1000; type ModalState = "idle" | "checking" | "results" | "updating"; @@ -70,7 +77,11 @@ export const UpdateWebServer = ({ const [modalState, setModalState] = useState("idle"); const [open, setOpen] = useState(false); const [healthResult, setHealthResult] = useState(null); + const [progress, setProgress] = useState(null); + const [now, setNow] = useState(() => Date.now()); + const unmountedRef = useRef(false); + const utils = api.useUtils(); const { mutateAsync: updateServer } = api.settings.updateServer.useMutation(); const { refetch: checkHealth } = api.settings.checkInfrastructureHealth.useQuery(undefined, { @@ -112,47 +123,108 @@ export const UpdateWebServer = ({ const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); - // The old server keeps answering while the new image is pulled, so wait - // until it goes down once before treating a healthy answer as the new one. - const waitForRestart = async () => { - const startedAt = Date.now(); - while (await isServerUp()) { - if (Date.now() - startedAt > RESTART_WAIT_MS) { - throw new Error("The server did not restart"); - } - await sleep(HEALTH_POLL_MS); + useEffect(() => { + unmountedRef.current = false; + return () => { + unmountedRef.current = true; + }; + }, []); + + const isTracking = + modalState === "updating" && + progress !== null && + !isUpdateFinished(progress); + + useEffect(() => { + if (!isTracking) { + return; } - while (!(await isServerUp())) { - await sleep(HEALTH_POLL_MS); + const interval = setInterval(() => setNow(Date.now()), 1000); + return () => clearInterval(interval); + }, [isTracking]); + + // The old server answers the status query while it pulls the image; while + // Dokploy restarts only /api/health tells when it is back. + const probe = async ( + current: UpdateProgress, + ): Promise => { + if (current.phase === "restarting") { + return (await isServerUp()) + ? { type: "up", at: Date.now() } + : { type: "tick", at: Date.now() }; + } + try { + const status = await utils.client.settings.getServerUpdateStatus.query( + undefined, + { signal: AbortSignal.timeout(HEALTH_TIMEOUT_MS) }, + ); + return { type: "status", status, at: Date.now() }; + } catch { + return (await isServerUp()) + ? { type: "tick", at: Date.now() } + : { type: "down", at: Date.now() }; } }; - const handleConfirm = async () => { + const trackUpdate = async () => { + let current = createUpdateProgress(Date.now()); + setNow(current.startedAt); + setProgress(current); + try { - setModalState("updating"); await updateServer(keepImages === undefined ? undefined : { keepImages }); - await waitForRestart(); + } catch (error) { + console.error("Error updating server:", error); + current = failUpdateProgress( + current, + "downloading", + error instanceof Error && error.message + ? error.message + : "Could not start the update.", + Date.now(), + ); + } + while (!isUpdateFinished(current) && !unmountedRef.current) { + await sleep(UPDATE_POLL_MS); + const event = await probe(current); + current = advanceUpdateProgress(advanceUpdateProgress(current, event), { + type: "tick", + at: Date.now(), + }); + setProgress(current); + } + setProgress(current); + setNow(Date.now()); + return current; + }; + + const handleConfirm = async () => { + setModalState("updating"); + const result = await trackUpdate(); + + if (result.phase === "done") { toast.success( "The server has been updated. The page will be reloaded to reflect the changes...", ); setTimeout(() => { window.location.reload(); }, 2000); - } catch (error) { - setModalState("results"); - console.error("Error updating server:", error); + } else if (result.phase === "failed") { toast.error( "The server did not restart with the new version. Check the Dokploy service logs and try again.", ); } }; + const updateFailed = progress?.phase === "failed"; + const handleClose = () => { - if (modalState !== "updating") { + if (modalState !== "updating" || updateFailed) { setOpen(false); setModalState("idle"); setHealthResult(null); + setProgress(null); } }; @@ -179,7 +251,12 @@ export const UpdateWebServer = ({ {modalState === "checking" && "Verifying Services..."} {modalState === "results" && (allHealthy ? "Ready to Update" : "Service Issues Detected")} - {modalState === "updating" && "Server update in progress"} + {modalState === "updating" && + (updateFailed + ? "Server update failed" + : progress?.phase === "done" + ? "Server updated" + : "Server update in progress")}
@@ -187,10 +264,9 @@ export const UpdateWebServer = ({ This will update the web server to the selected latest build. AgentHits fork installs update from the AgentHits GHCR image. - The new image is downloaded first, so the panel is - unavailable only for about 20 seconds while Dokploy - restarts. The page will be reloaded once the update is - finished. + The new image is downloaded first, so the panel is unavailable + only for about 20 seconds while Dokploy restarts. The page + will be reloaded once the update is finished.

We recommend verifying that all services are running before @@ -246,12 +322,8 @@ export const UpdateWebServer = ({
)} - {modalState === "updating" && ( - - - Downloading the new image. The panel keeps working meanwhile; - it is unavailable for about 20 seconds while Dokploy restarts. - + {modalState === "updating" && progress && ( + )} @@ -268,6 +340,11 @@ export const UpdateWebServer = ({ )} + {modalState === "updating" && updateFailed && ( + + Close + + )} {modalState === "results" && ( Cancel diff --git a/apps/dokploy/server/api/routers/settings.ts b/apps/dokploy/server/api/routers/settings.ts index e5af5dd28fe..e3f21147953 100644 --- a/apps/dokploy/server/api/routers/settings.ts +++ b/apps/dokploy/server/api/routers/settings.ts @@ -25,6 +25,7 @@ import { getDokployVersionData, getLogCleanupStatus, getOfficialUpdateCommand, + getServerUpdateStatus, getUpdateData, getWebServerSettings, IS_CLOUD, @@ -45,8 +46,8 @@ import { resolveDockerDiskUsageDetailLimit, sendDockerCleanupNotifications, setupGPUSupport, - spawnAsync, startLogCleanup, + startServerUpdate, stopLogCleanup, updateLetsEncryptEmail, updateServerById, @@ -658,22 +659,16 @@ export const settingsRouter = createTRPCRouter({ const keepImages = input?.keepImages; const data = await getUpdateData(packageInfo.version); if (data.updateAvailable) { - if (data.updateSource === "agenthits") { - void spawnAsync("sh", [ - "-c", - getAgentHitsUpdateCommand( - packageInfo.version, - data.latestVersion, - data.latestOfficialVersion, - keepImages, - ), - ]); - } else { - void spawnAsync("sh", [ - "-c", - getOfficialUpdateCommand(data.latestVersion ?? "", keepImages), - ]); - } + startServerUpdate( + data.updateSource === "agenthits" + ? getAgentHitsUpdateCommand( + packageInfo.version, + data.latestVersion, + data.latestOfficialVersion, + keepImages, + ) + : getOfficialUpdateCommand(data.latestVersion ?? "", keepImages), + ); await audit(ctx, { action: "update", resourceType: "settings", @@ -684,6 +679,10 @@ export const settingsRouter = createTRPCRouter({ return true; }), + getServerUpdateStatus: adminProcedure.query(() => { + return getServerUpdateStatus(); + }), + getDokployVersion: protectedProcedure.query(() => { return packageInfo.version; }), diff --git a/apps/dokploy/server/api/utils/procedure-access.ts b/apps/dokploy/server/api/utils/procedure-access.ts index 8a945508333..320a298f03d 100644 --- a/apps/dokploy/server/api/utils/procedure-access.ts +++ b/apps/dokploy/server/api/utils/procedure-access.ts @@ -599,6 +599,7 @@ export const PROCEDURE_ACCESS: Record = { "settings.getLogCleanupStatus": "read", "settings.getOpenApiDocument": "read", "settings.getReleaseTag": "read", + "settings.getServerUpdateStatus": "read", "settings.getTraefikPorts": "read", "settings.getUpdateData": "write", "settings.getWebServerSettings": "read", diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 25645ecc55d..55470a0f2bb 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -61,6 +61,7 @@ export * from "./services/user"; export * from "./services/vault-provider"; export * from "./services/volume-backups"; export * from "./services/web-server-settings"; +export * from "./services/web-server-update"; export * from "./setup/config-paths"; export * from "./setup/forward-auth-setup"; export * from "./setup/monitoring-setup"; diff --git a/packages/server/src/services/settings.ts b/packages/server/src/services/settings.ts index 20541750d4b..ea4029a792b 100644 --- a/packages/server/src/services/settings.ts +++ b/packages/server/src/services/settings.ts @@ -15,6 +15,7 @@ import { initializeTraefikService, type TraefikOptions, } from "../setup/traefik-setup"; +import { UPDATE_IMAGE_PULLED_MARKER } from "./web-server-update"; export interface IUpdateData { latestVersion: string | null; updateAvailable: boolean; @@ -496,6 +497,7 @@ export const getAgentHitsUpdateCommand = ( // stop-first, so a pull inside the update would happen with Dokploy down. return ` docker pull ${quoteShellArg(getAgentHitsUpdateImage())} || exit 1 +echo ${quoteShellArg(UPDATE_IMAGE_PULLED_MARKER)} fork_version_env_arg="" if docker service inspect dokploy --format '{{range .Spec.TaskTemplate.ContainerSpec.Env}}{{println .}}{{end}}' 2>/dev/null | grep -q '^DOKPLOY_FORK_VERSION='; then fork_version_env_arg="--env-rm DOKPLOY_FORK_VERSION" @@ -520,6 +522,7 @@ export const getOfficialUpdateCommand = ( return ` docker pull ${image} || exit 1 +echo ${quoteShellArg(UPDATE_IMAGE_PULLED_MARKER)} docker service update --force --image ${image} ${keepImagesArg} dokploy `; }; diff --git a/packages/server/src/services/web-server-update.ts b/packages/server/src/services/web-server-update.ts new file mode 100644 index 00000000000..4d82028348d --- /dev/null +++ b/packages/server/src/services/web-server-update.ts @@ -0,0 +1,158 @@ +import { spawnAsync } from "../utils/process/spawnAsync"; + +/** Printed by the update scripts once `docker pull` succeeded. */ +export const UPDATE_IMAGE_PULLED_MARKER = "[dokploy-update] image pulled"; + +export type ServerUpdateJobPhase = + | "idle" + | "pulling" + | "updating" + | "done" + | "failed"; + +export interface ServerUpdateStatus { + phase: ServerUpdateJobPhase; + startedAt: number | null; + pulledAt: number | null; + finishedAt: number | null; + error: string | null; + layersTotal: number; + layersDownloaded: number; + layersExtracted: number; + /** Last lines of the update script output: image refs and layer ids only. */ + output: string[]; +} + +const OUTPUT_LINES = 5; +const OUTPUT_LINE_MAX_LENGTH = 300; + +const createIdleStatus = (): ServerUpdateStatus => ({ + phase: "idle", + startedAt: null, + pulledAt: null, + finishedAt: null, + error: null, + layersTotal: 0, + layersDownloaded: 0, + layersExtracted: 0, + output: [], +}); + +// Lives only in this process: the update restarts Dokploy, and the new +// process starting from "idle" is how the UI knows the restart happened. +let status = createIdleStatus(); +let layers = new Map(); +let pendingOutput = ""; + +export const getServerUpdateStatus = (): ServerUpdateStatus => ({ + ...status, + output: [...status.output], +}); + +export const resetServerUpdateStatus = () => { + status = createIdleStatus(); + layers = new Map(); + pendingOutput = ""; +}; + +export const isServerUpdateRunning = () => + status.phase === "pulling" || status.phase === "updating"; + +const LAYER_LINE = /^([0-9a-f]{12,64}): (.+)$/; +const DOWNLOADED_LAYER_STATES = new Set([ + "Download complete", + "Pull complete", + "Already exists", +]); +const EXTRACTED_LAYER_STATES = new Set(["Pull complete", "Already exists"]); + +const countLayers = (states: Set) => + [...layers.values()].filter((state) => states.has(state)).length; + +const handleOutputLine = (rawLine: string) => { + const line = rawLine.trim(); + if (!line) { + return; + } + + if (line === UPDATE_IMAGE_PULLED_MARKER) { + if (status.phase === "pulling") { + status.phase = "updating"; + status.pulledAt = Date.now(); + } + return; + } + + const layer = line.match(LAYER_LINE); + if (layer && status.phase === "pulling") { + const [, id, state] = layer as unknown as [string, string, string]; + // Docker repeats a layer's earlier states while it is retried, so a + // layer that already finished must not go back to "downloading". + if (!EXTRACTED_LAYER_STATES.has(layers.get(id) ?? "")) { + layers.set(id, state); + } + status.layersTotal = layers.size; + status.layersDownloaded = countLayers(DOWNLOADED_LAYER_STATES); + status.layersExtracted = countLayers(EXTRACTED_LAYER_STATES); + } + + status.output = [ + ...status.output, + line.slice(0, OUTPUT_LINE_MAX_LENGTH), + ].slice(-OUTPUT_LINES); +}; + +const handleOutput = (chunk: string) => { + // Chunks split lines anywhere; docker also separates some updates with \r. + const parts = `${pendingOutput}${chunk}`.split(/\r?\n|\r/); + pendingOutput = parts.pop() ?? ""; + for (const part of parts) { + handleOutputLine(part); + } +}; + +const finish = (error: string | null) => { + if (pendingOutput) { + handleOutputLine(pendingOutput); + pendingOutput = ""; + } + status.finishedAt = Date.now(); + if (error) { + status.phase = "failed"; + status.error = error; + } else { + status.phase = "done"; + } +}; + +/** + * Runs an update script from `getAgentHitsUpdateCommand` or + * `getOfficialUpdateCommand` in the background and tracks it in + * `getServerUpdateStatus`. Returns false when an update is already running. + */ +export const startServerUpdate = (command: string) => { + if (isServerUpdateRunning()) { + return false; + } + + resetServerUpdateStatus(); + status.phase = "pulling"; + status.startedAt = Date.now(); + + spawnAsync("sh", ["-c", command], handleOutput).then( + () => finish(null), + (error: unknown) => { + const spawnFailed = + typeof (error as { code?: unknown })?.code !== "number"; + if (spawnFailed) { + finish("Could not start the update script."); + } else if (status.phase === "pulling") { + finish("Downloading the new image failed."); + } else { + finish("docker service update failed."); + } + }, + ); + + return true; +};