From 77f9732ad4c6f91425b6bf3c751f590f4e689ed6 Mon Sep 17 00:00:00 2001 From: merge-check <140916359+agentHits@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:13:12 -0400 Subject: [PATCH] feat(monitoring): disk usage history by category, with image and build cache cleanup on the Monitoring page The Disk Space card on the Monitoring page showed one number and a chart of the last few minutes, recorded only while the page was open. It could not answer what filled the disk over a day or a week, and its number did not match the Docker disk usage card under it. History. The panel now samples the disk in the background: the used space every minute, Docker's image, container and build cache sizes every five minutes, and once an hour the parts that need a walk over files (volumes, container logs, Dokploy's own data, swap files). When the used space moves by 0.5% of the disk (at least 256 MiB) since the last saved sample, the next sample is taken on that minute, so a short spike is recorded with its breakdown. A probe never takes more than 2% of the time: after a slow one the sampler waits fifty times its duration. Samples are kept for 30 days as JSON lines under the monitoring directory; "Clean Monitoring" removes them with the rest. Docker is asked through the Engine API, so no docker CLI process holds up the cleanup commands that wait for Docker to be idle. Container logs live under Docker's root directory, which the panel's container does not see; a throwaway busybox container with that directory mounted read-only measures them. When it cannot run, the logs stay in "System and other". Card. It shows the used space split into images, build cache, volumes, container logs, container files, Dokploy data, swap files and "System and other", and the rows add up to the used space. A stacked bar chart covers 1, 3, 7 or 30 days; each bar is the fullest moment of its interval. A table gives the change of every category over the period, and a list names the images, volumes and containers that grew or shrank the most. Sizes are in decimal units, as Docker prints them; the old card counted in GiB and labelled the result GB, which is one reason the two cards disagreed. Build cache is counted without the records that images share, which Docker's total counts twice. Cleanup. The block that removes old Dokploy images and, on request, the build cache was reachable only from the update dialog. It is now also in the Disk Space card, where it lists the builds it removes. After a cleanup the disk is sampled at once, so the card shows the freed space. The Docker disk usage chart takes the colours of the same categories in the new card. Co-Authored-By: Claude Opus 5.5 --- .../__test__/monitoring/disk-history.test.ts | 448 ++++++++++++++++ .../monitoring/disk-usage-format.test.ts | 143 +++++ .../__test__/monitoring/disk-usage.test.ts | 331 ++++++++++++ .../settings/docker-server-boundary.test.ts | 21 + .../request-log-authorization.test.ts | 1 + .../settings/update-disk-space.test.ts | 7 + .../disk-usage/disk-usage-breakdown.tsx | 162 ++++++ .../container/disk-usage/disk-usage-card.tsx | 30 ++ .../container/disk-usage/disk-usage-format.ts | 186 +++++++ .../disk-usage/disk-usage-history-chart.tsx | 178 +++++++ .../container/disk-usage/disk-usage-panel.tsx | 128 +++++ .../free/container/docker-disk-chart.tsx | 83 --- .../container/docker-disk-usage-chart.tsx | 9 +- .../show-free-container-monitoring.tsx | 18 +- .../settings/web-server/free-disk-space.tsx | 78 ++- apps/dokploy/server/api/routers/settings.ts | 16 + .../server/api/utils/procedure-access.ts | 1 + apps/dokploy/server/server.ts | 2 + apps/dokploy/styles/globals.css | 16 + docs/agenthits-fork.md | 8 +- docs/server-maintenance.md | 18 + packages/server/src/index.ts | 2 + .../server/src/monitoring/disk-history.ts | 498 ++++++++++++++++++ packages/server/src/monitoring/disk-usage.ts | 381 ++++++++++++++ packages/server/src/services/settings.ts | 3 + 25 files changed, 2645 insertions(+), 123 deletions(-) create mode 100644 apps/dokploy/__test__/monitoring/disk-history.test.ts create mode 100644 apps/dokploy/__test__/monitoring/disk-usage-format.test.ts create mode 100644 apps/dokploy/__test__/monitoring/disk-usage.test.ts create mode 100644 apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-breakdown.tsx create mode 100644 apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-card.tsx create mode 100644 apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-format.ts create mode 100644 apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-history-chart.tsx create mode 100644 apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-panel.tsx delete mode 100644 apps/dokploy/components/dashboard/monitoring/free/container/docker-disk-chart.tsx create mode 100644 packages/server/src/monitoring/disk-history.ts create mode 100644 packages/server/src/monitoring/disk-usage.ts diff --git a/apps/dokploy/__test__/monitoring/disk-history.test.ts b/apps/dokploy/__test__/monitoring/disk-history.test.ts new file mode 100644 index 00000000000..b854eea59dc --- /dev/null +++ b/apps/dokploy/__test__/monitoring/disk-history.test.ts @@ -0,0 +1,448 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { DiskUsageProbe } from "@dokploy/server/monitoring/disk-usage"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + monitoringPath: "", + getRootDiskStat: vi.fn(), + probeDockerUsage: vi.fn(), + probeSlowUsage: vi.fn(), +})); + +vi.mock("@dokploy/server/constants", () => ({ + docker: {}, + paths: () => ({ MONITORING_PATH: mocks.monitoringPath }), +})); + +vi.mock("@dokploy/server/monitoring/disk-usage", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@dokploy/server/monitoring/disk-usage") + >()), + getRootDiskStat: mocks.getRootDiskStat, + probeDockerUsage: mocks.probeDockerUsage, + probeSlowUsage: mocks.probeSlowUsage, +})); + +type DiskHistoryModule = + typeof import("@dokploy/server/monitoring/disk-history"); + +const GB = 1e9; +const MB = 1e6; +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; +const TOTAL = 50 * GB; +const START = Date.UTC(2026, 9, 13, 12, 0, 0); + +const diskUsed = (usedBytes: number) => { + mocks.getRootDiskStat.mockResolvedValue({ + totalBytes: TOTAL, + usedBytes, + availableBytes: TOTAL - usedBytes, + }); +}; + +const dockerProbe = ( + images: number, + buildCache: number, + items: DiskUsageProbe["items"] = {}, +): DiskUsageProbe => ({ + parts: { images, buildCache, containers: 0 }, + items, + itemKinds: ["image", "container"], +}); + +const slowProbe = ( + volumes: number, + items: DiskUsageProbe["items"] = {}, +): DiskUsageProbe => ({ + parts: { volumes, containerLogs: 0, dokployData: 0, swap: 2 * GB }, + items, + itemKinds: ["volume", "containerLog", "dokployData"], +}); + +const readLines = async (file: string) => + ( + await readFile( + path.join(mocks.monitoringPath, "disk-history", file), + "utf-8", + ) + ) + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); + +describe("disk history sampler", () => { + let history: DiskHistoryModule; + + beforeEach(async () => { + mocks.monitoringPath = await mkdtemp(path.join(tmpdir(), "disk-history-")); + vi.clearAllMocks(); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(START); + // The sampler keeps its state on globalThis so that bundled copies share it. + delete (globalThis as { diskHistorySampler?: unknown }).diskHistorySampler; + vi.resetModules(); + history = await import("@dokploy/server/monitoring/disk-history"); + + diskUsed(30 * GB); + mocks.probeDockerUsage.mockResolvedValue(dockerProbe(10 * GB, 5 * GB)); + mocks.probeSlowUsage.mockResolvedValue(slowProbe(8 * GB)); + }); + + afterEach(async () => { + vi.useRealTimers(); + await rm(mocks.monitoringPath, { recursive: true, force: true }); + }); + + it("saves a sample with every part and the item sizes on the first run", async () => { + mocks.probeDockerUsage.mockResolvedValue( + dockerProbe(10 * GB, 5 * GB, { "image:web": 4 * GB }), + ); + mocks.probeSlowUsage.mockResolvedValue( + slowProbe(8 * GB, { "volume:pg-data": 8 * GB }), + ); + + await history.refreshDiskUsage(); + + expect(await readLines("samples.jsonl")).toEqual([ + { + at: START, + totalBytes: TOTAL, + usedBytes: 30 * GB, + images: 10 * GB, + buildCache: 5 * GB, + containers: 0, + volumes: 8 * GB, + containerLogs: 0, + dokployData: 0, + swap: 2 * GB, + }, + ]); + expect(await readLines("items.jsonl")).toEqual([ + { + at: START, + items: { "image:web": 4 * GB, "volume:pg-data": 8 * GB }, + }, + ]); + }); + + it("asks Docker every five minutes and walks the disk every hour", async () => { + await history.refreshDiskUsage(); + + for (let minute = 1; minute <= 60; minute++) { + vi.setSystemTime(START + minute * MINUTE); + await history.refreshDiskUsage(); + } + + expect(mocks.getRootDiskStat).toHaveBeenCalledTimes(61); + expect(mocks.probeDockerUsage).toHaveBeenCalledTimes(13); + expect(mocks.probeSlowUsage).toHaveBeenCalledTimes(2); + expect(await readLines("samples.jsonl")).toHaveLength(13); + expect(await readLines("items.jsonl")).toHaveLength(2); + }); + + it("samples right away when the used space jumps", async () => { + await history.refreshDiskUsage(); + + vi.setSystemTime(START + MINUTE); + diskUsed(30 * GB + 100 * MB); + await history.refreshDiskUsage(); + expect(mocks.probeDockerUsage).toHaveBeenCalledTimes(1); + + vi.setSystemTime(START + 2 * MINUTE); + diskUsed(33 * GB); + mocks.probeDockerUsage.mockResolvedValue(dockerProbe(13 * GB, 5 * GB)); + await history.refreshDiskUsage(); + + expect(mocks.probeDockerUsage).toHaveBeenCalledTimes(2); + // The disk was walked two minutes ago; a jump repeats that after ten. + expect(mocks.probeSlowUsage).toHaveBeenCalledTimes(1); + expect((await readLines("samples.jsonl")).at(-1)).toMatchObject({ + at: START + 2 * MINUTE, + usedBytes: 33 * GB, + images: 13 * GB, + volumes: 8 * GB, + }); + }); + + it("waits fifty times as long as a slow probe took", async () => { + mocks.probeDockerUsage.mockImplementation(async () => { + vi.setSystemTime(Date.now() + 20_000); + return dockerProbe(10 * GB, 5 * GB); + }); + await history.refreshDiskUsage(); + + for (let minute = 1; minute <= 16; minute++) { + vi.setSystemTime(START + minute * MINUTE); + await history.refreshDiskUsage(); + } + expect(mocks.probeDockerUsage).toHaveBeenCalledTimes(1); + + vi.setSystemTime(START + 17 * MINUTE); + await history.refreshDiskUsage(); + expect(mocks.probeDockerUsage).toHaveBeenCalledTimes(2); + }); + + it("keeps the last known value of a part that a probe could not read", async () => { + await history.refreshDiskUsage(); + + vi.setSystemTime(START + 5 * MINUTE); + mocks.probeDockerUsage.mockResolvedValue({ + parts: {}, + items: {}, + itemKinds: [], + }); + await history.refreshDiskUsage(); + + expect((await readLines("samples.jsonl")).at(-1)).toMatchObject({ + at: START + 5 * MINUTE, + images: 10 * GB, + buildCache: 5 * GB, + }); + }); + + it("probes Docker after a cleanup even though it just did", async () => { + await history.refreshDiskUsage(); + + vi.setSystemTime(START + 10_000); + diskUsed(25 * GB); + mocks.probeDockerUsage.mockResolvedValue(dockerProbe(10 * GB, 0)); + await history.refreshDiskUsage({ force: true }); + + expect((await readLines("samples.jsonl")).at(-1)).toMatchObject({ + usedBytes: 25 * GB, + buildCache: 0, + }); + }); + + it("drops samples older than thirty days", async () => { + await history.refreshDiskUsage(); + const samplesFile = path.join( + mocks.monitoringPath, + "disk-history", + "samples.jsonl", + ); + const old = { at: START - 31 * 24 * HOUR, totalBytes: TOTAL, usedBytes: 1 }; + const recent = { at: START - 24 * HOUR, totalBytes: TOTAL, usedBytes: 2 }; + await writeFile( + samplesFile, + `${JSON.stringify(old)}\n${JSON.stringify(recent)}\n{"at":12`, + ); + + vi.setSystemTime(START + 6 * HOUR); + await history.refreshDiskUsage(); + + expect( + (await readLines("samples.jsonl")).map((sample) => sample.at), + ).toEqual([recent.at, START + 6 * HOUR]); + }); + + it("reports the period, what takes the space and what grew", async () => { + mocks.probeDockerUsage.mockResolvedValue( + dockerProbe(10 * GB, 5 * GB, { + "image:web": 4 * GB, + "image:api": 6 * GB, + }), + ); + mocks.probeSlowUsage.mockResolvedValue( + slowProbe(8 * GB, { "volume:pg-data": 8 * GB }), + ); + await history.refreshDiskUsage(); + + vi.setSystemTime(START + 3 * HOUR); + diskUsed(38 * GB); + mocks.probeDockerUsage.mockResolvedValue( + dockerProbe(13 * GB, 9 * GB, { + "image:web": 7 * GB, + "image:api": 6 * GB + 1 * MB, + }), + ); + mocks.probeSlowUsage.mockResolvedValue( + slowProbe(9 * GB, { "volume:pg-data": 9 * GB }), + ); + await history.refreshDiskUsage(); + + const result = await history.getDiskUsageHistory("1d", 180); + + expect(result.current).toMatchObject({ + usedBytes: 38 * GB, + totalBytes: TOTAL, + availableBytes: 12 * GB, + parts: { + images: 13 * GB, + buildCache: 9 * GB, + volumes: 9 * GB, + swap: 2 * GB, + other: 5 * GB, + }, + }); + expect(result.changeSince).toBe(START); + expect(result.usedDeltaBytes).toBe(8 * GB); + expect( + Object.fromEntries( + result.categories.map((row) => [row.category, row.deltaBytes]), + ), + ).toMatchObject({ + images: 3 * GB, + buildCache: 4 * GB, + volumes: 1 * GB, + swap: 0, + other: 0, + }); + // Build cache has no items, so it is listed as a whole; the 1 MB of + // "api" is below what counts as a change. + expect(result.movers).toEqual([ + { + kind: "category", + name: "buildCache", + bytes: 9 * GB, + deltaBytes: 4 * GB, + }, + { kind: "image", name: "web", bytes: 7 * GB, deltaBytes: 3 * GB }, + { kind: "volume", name: "pg-data", bytes: 9 * GB, deltaBytes: 1 * GB }, + ]); + + expect(result.buckets).toHaveLength(24); + expect(result.bucketMs).toBe(HOUR); + const filled = result.buckets.filter((bucket) => bucket.sample); + expect(filled.map((bucket) => bucket.from)).toEqual([ + START, + START + 3 * HOUR, + ]); + expect(result.to).toBe(START + 4 * HOUR); + }); + + it("has no changes to report while there is a single sample", async () => { + const result = await history.getDiskUsageHistory("7d"); + + expect(mocks.probeDockerUsage).toHaveBeenCalledTimes(1); + expect(result.current?.usedBytes).toBe(30 * GB); + expect(result.changeSince).toBeNull(); + expect(result.usedDeltaBytes).toBeNull(); + expect(result.movers).toEqual([]); + expect(result.categories.every((row) => row.deltaBytes === null)).toBe( + true, + ); + }); + + it("answers without data when the disk cannot be read", async () => { + mocks.getRootDiskStat.mockRejectedValue(new Error("no such device")); + vi.spyOn(console, "error").mockImplementation(() => {}); + + const result = await history.getDiskUsageHistory("1d"); + + expect(result.current).toBeNull(); + expect(result.categories).toEqual([]); + expect(result.buckets.every((bucket) => bucket.sample === null)).toBe(true); + }); +}); + +describe("disk history maths", () => { + let history: DiskHistoryModule; + + beforeEach(async () => { + history = await import("@dokploy/server/monitoring/disk-history"); + }); + + it("treats half a percent of the disk, but at least 256 MiB, as a jump", () => { + const { isDiskUsageJump } = history; + + expect(isDiskUsageJump(10 * GB, 10 * GB + 200 * MB, 20 * GB)).toBe(false); + expect(isDiskUsageJump(10 * GB, 10 * GB + 300 * MB, 20 * GB)).toBe(true); + expect(isDiskUsageJump(10 * GB, 10 * GB - 300 * MB, 20 * GB)).toBe(true); + expect(isDiskUsageJump(100 * GB, 104 * GB, 1000 * GB)).toBe(false); + expect(isDiskUsageJump(100 * GB, 106 * GB, 1000 * GB)).toBe(true); + }); + + it("puts the space no part explains into other, never below zero", () => { + const { toDiskUsageBreakdown } = history; + + expect( + toDiskUsageBreakdown({ + at: 1, + totalBytes: TOTAL, + usedBytes: 10 * GB, + images: 6 * GB, + }).parts, + ).toEqual({ + images: 6 * GB, + buildCache: null, + volumes: null, + containerLogs: null, + containers: null, + dokployData: null, + swap: null, + other: 4 * GB, + }); + expect( + toDiskUsageBreakdown({ + at: 1, + totalBytes: TOTAL, + usedBytes: 10 * GB, + images: 12 * GB, + }).parts.other, + ).toBe(0); + }); + + it("starts day-long bars at the viewer's midnight", () => { + const { getDiskHistoryRange } = history; + // 13 Oct 2026 22:30 UTC is 14 Oct 01:30 in UTC+3. + const now = Date.UTC(2026, 9, 13, 22, 30); + + const range = getDiskHistoryRange("30d", now, 180); + + expect(new Date(range.to).toISOString()).toBe("2026-10-14T21:00:00.000Z"); + expect(new Date(range.from).toISOString()).toBe("2026-09-14T21:00:00.000Z"); + expect(range.count).toBe(30); + }); + + it("keeps the fullest sample of every bar", () => { + const { bucketDiskSamples, getDiskHistoryRange } = history; + const range = getDiskHistoryRange("1d", START + 30 * MINUTE, 0); + const sample = (at: number, usedBytes: number) => ({ + at, + totalBytes: TOTAL, + usedBytes, + }); + + const buckets = bucketDiskSamples( + [ + sample(START - 2 * 24 * HOUR, 49 * GB), + sample(START - HOUR + 5 * MINUTE, 20 * GB), + sample(START - HOUR + 10 * MINUTE, 45 * GB), + sample(START - HOUR + 15 * MINUTE, 21 * GB), + sample(START + 5 * MINUTE, 22 * GB), + ], + range, + ); + + expect(buckets).toHaveLength(24); + expect(buckets.at(-2)?.sample?.usedBytes).toBe(45 * GB); + expect(buckets.at(-1)).toMatchObject({ + from: START, + sample: { usedBytes: 22 * GB }, + }); + expect(buckets.slice(0, -2).every((bucket) => bucket.sample === null)).toBe( + true, + ); + }); + + it("lists what shrank after what grew, and what vanished as freed", () => { + const { getDiskMovers } = history; + + expect( + getDiskMovers( + { "image:old": 2 * GB, "volume:data": 1 * GB }, + { "volume:data": 1.5 * GB, "containerLog:web": 3 * GB }, + [{ category: "other", bytes: 5 * GB, deltaBytes: -1 * GB }], + ), + ).toEqual([ + { kind: "containerLog", name: "web", bytes: 3 * GB, deltaBytes: 3 * GB }, + { kind: "volume", name: "data", bytes: 1.5 * GB, deltaBytes: 0.5 * GB }, + { kind: "category", name: "other", bytes: 5 * GB, deltaBytes: -1 * GB }, + { kind: "image", name: "old", bytes: 0, deltaBytes: -2 * GB }, + ]); + }); +}); diff --git a/apps/dokploy/__test__/monitoring/disk-usage-format.test.ts b/apps/dokploy/__test__/monitoring/disk-usage-format.test.ts new file mode 100644 index 00000000000..05d73f8f440 --- /dev/null +++ b/apps/dokploy/__test__/monitoring/disk-usage-format.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from "vitest"; +import { + DISK_CATEGORIES, + DISK_CATEGORY_ORDER, + describeDiskBucket, + formatDiskBucket, + formatDiskBytes, + formatDiskDelta, + formatDiskTimeTick, + getDiskAxisTicks, + getDiskMoverCategory, + getDiskMoverLabel, + getDiskTimeTicks, +} from "@/components/dashboard/monitoring/free/container/disk-usage/disk-usage-format"; + +const GB = 1e9; +const HOUR = 60 * 60 * 1000; +const localTime = (day: number, hour: number) => + new Date(2026, 9, day, hour).getTime(); + +describe("disk usage formatting", () => { + it("counts in decimal units, like Docker does", () => { + expect(formatDiskBytes(48_043_151_360)).toBe("48.0 GB"); + expect(formatDiskBytes(9_603_361_573)).toBe("9.60 GB"); + expect(formatDiskBytes(433_033_216)).toBe("433 MB"); + expect(formatDiskBytes(4096)).toBe("4 KB"); + expect(formatDiskBytes(0)).toBe("0 B"); + }); + + it("signs a change and calls a rounding-sized one no change", () => { + expect(formatDiskDelta(7.6 * GB)).toBe("+7.60 GB"); + expect(formatDiskDelta(-1.2 * GB)).toBe("−1.20 GB"); + expect(formatDiskDelta(300_000)).toBe("no change"); + expect(formatDiskDelta(0)).toBe("no change"); + }); + + it("keeps the stacking order the colours were checked in", () => { + expect(DISK_CATEGORY_ORDER).toEqual([ + "images", + "buildCache", + "volumes", + "containerLogs", + "containers", + "dokployData", + "swap", + "other", + ]); + expect( + new Set(Object.values(DISK_CATEGORIES).map((meta) => meta.color)).size, + ).toBe(8); + }); + + it("names what a bar covers", () => { + expect(describeDiskBucket(HOUR)).toBe("an hour"); + expect(describeDiskBucket(4 * HOUR)).toBe("4 hours"); + expect(describeDiskBucket(24 * HOUR)).toBe("a day"); + expect(formatDiskBucket(localTime(13, 16), 4 * HOUR)).toBe( + "13 Oct, 16:00–20:00", + ); + expect(formatDiskBucket(localTime(13, 0), 24 * HOUR)).toBe("13 Oct"); + }); +}); + +describe("disk chart axes", () => { + it("puts round values under the disk size", () => { + expect(getDiskAxisTicks(52.74 * GB)).toEqual([0, 20 * GB, 40 * GB]); + expect(getDiskAxisTicks(500 * GB)).toEqual([0, 200 * GB, 400 * GB]); + expect(getDiskAxisTicks(0)).toEqual([0]); + }); + + it("drops a value that would sit right under the disk size", () => { + // 100 GB would touch the "Disk size 105 GB" label. + expect(getDiskAxisTicks(105 * GB)).toEqual([0, 50 * GB]); + }); + + it("labels every third hour of a day", () => { + const starts = Array.from({ length: 24 }, (_, hour) => localTime(13, hour)); + + expect( + getDiskTimeTicks(starts, HOUR)?.map((from) => + formatDiskTimeTick(from, HOUR), + ), + ).toEqual([ + "13 Oct", + "03:00", + "06:00", + "09:00", + "12:00", + "15:00", + "18:00", + "21:00", + ]); + }); + + it("labels only the midnights of a week", () => { + const starts = Array.from({ length: 12 }, (_, index) => + localTime(12, index * 4), + ); + + expect( + getDiskTimeTicks(starts, 4 * HOUR)?.map((from) => + formatDiskTimeTick(from, 4 * HOUR), + ), + ).toEqual(["12 Oct", "13 Oct"]); + }); + + it("leaves day-long bars to the chart", () => { + expect(getDiskTimeTicks([localTime(13, 0)], 24 * HOUR)).toBeUndefined(); + expect(formatDiskTimeTick(localTime(13, 0), 24 * HOUR)).toBe("13 Oct"); + }); +}); + +describe("disk movers", () => { + it("labels an item with its kind and takes the colour of its category", () => { + const mover = { + kind: "containerLog" as const, + name: "web", + bytes: 1, + deltaBytes: 1, + }; + + expect(getDiskMoverLabel(mover)).toEqual({ + kind: "Container logs", + name: "web", + }); + expect(getDiskMoverCategory(mover)).toBe("containerLogs"); + }); + + it("shows a whole category by its own name", () => { + const mover = { + kind: "category" as const, + name: "buildCache", + bytes: 1, + deltaBytes: 1, + }; + + expect(getDiskMoverLabel(mover)).toEqual({ + kind: null, + name: "Build cache", + }); + expect(getDiskMoverCategory(mover)).toBe("buildCache"); + }); +}); diff --git a/apps/dokploy/__test__/monitoring/disk-usage.test.ts b/apps/dokploy/__test__/monitoring/disk-usage.test.ts new file mode 100644 index 00000000000..ac72f8e068b --- /dev/null +++ b/apps/dokploy/__test__/monitoring/disk-usage.test.ts @@ -0,0 +1,331 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + dial: vi.fn(), + info: vi.fn(), + listContainers: vi.fn(), + execAsync: vi.fn(), + readFile: vi.fn(), +})); + +vi.mock("@dokploy/server/constants", () => ({ + docker: { + modem: { dial: mocks.dial }, + info: mocks.info, + listContainers: mocks.listContainers, + }, + paths: () => ({ BASE_PATH: "/etc/dokploy" }), +})); + +vi.mock("@dokploy/server/utils/process/execAsync", () => ({ + execAsync: mocks.execAsync, +})); + +vi.mock("node:fs", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + promises: { ...original.promises, readFile: mocks.readFile }, + }; +}); + +const { + parseDuOutput, + parseSwapFileBytes, + probeDockerUsage, + probeSlowUsage, + summarizeDockerDiskUsage, +} = await import("@dokploy/server/monitoring/disk-usage"); + +const GB = 1e9; +const ANONYMOUS = "a".repeat(64); + +const answerDiskUsage = (answers: Record) => { + mocks.dial.mockImplementation( + ( + request: { options: { type: string[] } }, + callback: (error: Error | null, data?: unknown) => void, + ) => { + const answer = answers[request.options.type.join(",")]; + if (answer instanceof Error) { + callback(answer); + return; + } + callback(null, answer); + }, + ); +}; + +describe("summarizeDockerDiskUsage", () => { + it("groups images by repository and counts only what each one owns", () => { + const { parts, items } = summarizeDockerDiskUsage({ + LayersSize: 5 * GB, + Images: [ + { + RepoTags: ["registry.example:5000/shop/web:v2"], + Size: 3 * GB, + SharedSize: 2 * GB, + }, + { + RepoTags: [":"], + RepoDigests: ["registry.example:5000/shop/web@sha256:abc"], + Size: 3 * GB, + SharedSize: 2.5 * GB, + }, + { RepoTags: null, RepoDigests: [], Size: 1 * GB, SharedSize: -1 }, + ], + }); + + expect(parts).toEqual({ images: 5 * GB }); + expect(items).toEqual({ + "image:registry.example:5000/shop/web": 1.5 * GB, + "image:untagged": 1 * GB, + }); + }); + + it("adds up the tasks of one swarm service", () => { + const { parts, items, itemKinds } = summarizeDockerDiskUsage({ + Containers: [ + { + Names: ["/web.1.abc"], + Labels: { "com.docker.swarm.service.name": "web" }, + SizeRw: 10, + }, + { + Names: ["/web.1.def"], + Labels: { "com.docker.swarm.service.name": "web" }, + SizeRw: 5, + }, + { Names: ["/compose-db-1"], Labels: {}, SizeRw: 7 }, + { Id: "0123456789abcdef", Names: [] }, + ], + }); + + expect(parts).toEqual({ containers: 22 }); + expect(items).toEqual({ + "container:web": 15, + "container:compose-db-1": 7, + "container:0123456789ab": 0, + }); + expect(itemKinds).toEqual(["container"]); + }); + + it("folds anonymous volumes into one item", () => { + const { parts, items } = summarizeDockerDiskUsage({ + Volumes: [ + { Name: "pg-data", UsageData: { Size: 4 * GB } }, + { Name: ANONYMOUS, UsageData: { Size: 100 } }, + { Name: "b".repeat(64), UsageData: { Size: 50 } }, + { Name: "unmeasured", UsageData: { Size: -1 } }, + ], + }); + + expect(parts).toEqual({ volumes: 4 * GB + 150 }); + expect(items).toEqual({ + "volume:pg-data": 4 * GB, + "volume:anonymous volumes": 150, + "volume:unmeasured": 0, + }); + }); + + it("leaves out build cache that images already count", () => { + const { parts, items } = summarizeDockerDiskUsage({ + BuildCache: [ + { Size: 6 * GB, Shared: false }, + { Size: 1 * GB, Shared: true }, + { Size: 2 * GB }, + ], + }); + + expect(parts).toEqual({ buildCache: 8 * GB }); + expect(items).toEqual({}); + }); + + it("reads the sections Docker moved under the Usage objects", () => { + const { parts, items, itemKinds } = summarizeDockerDiskUsage({ + ImageUsage: { + TotalSize: 2 * GB, + Items: [{ RepoTags: ["web:latest"], Size: 2 * GB, SharedSize: 0 }], + }, + ContainerUsage: { + TotalSize: 9, + Items: [{ Names: ["/web"], SizeRw: 9 }], + }, + VolumeUsage: { + TotalSize: 3, + Items: [{ Name: "data", UsageData: { Size: 3 } }], + }, + BuildCacheUsage: { TotalSize: 4, Items: [{ Size: 4, Shared: false }] }, + }); + + expect(parts).toEqual({ + images: 2 * GB, + containers: 9, + volumes: 3, + buildCache: 4, + }); + expect(items).toEqual({ + "image:web": 2 * GB, + "container:web": 9, + "volume:data": 3, + }); + expect(itemKinds).toEqual(["image", "container", "volume"]); + }); + + it("says nothing about a section Docker was not asked for", () => { + expect(summarizeDockerDiskUsage({ Images: null })).toEqual({ + parts: {}, + items: {}, + itemKinds: [], + }); + }); +}); + +describe("parseDuOutput", () => { + it("turns kilobytes into bytes per path", () => { + expect([ + ...parseDuOutput("4\t/data/a\n1024\t/data/with space\n2048\t/data\n"), + ]).toEqual([ + ["/data/a", 4096], + ["/data/with space", 1024 * 1024], + ["/data", 2048 * 1024], + ]); + }); + + it("ignores lines that are not sizes", () => { + expect(parseDuOutput("du: cannot read directory\n").size).toBe(0); + }); +}); + +describe("parseSwapFileBytes", () => { + it("counts swap files and skips swap partitions", () => { + const procSwaps = [ + "Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority", + "/swapfile file\t\t2097148\t\t178280\t\t-3", + "/dev/sda2 partition\t4194300\t\t0\t\t-2", + "/mnt/my\\040swap file\t\t1048572\t\t0\t\t-4", + ].join("\n"); + + expect(parseSwapFileBytes(procSwaps)).toBe((2097148 + 1048572) * 1024); + }); + + it("is zero without swap", () => { + expect( + parseSwapFileBytes("Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n"), + ).toBe(0); + }); +}); + +describe("disk usage probes", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(console, "error").mockImplementation(() => {}); + }); + + it("asks Docker only for what it answers from metadata", async () => { + answerDiskUsage({ + "image,container,build-cache": { + LayersSize: 10, + Images: [{ RepoTags: ["web:1"], Size: 10, SharedSize: 0 }], + Containers: [], + BuildCache: [{ Size: 3, Shared: false }], + }, + }); + + await expect(probeDockerUsage()).resolves.toEqual({ + parts: { images: 10, containers: 0, buildCache: 3 }, + items: { "image:web": 10 }, + itemKinds: ["image", "container"], + }); + expect(mocks.dial.mock.calls[0]?.[0]).toMatchObject({ + path: "/system/df?", + method: "GET", + }); + }); + + it("reports nothing when Docker does not answer", async () => { + answerDiskUsage({ + "image,container,build-cache": new Error("docker is down"), + }); + + await expect(probeDockerUsage()).resolves.toEqual({ + parts: {}, + items: {}, + itemKinds: [], + }); + }); + + it("measures volumes, container logs, Dokploy data and swap files", async () => { + const containerId = "c".repeat(64); + answerDiskUsage({ + volume: { Volumes: [{ Name: "pg-data", UsageData: { Size: 500 } }] }, + }); + mocks.info.mockResolvedValue({ DockerRootDir: "/var/lib/docker" }); + mocks.listContainers.mockResolvedValue([ + { + Id: containerId, + Names: ["/web.1.abc"], + Labels: { "com.docker.swarm.service.name": "web" }, + }, + ]); + mocks.execAsync.mockImplementation(async (command: string) => { + if (command.startsWith("docker run")) { + return { + stdout: `100\t/docker-containers/${containerId}\n8\t/docker-containers/${"d".repeat(64)}\n108\t/docker-containers\n`, + stderr: "", + }; + } + return { + stdout: + "40\t/etc/dokploy/logs\n4\t/etc/dokploy/ssh\n50\t/etc/dokploy\n", + stderr: "", + }; + }); + mocks.readFile.mockResolvedValue( + "Filename Type Size Used Priority\n/swapfile file 1024 0 -2\n", + ); + + await expect(probeSlowUsage()).resolves.toEqual({ + parts: { + volumes: 500, + containerLogs: 108 * 1024, + dokployData: 50 * 1024, + swap: 1024 * 1024, + }, + items: { + "volume:pg-data": 500, + "containerLog:web": 100 * 1024, + "containerLog:dddddddddddd": 8 * 1024, + "dokployData:logs": 40 * 1024, + "dokployData:ssh": 4 * 1024, + }, + itemKinds: ["volume", "containerLog", "dokployData"], + }); + + const logsCommand = mocks.execAsync.mock.calls + .map(([command]) => command as string) + .find((command) => command.startsWith("docker run")); + expect(logsCommand).toContain("--network none"); + // shell-quote escapes the colons of the mount. + expect(logsCommand?.replaceAll("\\", "")).toContain( + "-v /var/lib/docker/containers:/docker-containers:ro busybox du", + ); + }); + + it("keeps the parts it could measure when the others fail", async () => { + answerDiskUsage({ volume: new Error("docker is down") }); + mocks.info.mockRejectedValue(new Error("docker is down")); + mocks.listContainers.mockRejectedValue(new Error("docker is down")); + mocks.execAsync.mockResolvedValue({ + stdout: "50\t/etc/dokploy\n", + stderr: "", + }); + mocks.readFile.mockRejectedValue(new Error("no /proc on this system")); + + await expect(probeSlowUsage()).resolves.toEqual({ + parts: { dokployData: 50 * 1024 }, + items: {}, + itemKinds: ["dokployData"], + }); + }); +}); diff --git a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts index 53151907501..6a3a42334ce 100644 --- a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts +++ b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts @@ -50,6 +50,7 @@ const mocks = vi.hoisted(() => ({ readPorts: vi.fn(), recreateDirectory: vi.fn(), reloadDockerResource: vi.fn(), + refreshDiskUsage: vi.fn(), removeOldDokployImages: vi.fn(), removeJob: vi.fn(), schedule: vi.fn(), @@ -76,6 +77,7 @@ const redactWebServerSettings = (settings: T) => settings; vi.mock("@dokploy/server", async () => ({ CLEANUP_CRON_JOB: "0 0 * * *", DEFAULT_UPDATE_DATA: {}, + diskHistoryInputSchema: (await import("zod")).z.any(), dockerDiskUsageDetailLimitSchema: (await import("zod")).z.any(), DOKPLOY_KEEP_IMAGES_MAX: 5, DOKPLOY_KEEP_IMAGES_MIN: 3, @@ -131,6 +133,7 @@ vi.mock("@dokploy/server", async () => ({ readPorts: mocks.readPorts, recreateDirectory: mocks.recreateDirectory, reloadDockerResource: mocks.reloadDockerResource, + refreshDiskUsage: mocks.refreshDiskUsage, removeOldDokployImages: mocks.removeOldDokployImages, sendDockerCleanupNotifications: mocks.sendDockerCleanupNotifications, setupGPUSupport: mocks.setupGPUSupport, @@ -247,6 +250,7 @@ describe("settings Docker server boundary", () => { }); mocks.getAccessibleServerIds.mockResolvedValue(new Set(["server-1"])); mocks.cleanupBuilders.mockResolvedValue(undefined); + mocks.refreshDiskUsage.mockResolvedValue(undefined); mocks.cleanupContainers.mockResolvedValue(undefined); mocks.cleanupImages.mockResolvedValue(undefined); mocks.cleanupSystem.mockResolvedValue(undefined); @@ -465,6 +469,23 @@ describe("settings Docker server boundary", () => { expect(mocks.cleanupBuilders).not.toHaveBeenCalled(); }); + it("samples the disk after a cleanup, and answers even when that fails", async () => { + const space = { availableBytes: 2, oldImageCount: 0 }; + mocks.isServerUpdateRunning.mockReturnValue(false); + mocks.getUpdateDiskSpace.mockResolvedValue(space); + mocks.refreshDiskUsage.mockRejectedValue(new Error("docker is down")); + const consoleError = vi + .spyOn(console, "error") + .mockImplementation(() => {}); + + await expect( + createCaller().freeUpdateDiskSpace({ buildCache: false }), + ).resolves.toEqual(space); + + expect(mocks.refreshDiskUsage).toHaveBeenCalledWith({ force: true }); + consoleError.mockRestore(); + }); + it("clears the build cache only when asked", async () => { mocks.isServerUpdateRunning.mockReturnValue(false); diff --git a/apps/dokploy/__test__/settings/request-log-authorization.test.ts b/apps/dokploy/__test__/settings/request-log-authorization.test.ts index 6def3f05dc4..2648b3cd82e 100644 --- a/apps/dokploy/__test__/settings/request-log-authorization.test.ts +++ b/apps/dokploy/__test__/settings/request-log-authorization.test.ts @@ -93,6 +93,7 @@ const mocks = vi.hoisted(() => ({ vi.mock("@dokploy/server", async () => ({ CLEANUP_CRON_JOB: "0 0 * * *", DEFAULT_UPDATE_DATA: {}, + diskHistoryInputSchema: (await import("zod")).z.any(), dockerDiskUsageDetailLimitSchema: (await import("zod")).z.any(), DOKPLOY_KEEP_IMAGES_MAX: 5, DOKPLOY_KEEP_IMAGES_MIN: 3, diff --git a/apps/dokploy/__test__/settings/update-disk-space.test.ts b/apps/dokploy/__test__/settings/update-disk-space.test.ts index abd9266db86..0a2fcc664a9 100644 --- a/apps/dokploy/__test__/settings/update-disk-space.test.ts +++ b/apps/dokploy/__test__/settings/update-disk-space.test.ts @@ -152,6 +152,12 @@ describe("update disk space", () => { requiredBytes: 4_450_000_000, oldImageCount: 1, oldImageBytes: 1_500_000_000, + oldImages: [ + expect.objectContaining({ + id: "sha256:old", + uniqueSizeBytes: 1_500_000_000, + }), + ], buildCacheBytes: 6_193_000_000, }); }); @@ -166,6 +172,7 @@ describe("update disk space", () => { requiredBytes: null, oldImageCount: 0, oldImageBytes: 0, + oldImages: [], buildCacheBytes: 0, }); }); diff --git a/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-breakdown.tsx b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-breakdown.tsx new file mode 100644 index 00000000000..5928bd1c588 --- /dev/null +++ b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-breakdown.tsx @@ -0,0 +1,162 @@ +import type { DiskUsageHistory } from "@dokploy/server/index"; +import { cn } from "@/lib/utils"; +import { + DISK_CATEGORIES, + formatDiskBytes, + formatDiskDelta, + getDiskMoverCategory, + getDiskMoverLabel, +} from "./disk-usage-format"; + +type Current = NonNullable; + +const Swatch = ({ color }: { color: string }) => ( + +); + +const Delta = ({ bytes }: { bytes: number | null }) => ( + + {bytes === null ? "—" : formatDiskDelta(bytes)} + +); + +export const DiskCompositionBar = ({ + current, + categories, +}: { + current: Current; + categories: DiskUsageHistory["categories"]; +}) => ( +
+ {categories + .filter((row) => (row.bytes ?? 0) > 0) + .map((row) => ( +
+ ))} +
+); + +export const DiskCategoryTable = ({ + history, + current, +}: { + history: DiskUsageHistory; + current: Current; +}) => ( +
+
+ What takes the space + Now + Change +
+
    + {history.categories.map((row) => { + const meta = DISK_CATEGORIES[row.category]; + return ( +
  • +
    + +
    +

    {meta.label}

    +

    + {meta.description} +

    +
    +
    + + {row.bytes === null ? "—" : formatDiskBytes(row.bytes)} + + + + +
  • + ); + })} +
+
+ Used in total + + {formatDiskBytes(current.usedBytes)} + + + + +
+
+); + +export const DiskMoversList = ({ history }: { history: DiskUsageHistory }) => { + if (history.changeSince === null) { + return ( +

+ The history has just started. What grows shows up here after the next + measurements. +

+ ); + } + if (history.movers.length === 0) { + return ( +

+ Nothing changed by more than 10 MB. +

+ ); + } + return ( +
    + {history.movers.map((mover) => { + const label = getDiskMoverLabel(mover); + return ( +
  • +
    + +
    +

    + {label.name} +

    + {label.kind && ( +

    {label.kind}

    + )} +
    +
    +
    + +

    + now {formatDiskBytes(mover.bytes)} +

    +
    +
  • + ); + })} +
+ ); +}; diff --git a/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-card.tsx b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-card.tsx new file mode 100644 index 00000000000..18000a9d1e5 --- /dev/null +++ b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-card.tsx @@ -0,0 +1,30 @@ +import type { DiskHistoryPeriod } from "@dokploy/server/index"; +import { useState } from "react"; +import { FreeDiskSpace } from "@/components/dashboard/settings/web-server/free-disk-space"; +import { api } from "@/utils/api"; +import { DiskUsagePanel } from "./disk-usage-panel"; + +const REFRESH_INTERVAL_MS = 60_000; + +export const DiskUsageCard = () => { + const [period, setPeriod] = useState("1d"); + const { data, isLoading } = api.settings.getDiskUsageHistory.useQuery( + { period, utcOffsetMinutes: -new Date().getTimezoneOffset() }, + { + refetchOnWindowFocus: false, + refetchInterval: REFRESH_INTERVAL_MS, + placeholderData: (previous) => previous, + }, + ); + + return ( + + + + ); +}; diff --git a/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-format.ts b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-format.ts new file mode 100644 index 00000000000..ce68d09a9d2 --- /dev/null +++ b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-format.ts @@ -0,0 +1,186 @@ +import type { + DiskCategory, + DiskHistoryPeriod, + DiskMover, +} from "@dokploy/server/index"; +import { format } from "date-fns"; + +interface DiskCategoryMeta { + label: string; + description: string; + color: string; +} + +// The order is the stacking order, bottom to top. The colors were checked as +// neighbours in exactly this order, so a new category goes through the palette +// validator before it is inserted. +export const DISK_CATEGORIES: Record = { + images: { + label: "Images", + description: "Docker images of your apps, databases and Dokploy itself.", + color: "var(--disk-images)", + }, + buildCache: { + label: "Build cache", + description: + "Layers kept to make the next builds faster, without those that images share. Safe to clear.", + color: "var(--disk-build-cache)", + }, + volumes: { + label: "Volumes", + description: "Data of databases and apps stored in Docker volumes.", + color: "var(--disk-volumes)", + }, + containerLogs: { + label: "Container logs", + description: "Output that Docker stores for every container.", + color: "var(--disk-container-logs)", + }, + containers: { + label: "Container files", + description: "Files written inside containers, outside of volumes.", + color: "var(--disk-containers)", + }, + dokployData: { + label: "Dokploy data", + description: "Deployment logs, source checkouts and backups of Dokploy.", + color: "var(--disk-dokploy-data)", + }, + swap: { + label: "Swap files", + description: "Disk space the system reserved to extend the memory.", + color: "var(--disk-swap)", + }, + other: { + label: "System and other", + description: + "The operating system, system logs and everything Docker does not report.", + color: "var(--disk-other)", + }, +}; + +export const DISK_CATEGORY_ORDER = Object.keys( + DISK_CATEGORIES, +) as DiskCategory[]; + +export const DISK_PERIOD_LABELS: Record = { + "1d": "1 day", + "3d": "3 days", + "7d": "7 days", + "30d": "30 days", +}; + +// Decimal units, the same Docker prints, so the numbers match `docker system df`. +export const formatDiskBytes = (bytes: number) => { + const absolute = Math.abs(bytes); + if (absolute >= 1e9) { + return `${(bytes / 1e9).toFixed(absolute >= 1e10 ? 1 : 2)} GB`; + } + if (absolute >= 1e6) { + return `${Math.round(bytes / 1e6)} MB`; + } + if (absolute >= 1e3) { + return `${Math.round(bytes / 1e3)} KB`; + } + return `${bytes} B`; +}; + +export const formatDiskDelta = (bytes: number) => { + // A change that rounds to zero reads as "no change", not as "+0 MB". + if (Math.abs(bytes) < 5e5) { + return "no change"; + } + return `${bytes > 0 ? "+" : "−"}${formatDiskBytes(Math.abs(bytes))}`; +}; + +export const formatDiskBucket = (from: number, bucketMs: number) => { + const start = new Date(from); + if (bucketMs >= 24 * 60 * 60 * 1000) { + return format(start, "d MMM"); + } + return `${format(start, "d MMM")}, ${format(start, "HH:mm")}–${format(new Date(from + bucketMs), "HH:mm")}`; +}; + +export const describeDiskBucket = (bucketMs: number) => { + const hours = bucketMs / (60 * 60 * 1000); + if (hours >= 24) { + return "a day"; + } + return hours === 1 ? "an hour" : `${hours} hours`; +}; + +const MOVER_KIND_LABELS: Record = { + image: "Images", + volume: "Volume", + container: "Container files", + containerLog: "Container logs", + dokployData: "Dokploy data", + category: null, +}; + +export const getDiskMoverLabel = (mover: DiskMover) => { + if (mover.kind === "category") { + return { + kind: null, + name: DISK_CATEGORIES[mover.name as DiskCategory]?.label ?? mover.name, + }; + } + return { kind: MOVER_KIND_LABELS[mover.kind], name: mover.name }; +}; + +const MOVER_CATEGORIES: Record< + Exclude, + DiskCategory +> = { + image: "images", + volume: "volumes", + container: "containers", + containerLog: "containerLogs", + dokployData: "dokployData", +}; + +export const getDiskMoverCategory = (mover: DiskMover): DiskCategory => + mover.kind === "category" + ? (mover.name as DiskCategory) + : MOVER_CATEGORIES[mover.kind]; + +/** Round axis values (1, 2 or 5 times a power of ten) below the disk size. */ +export const getDiskAxisTicks = (totalBytes: number) => { + if (totalBytes <= 0) { + return [0]; + } + const rough = totalBytes / 4; + const magnitude = 10 ** Math.floor(Math.log10(rough)); + const step = + magnitude * + ([1, 2, 5, 10].find((factor) => factor * magnitude >= rough) ?? 10); + const ticks: number[] = []; + // A value right under the disk size would collide with the label of that line. + for (let value = 0; value <= totalBytes * 0.92; value += step) { + ticks.push(value); + } + return ticks; +}; + +const HOUR_MS = 60 * 60 * 1000; + +/** + * Bars that get a label under the axis: every third hour of a day, noon and + * midnight of three days, midnight of a week. Day-long bars are left to the chart. + */ +export const getDiskTimeTicks = (bucketStarts: number[], bucketMs: number) => { + const hours = bucketMs / HOUR_MS; + if (hours >= 24) { + return undefined; + } + const step = hours <= 1 ? 3 : hours <= 2 ? 12 : 24; + return bucketStarts.filter((from) => new Date(from).getHours() % step === 0); +}; + +/** Midnight carries the date, so the days of a longer period can be told apart. */ +export const formatDiskTimeTick = (from: number, bucketMs: number) => { + const start = new Date(from); + return bucketMs >= 24 * HOUR_MS || start.getHours() === 0 + ? format(start, "d MMM") + : format(start, "HH:mm"); +}; diff --git a/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-history-chart.tsx b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-history-chart.tsx new file mode 100644 index 00000000000..06a8644affe --- /dev/null +++ b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-history-chart.tsx @@ -0,0 +1,178 @@ +import type { DiskCategory, DiskUsageHistory } from "@dokploy/server/index"; +import { + Bar, + BarChart, + CartesianGrid, + ReferenceLine, + XAxis, + YAxis, +} from "recharts"; +import { + type ChartConfig, + ChartContainer, + ChartTooltip, +} from "@/components/ui/chart"; +import { + DISK_CATEGORIES, + DISK_CATEGORY_ORDER, + describeDiskBucket, + formatDiskBucket, + formatDiskBytes, + formatDiskTimeTick, + getDiskAxisTicks, + getDiskTimeTicks, +} from "./disk-usage-format"; + +const chartConfig = Object.fromEntries( + DISK_CATEGORY_ORDER.map((category) => [ + category, + { + label: DISK_CATEGORIES[category].label, + color: DISK_CATEGORIES[category].color, + }, + ]), +) satisfies ChartConfig; + +type ChartRow = { from: number; usedBytes: number | null } & Record< + DiskCategory, + number +>; + +const BucketTooltip = ({ + active, + payload, + bucketMs, + totalBytes, +}: { + active?: boolean; + payload?: { payload?: ChartRow }[]; + bucketMs: number; + totalBytes: number; +}) => { + const row = payload?.[0]?.payload; + if (!active || !row || row.usedBytes === null) { + return null; + } + return ( +
+
{formatDiskBucket(row.from, bucketMs)}
+
+ Used {formatDiskBytes(row.usedBytes)} of {formatDiskBytes(totalBytes)} +
+
+ {DISK_CATEGORY_ORDER.filter((category) => row[category] > 0).map( + (category) => ( +
+ + + {DISK_CATEGORIES[category].label} + + + {formatDiskBytes(row[category])} + +
+ ), + )} +
+
+ ); +}; + +interface Props { + history: DiskUsageHistory; +} + +export const DiskUsageHistoryChart = ({ history }: Props) => { + const totalBytes = + history.current?.totalBytes ?? + history.buckets + .map((bucket) => bucket.sample?.totalBytes ?? 0) + .reduce((latest, bytes) => bytes || latest, 0); + const rows: ChartRow[] = history.buckets.map((bucket) => ({ + from: bucket.from, + usedBytes: bucket.sample?.usedBytes ?? null, + ...(Object.fromEntries( + DISK_CATEGORY_ORDER.map((category) => [ + category, + bucket.sample?.parts[category] ?? 0, + ]), + ) as Record), + })); + + return ( +
+ + + + row.from), + history.bucketMs, + )} + tickFormatter={(from: number) => + formatDiskTimeTick(from, history.bucketMs) + } + /> + `${Math.round(bytes / 1e9)} GB`} + /> + + + } + /> + {DISK_CATEGORY_ORDER.map((category, index) => ( + + ))} + + +

+ Each bar covers {describeDiskBucket(history.bucketMs)} and shows the + fullest moment of it, so a short spike stays visible. +

+
+ ); +}; diff --git a/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-panel.tsx b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-panel.tsx new file mode 100644 index 00000000000..c0d7a4083ee --- /dev/null +++ b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-panel.tsx @@ -0,0 +1,128 @@ +import type { + DiskHistoryPeriod, + DiskUsageHistory, +} from "@dokploy/server/index"; +import { format } from "date-fns"; +import { AlertTriangle, Loader2 } from "lucide-react"; +import type { ReactNode } from "react"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { + DiskCategoryTable, + DiskCompositionBar, + DiskMoversList, +} from "./disk-usage-breakdown"; +import { DISK_PERIOD_LABELS, formatDiskBytes } from "./disk-usage-format"; +import { DiskUsageHistoryChart } from "./disk-usage-history-chart"; + +const ALMOST_FULL_SHARE = 0.9; + +interface Props { + history: DiskUsageHistory | null | undefined; + isLoading: boolean; + period: DiskHistoryPeriod; + onPeriodChange: (period: DiskHistoryPeriod) => void; + /** Rendered under the tables; the page puts the cleanup actions here. */ + children?: ReactNode; +} + +export const DiskUsagePanel = ({ + history, + isLoading, + period, + onPeriodChange, + children, +}: Props) => { + const current = history?.current; + + if (isLoading && !history) { + return ( +
+ +
+ ); + } + if (!history || !current) { + return ( +

+ No disk usage data available. +

+ ); + } + + const usedShare = current.usedBytes / current.totalBytes; + // The first saved sample is later than the period's start while the + // history is younger than the period. + const isShortHistory = + history.changeSince !== null && + history.changeSince > history.from + history.bucketMs; + + return ( +
+
+
+

+ {formatDiskBytes(current.usedBytes)} + + used of {formatDiskBytes(current.totalBytes)} + +

+

+ + {formatDiskBytes(current.availableBytes)} free ·{" "} + {Math.round(usedShare * 100)}% full + + {usedShare >= ALMOST_FULL_SHARE && ( + + + Almost full + + )} +

+
+ onPeriodChange(value as DiskHistoryPeriod)} + > + + {(Object.keys(DISK_PERIOD_LABELS) as DiskHistoryPeriod[]).map( + (option) => ( + + {DISK_PERIOD_LABELS[option]} + + ), + )} + + +
+ + + + +
+ +
+
+ {history.changeSince === null + ? "What changed the most" + : `What changed the most since ${format(new Date(history.changeSince), "d MMM, HH:mm")}`} +
+
+ +
+ {isShortHistory && ( +

+ The history is shorter than {DISK_PERIOD_LABELS[period]}, so the + changes are counted from its first measurement. +

+ )} +
+
+ +

+ 1 GB is 1000 MB here, the same units Docker reports. +

+ + {children} +
+ ); +}; diff --git a/apps/dokploy/components/dashboard/monitoring/free/container/docker-disk-chart.tsx b/apps/dokploy/components/dashboard/monitoring/free/container/docker-disk-chart.tsx deleted file mode 100644 index 987d956d090..00000000000 --- a/apps/dokploy/components/dashboard/monitoring/free/container/docker-disk-chart.tsx +++ /dev/null @@ -1,83 +0,0 @@ -import { format } from "date-fns"; -import { Area, AreaChart, CartesianGrid, YAxis } from "recharts"; -import { - type ChartConfig, - ChartContainer, - ChartTooltip, - ChartTooltipContent, -} from "@/components/ui/chart"; -import type { DockerStatsJSON } from "./show-free-container-monitoring"; - -interface Props { - accumulativeData: DockerStatsJSON["disk"]; - diskTotal: number; -} - -const chartConfig = { - usedGb: { - label: "Used (GB)", - color: "hsl(var(--chart-3))", - }, -} satisfies ChartConfig; - -export const DockerDiskChart = ({ accumulativeData, diskTotal }: Props) => { - const transformedData = accumulativeData.map((item, index) => ({ - time: item.time, - name: `Point ${index + 1}`, - usedGb: +item.value.diskUsage, - totalGb: +item.value.diskTotal, - })); - - return ( - - - - - - - - - - `${value} GB`} - /> - { - const time = payload?.[0]?.payload?.time; - return time ? format(new Date(time), "PPpp") : ""; - }} - formatter={(value) => { - return [`${value} GB`, "Used"]; - }} - /> - } - /> - - - - ); -}; diff --git a/apps/dokploy/components/dashboard/monitoring/free/container/docker-disk-usage-chart.tsx b/apps/dokploy/components/dashboard/monitoring/free/container/docker-disk-usage-chart.tsx index 6860e218f29..6e5ca7dec84 100644 --- a/apps/dokploy/components/dashboard/monitoring/free/container/docker-disk-usage-chart.tsx +++ b/apps/dokploy/components/dashboard/monitoring/free/container/docker-disk-usage-chart.tsx @@ -41,21 +41,22 @@ const chartConfig = { value: { label: "Size", }, + // The same colours as in the disk space card above it on the page. images: { label: "Images", - color: "hsl(var(--chart-1))", + color: "var(--disk-images)", }, containers: { label: "Containers", - color: "hsl(var(--chart-2))", + color: "var(--disk-containers)", }, volumes: { label: "Volumes", - color: "hsl(var(--chart-3))", + color: "var(--disk-volumes)", }, buildCache: { label: "Build Cache", - color: "hsl(var(--chart-4))", + color: "var(--disk-build-cache)", }, } satisfies ChartConfig; diff --git a/apps/dokploy/components/dashboard/monitoring/free/container/show-free-container-monitoring.tsx b/apps/dokploy/components/dashboard/monitoring/free/container/show-free-container-monitoring.tsx index 9e10bfb3a9c..ebad53468f4 100644 --- a/apps/dokploy/components/dashboard/monitoring/free/container/show-free-container-monitoring.tsx +++ b/apps/dokploy/components/dashboard/monitoring/free/container/show-free-container-monitoring.tsx @@ -5,9 +5,9 @@ import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Progress } from "@/components/ui/progress"; import { api } from "@/utils/api"; import { ContainerResourceUsage } from "../../container-resource-usage/container-resource-usage"; +import { DiskUsageCard } from "./disk-usage/disk-usage-card"; import { DockerBlockChart } from "./docker-block-chart"; import { DockerCpuChart } from "./docker-cpu-chart"; -import { DockerDiskChart } from "./docker-disk-chart"; import { DockerDiskUsageChart } from "./docker-disk-usage-chart"; import { getDockerDiskUsageCardClassName } from "./docker-disk-usage-layout"; import { DockerMemoryChart } from "./docker-memory-chart"; @@ -314,24 +314,12 @@ export const ContainerFreeMonitoring = ({ {appName === "dokploy" && ( - + Disk Space -
- - {`Used: ${currentData.disk.value.diskUsage} GB / Limit: ${currentData.disk.value.diskTotal} GB`} - - - -
+
)} diff --git a/apps/dokploy/components/dashboard/settings/web-server/free-disk-space.tsx b/apps/dokploy/components/dashboard/settings/web-server/free-disk-space.tsx index 8fa7ce9f181..de53f36a822 100644 --- a/apps/dokploy/components/dashboard/settings/web-server/free-disk-space.tsx +++ b/apps/dokploy/components/dashboard/settings/web-server/free-disk-space.tsx @@ -1,21 +1,32 @@ +import { format } from "date-fns"; import { AlertTriangle, HardDrive, Loader2, Trash2 } from "lucide-react"; -import { useState } from "react"; +import { useId, useState } from "react"; import { toast } from "sonner"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; import { Label } from "@/components/ui/label"; import { cn } from "@/lib/utils"; import { api } from "@/utils/api"; -import { formatImageSize } from "./dokploy-image-plan"; +import { formatImageSize, getDokployImageVersion } from "./dokploy-image-plan"; interface Props { /** The last update failed because the disk was full. */ diskFull?: boolean; disabled?: boolean; + /** + * Lists the builds a cleanup removes and leaves out the free space line, + * for a page that already shows the disk. + */ + showOldImages?: boolean; } -export const FreeDiskSpace = ({ diskFull = false, disabled }: Props) => { +export const FreeDiskSpace = ({ + diskFull = false, + disabled, + showOldImages = false, +}: Props) => { const [buildCache, setBuildCache] = useState(false); + const buildCacheId = useId(); const utils = api.useUtils(); const { data: space, isLoading } = api.settings.getUpdateDiskSpace.useQuery( undefined, @@ -58,7 +69,11 @@ export const FreeDiskSpace = ({ diskFull = false, disabled }: Props) => { : "Cleanup finished", ); } - await utils.settings.getDokployImages.invalidate(); + await Promise.all([ + utils.settings.getDokployImages.invalidate(), + utils.settings.getDiskUsageHistory.invalidate(), + utils.settings.getDockerDiskUsage.invalidate(), + ]); } catch (error) { toast.error( error instanceof Error && error.message @@ -77,19 +92,23 @@ export const FreeDiskSpace = ({ diskFull = false, disabled }: Props) => { : "border-border", )} > -
- - - Free disk space:{" "} - - {space.availableBytes !== null - ? formatImageSize(space.availableBytes) - : "unknown"} + {showOldImages ? ( + Free up space + ) : ( +
+ + + Free disk space:{" "} + + {space.availableBytes !== null + ? formatImageSize(space.availableBytes) + : "unknown"} + + {space.totalBytes !== null && + ` of ${formatImageSize(space.totalBytes)}`} - {space.totalBytes !== null && - ` of ${formatImageSize(space.totalBytes)}`} - -
+
+ )} {(diskFull || isLow) && (
@@ -108,17 +127,37 @@ export const FreeDiskSpace = ({ diskFull = false, disabled }: Props) => { : "No old Dokploy builds to remove. The running build and the previous one always stay."} + {showOldImages && space.oldImages.length > 0 && ( +
    + {space.oldImages.map((image) => ( +
  • + + {getDokployImageVersion(image)} + + {" · "} + {format(new Date(image.createdAt), "dd.MM.yyyy HH:mm")} + + + + frees{" "} + {formatImageSize(image.uniqueSizeBytes ?? image.sizeBytes)} + +
  • + ))} +
+ )} + {space.buildCacheBytes > 0 && (
setBuildCache(checked === true)} disabled={disabled || isPending} className="mt-0.5" />
diff --git a/apps/dokploy/server/api/routers/settings.ts b/apps/dokploy/server/api/routers/settings.ts index feb062f64f1..17ed0f9b006 100644 --- a/apps/dokploy/server/api/routers/settings.ts +++ b/apps/dokploy/server/api/routers/settings.ts @@ -16,11 +16,13 @@ import { DEFAULT_UPDATE_DATA, DOKPLOY_KEEP_IMAGES_MAX, DOKPLOY_KEEP_IMAGES_MIN, + diskHistoryInputSchema, dockerDiskUsageDetailLimitSchema, filterProtectedTraefikEntries, findServerById, getAgentHitsUpdateCommand, getAgentHitsUpdateImage, + getDiskUsageHistory, getDockerDiskUsage, getDokployImageKeepCount, getDokployImages, @@ -53,6 +55,7 @@ import { readPorts, recreateDirectory, redactWebServerSettings, + refreshDiskUsage, reloadDockerResource, removeOldDokployImages, resolveDockerDiskUsageDetailLimit, @@ -441,6 +444,14 @@ export const settingsRouter = createTRPCRouter({ resolveDockerDiskUsageDetailLimit(input?.detailLimit), ); }), + getDiskUsageHistory: adminProcedure + .input(diskHistoryInputSchema) + .query(async ({ input }) => { + if (IS_CLOUD) { + return null; + } + return await getDiskUsageHistory(input.period, input.utcOffsetMinutes); + }), saveSSHPrivateKey: adminProcedure .input(apiSaveSSHKey) .mutation(async ({ input, ctx }) => { @@ -817,6 +828,11 @@ export const settingsRouter = createTRPCRouter({ ? "free-update-disk-space-with-build-cache" : "free-update-disk-space", }); + // The page asks for the history right after this; without a new + // sample it would still show the space as taken. + await refreshDiskUsage({ force: true }).catch((error) => { + console.error("Disk usage: sampling after the cleanup failed", error); + }); return await getUpdateDiskSpace(); }), diff --git a/apps/dokploy/server/api/utils/procedure-access.ts b/apps/dokploy/server/api/utils/procedure-access.ts index 0f980437183..25bb6b10baf 100644 --- a/apps/dokploy/server/api/utils/procedure-access.ts +++ b/apps/dokploy/server/api/utils/procedure-access.ts @@ -599,6 +599,7 @@ export const PROCEDURE_ACCESS: Record = { "settings.cleanUnusedImages": "write", "settings.cleanUnusedVolumes": "dangerous", "settings.freeUpdateDiskSpace": "write", + "settings.getDiskUsageHistory": "read", "settings.getDockerDiskUsage": "read", "settings.getDokployCloudIps": "read", "settings.getDokployImageKeepCount": "read", diff --git a/apps/dokploy/server/server.ts b/apps/dokploy/server/server.ts index 6c236a02267..c70e3c48ae7 100644 --- a/apps/dokploy/server/server.ts +++ b/apps/dokploy/server/server.ts @@ -14,6 +14,7 @@ import { initVolumeBackupsCronJobs, sendDokployRestartNotifications, setupDirectories, + startDiskHistorySampler, } from "@dokploy/server"; import { warnAboutInterruptedTraefikDeletion } from "@dokploy/server/services/domain-overview"; import { config } from "dotenv"; @@ -158,6 +159,7 @@ void app.prepare().then(async () => { await initVolumeBackupsCronJobs(); await sendDokployRestartNotifications(); void cleanupOldDokployImages(); + startDiskHistorySampler(); void warnAboutInterruptedTraefikDeletion(); } await initEnterpriseBackupCronJobs(); diff --git a/apps/dokploy/styles/globals.css b/apps/dokploy/styles/globals.css index ae5e6fb8125..4937506f3de 100644 --- a/apps/dokploy/styles/globals.css +++ b/apps/dokploy/styles/globals.css @@ -206,6 +206,14 @@ body[data-scroll-locked] [data-slot="sidebar-inset"] { --chart-3: 197 37% 24%; --chart-4: 43 74% 66%; --chart-5: 27 87% 67%; + --disk-images: #2a78d6; + --disk-build-cache: #eb6834; + --disk-volumes: #1baf7a; + --disk-container-logs: #eda100; + --disk-containers: #e87ba4; + --disk-dokploy-data: #008300; + --disk-swap: #4a3aa7; + --disk-other: #898781; --sidebar-background: 0 0% 98%; --sidebar-foreground: 240 5.3% 26.1%; --sidebar-primary: 240 5.9% 10%; @@ -253,6 +261,14 @@ body[data-scroll-locked] [data-slot="sidebar-inset"] { --chart-3: 30 80% 55%; --chart-4: 280 65% 60%; --chart-2: 340 75% 55%; + --disk-images: #3987e5; + --disk-build-cache: #d95926; + --disk-volumes: #199e70; + --disk-container-logs: #c98500; + --disk-containers: #d55181; + --disk-dokploy-data: #008300; + --disk-swap: #9085e9; + --disk-other: #898781; --sidebar-background: 240 5.9% 10%; --sidebar-foreground: 240 4.8% 95.9%; --sidebar-primary: 224.3 76.3% 48%; diff --git a/docs/agenthits-fork.md b/docs/agenthits-fork.md index 3d3b361d999..3948d75256e 100644 --- a/docs/agenthits-fork.md +++ b/docs/agenthits-fork.md @@ -37,7 +37,9 @@ Fork: off_v0.29.8/Fork_+ - Installer `install-agenthits.sh` ставит форк напрямую из GHCR, создает Docker secrets для runtime secret values и поддерживает `postgres:18+` volume target. - Мониторинг содержит дополнительные улучшения: breakdown ресурсов контейнеров, - детализацию Docker disk usage и отображение swap usage. + детализацию Docker disk usage, отображение swap usage и историю занятого + диска по категориям за день, 3, 7 и 30 дней с очисткой старых образов панели + и кэша сборки. - Добавлена безопасная upsert-логика application environment variables. - Исправлено сопоставление GitHub webhook deploys по owner login. - Ветка содержит security hardening поверх официального `v0.29.8`: проверки @@ -95,7 +97,9 @@ commits, the fork version looks like `off_v0.29.8/Fork_157+abc123def456`. secrets for runtime secret values, and supports the `postgres:18+` volume target. - Monitoring includes extra improvements: container resource breakdown, Docker - disk usage details, and swap usage display. + disk usage details, swap usage display, and a disk usage history by category + for 1, 3, 7 and 30 days with a cleanup of old panel images and the build + cache. - Safe application environment variable upsert logic was added. - GitHub webhook deploy matching was fixed to use owner login. - The branch includes security hardening on top of official `v0.29.8`: access diff --git a/docs/server-maintenance.md b/docs/server-maintenance.md index 7da63ad48cc..40743d3f9a5 100644 --- a/docs/server-maintenance.md +++ b/docs/server-maintenance.md @@ -79,3 +79,21 @@ Docker и Postgres кнопкой не обновляются. Карточка - Traefik: `docker logs --tail 120 dokploy-traefik` - Резервные копии Postgres: `/var/backups/dokploy/`, файлы вида `postgres-<дата>.sql.gz`. Доступны только root, сами не удаляются. - Копии пакетов Docker, если обновляли Docker: `/var/backups/dokploy/docker-rollback-<дата>/`. + +## 6. Диск: что занимает место и как его освободить + +Страница **Monitoring**, карточка **Disk Space**. + +- Панель раз в минуту читает, сколько занято на диске, и раз в 5 минут спрашивает у Docker размеры образов, кэша сборки и файлов контейнеров. Раз в час она измеряет тома, логи контейнеров, папку `/etc/dokploy` и файлы подкачки. Если занятое место изменилось больше чем на 0,5% диска (минимум 256 МБ), замер делается сразу, поэтому короткий всплеск тоже попадает в историю. +- График показывает день, 3, 7 или 30 дней. Каждый столбец показывает самый заполненный момент своего интервала. +- Таблица «What takes the space» в сумме даёт ровно занятое место. Строка «System and other» содержит всё, о чём Docker не сообщает: систему, системные логи, служебные файлы Docker. +- Список «What changed the most» называет конкретные образы, тома и контейнеры, которые выросли или уменьшились с начала периода. +- Все числа в десятичных гигабайтах (1 ГБ = 1000 МБ), как у Docker. Команда `df -h` считает в двоичных, поэтому показывает меньше: 52,7 ГБ здесь и 50G там — один и тот же диск. +- История хранится 30 дней в `/etc/dokploy/monitoring/disk-history/`. Действие **Clean Monitoring** удаляет её вместе с остальными данными мониторинга. + +Блок **Free up space** в той же карточке: + +- удаляет старые образы панели, кроме работающего и предыдущего (он нужен для отката), и показывает их список до удаления; +- по галочке очищает кэш сборки Docker. Это безопасно: ничего не ломается, только следующие сборки приложений идут без кэша и дольше. + +Логи контейнеров панель измеряет временным контейнером `busybox`, которому каталог Docker с контейнерами подключён только для чтения. Если образ `busybox` скачать нельзя, логи попадают в строку «System and other». diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 69a0f66036b..6706464e8c3 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -6,6 +6,8 @@ export * from "./db/validations/domain"; export * from "./db/validations/index"; export * from "./lib/auth"; export * from "./lib/logger"; +export * from "./monitoring/disk-history"; +export * from "./monitoring/disk-usage"; export * from "./monitoring/utils"; export * from "./services/account-deletion"; export * from "./services/admin"; diff --git a/packages/server/src/monitoring/disk-history.ts b/packages/server/src/monitoring/disk-history.ts new file mode 100644 index 00000000000..c9e4e916552 --- /dev/null +++ b/packages/server/src/monitoring/disk-history.ts @@ -0,0 +1,498 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { z } from "zod"; +import { paths } from "../constants"; +import { + DISK_PARTS, + type DiskCategory, + type DiskItemKind, + type DiskItems, + type DiskPart, + type DiskParts, + type DiskStat, + type DiskUsageProbe, + getRootDiskStat, + parseDiskItemKey, + probeDockerUsage, + probeSlowUsage, +} from "./disk-usage"; + +export const DISK_HISTORY_PERIODS = ["1d", "3d", "7d", "30d"] as const; +export type DiskHistoryPeriod = (typeof DISK_HISTORY_PERIODS)[number]; + +export const diskHistoryInputSchema = z.object({ + period: z.enum(DISK_HISTORY_PERIODS), + /** Minutes east of UTC, so that day-long bars start at the viewer's midnight. */ + utcOffsetMinutes: z.number().int().min(-840).max(840).default(0), +}); + +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; +const DAY = 24 * HOUR; + +const PERIOD_BUCKETS: Record< + DiskHistoryPeriod, + { bucketMs: number; count: number } +> = { + "1d": { bucketMs: HOUR, count: 24 }, + "3d": { bucketMs: 2 * HOUR, count: 36 }, + "7d": { bucketMs: 4 * HOUR, count: 42 }, + "30d": { bucketMs: DAY, count: 30 }, +}; + +const STAT_INTERVAL_MS = MINUTE; +const PROBE_INTERVAL_MS = 5 * MINUTE; +const SLOW_PROBE_INTERVAL_MS = HOUR; +const SLOW_PROBE_JUMP_GAP_MS = 10 * MINUTE; +// A probe may take at most 2% of the time, however slow the disk is. +const PROBE_COOLDOWN_FACTOR = 50; +const RETENTION_MS = 30 * DAY; +const PRUNE_INTERVAL_MS = 6 * HOUR; +const JUMP_MIN_BYTES = 256 * 1024 ** 2; +const JUMP_DISK_SHARE = 0.005; +const MOVER_MIN_BYTES = 10 * 1024 ** 2; +const MOVERS_LIMIT = 12; + +export interface DiskSample extends DiskParts { + at: number; + totalBytes: number; + usedBytes: number; +} + +interface DiskItemsSnapshot { + at: number; + items: DiskItems; +} + +export interface DiskUsageBreakdown { + at: number; + totalBytes: number; + usedBytes: number; + /** Null for a part that could not be measured; its bytes are in `other`. */ + parts: Record; +} + +export interface DiskCategoryChange { + category: DiskCategory; + bytes: number | null; + deltaBytes: number | null; +} + +export interface DiskMover { + kind: DiskItemKind | "category"; + /** Item name, or the category key when the category has no items. */ + name: string; + bytes: number; + deltaBytes: number; +} + +export interface DiskUsageHistory { + period: DiskHistoryPeriod; + from: number; + to: number; + bucketMs: number; + current: (DiskUsageBreakdown & { availableBytes: number }) | null; + /** One entry per bar: the fullest sample of that interval, or null without data. */ + buckets: { from: number; sample: DiskUsageBreakdown | null }[]; + /** Time of the sample the changes are measured from. */ + changeSince: number | null; + usedDeltaBytes: number | null; + categories: DiskCategoryChange[]; + movers: DiskMover[]; +} + +/** Used space that changed this much since the last saved sample is worth a new one. */ +export const isDiskUsageJump = ( + previousUsedBytes: number, + usedBytes: number, + totalBytes: number, +) => + Math.abs(usedBytes - previousUsedBytes) >= + Math.max(JUMP_MIN_BYTES, totalBytes * JUMP_DISK_SHARE); + +export const toDiskUsageBreakdown = ( + sample: DiskSample, +): DiskUsageBreakdown => { + const parts = Object.fromEntries( + DISK_PARTS.map((part) => [part, sample[part] ?? null]), + ) as Record; + const known = Object.values(parts).reduce( + (total, bytes) => total + (bytes ?? 0), + 0, + ); + return { + at: sample.at, + totalBytes: sample.totalBytes, + usedBytes: sample.usedBytes, + // Docker's data may live on another disk; the rest cannot go below zero. + parts: { ...parts, other: Math.max(sample.usedBytes - known, 0) }, + }; +}; + +/** The bars of a period, aligned to the viewer's clock; the last one holds `now`. */ +export const getDiskHistoryRange = ( + period: DiskHistoryPeriod, + now: number, + utcOffsetMinutes: number, +) => { + const { bucketMs, count } = PERIOD_BUCKETS[period]; + const offsetMs = utcOffsetMinutes * MINUTE; + const lastBucketFrom = + Math.floor((now + offsetMs) / bucketMs) * bucketMs - offsetMs; + return { + from: lastBucketFrom - (count - 1) * bucketMs, + to: lastBucketFrom + bucketMs, + bucketMs, + count, + }; +}; + +/** Keeps the fullest sample of each bar, so a short spike stays visible. */ +export const bucketDiskSamples = ( + samples: DiskSample[], + range: ReturnType, +) => { + const fullest: (DiskSample | null)[] = Array.from( + { length: range.count }, + () => null, + ); + for (const sample of samples) { + const index = Math.floor((sample.at - range.from) / range.bucketMs); + if (index < 0 || index >= range.count) { + continue; + } + const current = fullest[index]; + if (!current || sample.usedBytes >= current.usedBytes) { + fullest[index] = sample; + } + } + return fullest.map((sample, index) => ({ + from: range.from + index * range.bucketMs, + sample: sample ? toDiskUsageBreakdown(sample) : null, + })); +}; + +export const getDiskCategoryChanges = ( + start: DiskUsageBreakdown | null, + end: DiskUsageBreakdown, +): DiskCategoryChange[] => + ([...DISK_PARTS, "other"] as DiskCategory[]).map((category) => { + const bytes = end.parts[category]; + const startBytes = start?.parts[category] ?? null; + return { + category, + bytes, + deltaBytes: + bytes !== null && startBytes !== null ? bytes - startBytes : null, + }; + }); + +// Parts without single items; each one is listed among the movers as a whole. +const WHOLE_CATEGORY_MOVERS: DiskCategory[] = ["buildCache", "swap", "other"]; + +/** What grew or shrank the most between two moments, biggest growth first. */ +export const getDiskMovers = ( + startItems: DiskItems, + endItems: DiskItems, + categories: DiskCategoryChange[], +): DiskMover[] => { + const movers: DiskMover[] = []; + for (const key of new Set([ + ...Object.keys(startItems), + ...Object.keys(endItems), + ])) { + const bytes = endItems[key] ?? 0; + movers.push({ + ...parseDiskItemKey(key), + bytes, + deltaBytes: bytes - (startItems[key] ?? 0), + }); + } + for (const change of categories) { + if ( + WHOLE_CATEGORY_MOVERS.includes(change.category) && + change.bytes !== null && + change.deltaBytes !== null + ) { + movers.push({ + kind: "category", + name: change.category, + bytes: change.bytes, + deltaBytes: change.deltaBytes, + }); + } + } + return movers + .filter((mover) => Math.abs(mover.deltaBytes) >= MOVER_MIN_BYTES) + .sort((a, b) => b.deltaBytes - a.deltaBytes) + .slice(0, MOVERS_LIMIT); +}; + +const getHistoryFiles = () => { + const directory = path.join(paths().MONITORING_PATH, "disk-history"); + return { + directory, + samples: path.join(directory, "samples.jsonl"), + items: path.join(directory, "items.jsonl"), + }; +}; + +const readJsonLines = async ( + file: string, + since = 0, +): Promise => { + const content = await fs.readFile(file, "utf-8").catch(() => ""); + const rows: T[] = []; + for (const line of content.split("\n")) { + if (!line) { + continue; + } + try { + const row = JSON.parse(line) as T; + if (row.at >= since) { + rows.push(row); + } + } catch { + // A line cut short by a restart or a full disk. + } + } + return rows; +}; + +const appendJsonLine = async ( + file: string, + row: T, +) => { + await fs.mkdir(path.dirname(file), { recursive: true }); + // A full disk can cut the previous line short; the leading newline keeps + // this row from being glued to it and lost with it. + await fs.appendFile(file, `\n${JSON.stringify(row)}\n`); +}; + +const pruneJsonLines = async (file: string, since: number) => { + const content = await fs.readFile(file, "utf-8").catch(() => null); + if (content === null) { + return; + } + const kept = content.split("\n").filter((line) => { + const at = Number(line.match(/^\{"at":(\d+)/)?.[1]); + return Number.isFinite(at) && at >= since; + }); + // Written next to the file and renamed, so a crash cannot leave half of it. + const temporary = `${file}.tmp`; + await fs.writeFile(temporary, kept.length > 0 ? `${kept.join("\n")}\n` : ""); + await fs.rename(temporary, file); +}; + +interface ProbeRun { + at: number; + durationMs: number; +} + +interface SamplerState { + stat: (DiskStat & { at: number }) | null; + parts: DiskParts; + items: DiskItems; + /** Last sample written to disk; jumps are measured against it. */ + savedSample: DiskSample | null; + probe: ProbeRun | null; + slowProbe: ProbeRun | null; + prunedAt: number; + running: Promise | null; + timer: ReturnType | null; +} + +// Like the Docker client: the module is evaluated once per bundled copy, and +// two samplers would write every sample twice. +const globalForDiskHistory = globalThis as unknown as { + diskHistorySampler?: SamplerState; +}; + +if (!globalForDiskHistory.diskHistorySampler) { + globalForDiskHistory.diskHistorySampler = { + stat: null, + parts: {}, + items: {}, + savedSample: null, + probe: null, + slowProbe: null, + prunedAt: 0, + running: null, + timer: null, + }; +} + +const state = globalForDiskHistory.diskHistorySampler; + +const isProbeDue = (last: ProbeRun | null, now: number, intervalMs: number) => { + if (!last) { + return true; + } + // The timer fires a few milliseconds early or late; without the slack a + // probe due on this tick would wait for the next one. + const elapsed = now - last.at + STAT_INTERVAL_MS / 2; + return ( + elapsed >= intervalMs && elapsed >= last.durationMs * PROBE_COOLDOWN_FACTOR + ); +}; + +const applyProbe = (probe: DiskUsageProbe) => { + state.parts = { ...state.parts, ...probe.parts }; + const kept = Object.entries(state.items).filter( + ([key]) => !probe.itemKinds.includes(parseDiskItemKey(key).kind), + ); + state.items = { ...Object.fromEntries(kept), ...probe.items }; +}; + +const timeProbe = async ( + now: number, + run: () => Promise, +): Promise => { + const startedAt = Date.now(); + applyProbe(await run()); + return { at: now, durationMs: Date.now() - startedAt }; +}; + +const sampleDiskUsage = async (now: number, force: boolean) => { + const stat = await getRootDiskStat(); + state.stat = { ...stat, at: now }; + + const saved = state.savedSample; + const jumped = + !!saved && + isDiskUsageJump(saved.usedBytes, stat.usedBytes, stat.totalBytes); + const probeDue = + force || + isProbeDue(state.probe, now, jumped ? STAT_INTERVAL_MS : PROBE_INTERVAL_MS); + if (!probeDue) { + return; + } + const slowProbeDue = isProbeDue( + state.slowProbe, + now, + jumped ? SLOW_PROBE_JUMP_GAP_MS : SLOW_PROBE_INTERVAL_MS, + ); + + state.probe = await timeProbe(now, probeDockerUsage); + if (slowProbeDue) { + state.slowProbe = await timeProbe(now, probeSlowUsage); + } + + const files = getHistoryFiles(); + const sample: DiskSample = { + at: now, + totalBytes: stat.totalBytes, + usedBytes: stat.usedBytes, + ...state.parts, + }; + await appendJsonLine(files.samples, sample); + state.savedSample = sample; + if (slowProbeDue) { + // Saved together with a sample, so both describe the same moment. + await appendJsonLine(files.items, { at: now, items: state.items }); + } + + if (now - state.prunedAt >= PRUNE_INTERVAL_MS) { + state.prunedAt = now; + await pruneJsonLines(files.samples, now - RETENTION_MS); + await pruneJsonLines(files.items, now - RETENTION_MS); + } +}; + +/** + * Reads the disk and, when it is time, Docker's usage, and saves a sample. With + * `force` the Docker probe runs now, e.g. right after a cleanup. + */ +export const refreshDiskUsage = async ({ force = false } = {}) => { + // A forced refresh must not settle for a run that started before the cleanup. + if (state.running && force) { + await state.running.catch(() => {}); + } + state.running ??= sampleDiskUsage(Date.now(), force).finally(() => { + state.running = null; + }); + await state.running; +}; + +/** Samples the disk every minute for as long as the process lives. */ +export const startDiskHistorySampler = () => { + if (state.timer) { + return; + } + const tick = () => { + refreshDiskUsage().catch((error) => { + console.error("Disk usage: sampling failed", error); + }); + }; + state.timer = setInterval(tick, STAT_INTERVAL_MS); + state.timer.unref(); + tick(); +}; + +const getCurrentSample = async (now: number): Promise => { + // Without the sampler (development) a query is what keeps the numbers fresh. + if (!state.stat || now - state.stat.at > 2 * STAT_INTERVAL_MS) { + await refreshDiskUsage().catch((error) => { + console.error("Disk usage: sampling failed", error); + }); + } + if (!state.stat) { + return null; + } + return { + at: state.stat.at, + totalBytes: state.stat.totalBytes, + usedBytes: state.stat.usedBytes, + ...state.parts, + }; +}; + +export const getDiskUsageHistory = async ( + period: DiskHistoryPeriod, + utcOffsetMinutes = 0, + now = Date.now(), +): Promise => { + const currentSample = await getCurrentSample(now); + const files = getHistoryFiles(); + const range = getDiskHistoryRange(period, now, utcOffsetMinutes); + const [samples, snapshots] = await Promise.all([ + readJsonLines(files.samples, range.from), + readJsonLines(files.items, range.from), + ]); + const buckets = bucketDiskSamples( + currentSample ? [...samples, currentSample] : samples, + range, + ); + + const current = currentSample ? toDiskUsageBreakdown(currentSample) : null; + // Item sizes are saved less often than samples; starting from the first + // saved set keeps the category changes and the movers on the same moment. + const startItems = snapshots[0]; + const startSample = + samples.find((sample) => sample.at === startItems?.at) ?? samples[0]; + const start = + startSample && current && startSample.at < current.at + ? toDiskUsageBreakdown(startSample) + : null; + const categories = current ? getDiskCategoryChanges(start, current) : []; + + return { + period, + from: range.from, + to: range.to, + bucketMs: range.bucketMs, + current: + current && state.stat + ? { ...current, availableBytes: state.stat.availableBytes } + : null, + buckets, + changeSince: start?.at ?? null, + usedDeltaBytes: + start && current ? current.usedBytes - start.usedBytes : null, + categories, + movers: + start && startItems?.at === start.at + ? getDiskMovers(startItems.items, state.items, categories) + : [], + }; +}; diff --git a/packages/server/src/monitoring/disk-usage.ts b/packages/server/src/monitoring/disk-usage.ts new file mode 100644 index 00000000000..db1acf09ac8 --- /dev/null +++ b/packages/server/src/monitoring/disk-usage.ts @@ -0,0 +1,381 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { docker, paths } from "../constants"; +import { quoteShellArg } from "../utils/filesystem/safe-path"; +import { execAsync } from "../utils/process/execAsync"; + +/** What the disk usage is split into; the rest of the used space is "other". */ +export const DISK_PARTS = [ + "images", + "buildCache", + "volumes", + "containerLogs", + "containers", + "dokployData", + "swap", +] as const; + +export type DiskPart = (typeof DISK_PARTS)[number]; +export type DiskCategory = DiskPart | "other"; +export type DiskParts = Partial>; + +export type DiskItemKind = + | "image" + | "volume" + | "container" + | "containerLog" + | "dokployData"; + +/** Sizes of single images, volumes and so on, keyed by `:`. */ +export type DiskItems = Record; + +export interface DiskUsageProbe { + parts: DiskParts; + items: DiskItems; + /** Kinds this probe measured; items of other kinds keep their last value. */ + itemKinds: DiskItemKind[]; +} + +export interface DiskStat { + totalBytes: number; + usedBytes: number; + availableBytes: number; +} + +const PROBE_TIMEOUT_MS = 60_000; +const ANONYMOUS_VOLUME_NAME = /^[a-f0-9]{64}$/; +const CONTAINER_LOGS_MOUNT = "/docker-containers"; + +export const diskItemKey = (kind: DiskItemKind, name: string) => + `${kind}:${name}`; + +export const parseDiskItemKey = (key: string) => { + const separator = key.indexOf(":"); + return { + kind: key.slice(0, separator) as DiskItemKind, + name: key.slice(separator + 1), + }; +}; + +// The same numbers `df /` prints. Dokploy's container root is an overlay on the +// host disk, so this is the host's disk even without /var/lib/docker mounted. +export const getRootDiskStat = async (): Promise => { + const stat = await fs.statfs("/"); + return { + totalBytes: stat.blocks * stat.bsize, + usedBytes: (stat.blocks - stat.bfree) * stat.bsize, + availableBytes: stat.bavail * stat.bsize, + }; +}; + +interface DockerUsageSection { + TotalSize?: number; + Items?: T[] | null; +} + +interface DockerImageUsage { + Id?: string; + RepoTags?: string[] | null; + RepoDigests?: string[] | null; + Size?: number; + SharedSize?: number; +} + +interface DockerContainerUsage { + Id?: string; + Names?: string[] | null; + Labels?: Record | null; + SizeRw?: number; +} + +interface DockerVolumeUsage { + Name?: string; + UsageData?: { Size?: number } | null; +} + +interface DockerBuildCacheUsage { + Size?: number; + Shared?: boolean; +} + +// Docker 29 reports each section twice: the legacy top-level lists and the +// `*Usage` objects that replace them in later API versions. +export interface DockerDiskUsageResponse { + LayersSize?: number; + Images?: DockerImageUsage[] | null; + Containers?: DockerContainerUsage[] | null; + Volumes?: DockerVolumeUsage[] | null; + BuildCache?: DockerBuildCacheUsage[] | null; + ImageUsage?: DockerUsageSection | null; + ContainerUsage?: DockerUsageSection | null; + VolumeUsage?: DockerUsageSection | null; + BuildCacheUsage?: DockerUsageSection | null; +} + +type DockerUsageType = "image" | "container" | "volume" | "build-cache"; + +const getImageRepository = (image: DockerImageUsage) => { + const reference = + image.RepoTags?.find((tag) => !tag.startsWith("")) ?? + image.RepoDigests?.[0]; + if (!reference) { + return "untagged"; + } + const withoutDigest = reference.split("@")[0] ?? reference; + const lastSlash = withoutDigest.lastIndexOf("/"); + const lastColon = withoutDigest.lastIndexOf(":"); + // A colon before the last slash is a registry port, not a tag. + return lastColon > lastSlash + ? withoutDigest.slice(0, lastColon) + : withoutDigest; +}; + +// Swarm gives every task its own container name, so a redeploy would look like +// one container vanishing and another one appearing. +const getContainerName = (container: DockerContainerUsage) => + container.Labels?.["com.docker.swarm.service.name"] ?? + container.Names?.[0]?.replace(/^\//, "") ?? + container.Id?.slice(0, 12) ?? + "unknown"; + +const addBytes = (items: DiskItems, key: string, bytes: number) => { + items[key] = (items[key] ?? 0) + bytes; +}; + +const sum = (values: number[]) => + values.reduce((total, value) => total + value, 0); + +/** + * Turns a `GET /system/df` answer into part totals and per-item sizes. Only the + * sections Docker was asked for are present, so the others stay out of the result. + */ +export const summarizeDockerDiskUsage = ( + usage: DockerDiskUsageResponse, +): DiskUsageProbe => { + const parts: DiskParts = {}; + const items: DiskItems = {}; + const itemKinds: DiskItemKind[] = []; + + const images = usage.ImageUsage?.Items ?? usage.Images; + if (images) { + itemKinds.push("image"); + for (const image of images) { + const size = image.Size ?? 0; + const shared = image.SharedSize ?? -1; + // Docker answers -1 when it did not work out the shared part. + const unique = shared >= 0 ? Math.max(size - shared, 0) : size; + addBytes(items, diskItemKey("image", getImageRepository(image)), unique); + } + parts.images = + usage.ImageUsage?.TotalSize ?? + usage.LayersSize ?? + sum(images.map((image) => image.Size ?? 0)); + } + + const containers = usage.ContainerUsage?.Items ?? usage.Containers; + if (containers) { + itemKinds.push("container"); + for (const container of containers) { + addBytes( + items, + diskItemKey("container", getContainerName(container)), + container.SizeRw ?? 0, + ); + } + parts.containers = + usage.ContainerUsage?.TotalSize ?? + sum(containers.map((container) => container.SizeRw ?? 0)); + } + + const volumes = usage.VolumeUsage?.Items ?? usage.Volumes; + if (volumes) { + itemKinds.push("volume"); + for (const volume of volumes) { + const name = volume.Name ?? "unknown"; + addBytes( + items, + diskItemKey( + "volume", + ANONYMOUS_VOLUME_NAME.test(name) ? "anonymous volumes" : name, + ), + Math.max(volume.UsageData?.Size ?? 0, 0), + ); + } + parts.volumes = + usage.VolumeUsage?.TotalSize ?? + sum(volumes.map((volume) => Math.max(volume.UsageData?.Size ?? 0, 0))); + } + + const buildCache = usage.BuildCacheUsage?.Items ?? usage.BuildCache; + if (buildCache) { + // Shared records are image layers, and the images part already counts them. + parts.buildCache = sum( + buildCache + .filter((record) => !record.Shared) + .map((record) => record.Size ?? 0), + ); + } + + return { parts, items, itemKinds }; +}; + +const requestDockerDiskUsage = (types: DockerUsageType[]) => + new Promise((resolve, reject) => { + docker.modem.dial( + { + // The modem appends the query only to a path that ends with "?". + path: "/system/df?", + method: "GET", + options: { type: types }, + statusCodes: { 200: true, 500: "server error" }, + }, + (error, data) => { + if (error) { + reject(error); + return; + } + resolve((data ?? {}) as DockerDiskUsageResponse); + }, + ); + }); + +/** Parses `du -k` output into bytes per path. */ +export const parseDuOutput = (stdout: string) => { + const sizes = new Map(); + for (const line of stdout.split("\n")) { + const match = line.match(/^(\d+)\s+(.+)$/); + if (match) { + sizes.set(match[2] as string, Number(match[1]) * 1024); + } + } + return sizes; +}; + +/** Bytes of the swap files listed in /proc/swaps; swap partitions take no disk space. */ +export const parseSwapFileBytes = (procSwaps: string) => + sum( + procSwaps + .split("\n") + .slice(1) + .map((line) => line.trim().split(/\s+/)) + // The file name may hold spaces, so the columns are read from the end. + .filter((columns) => columns.at(-4) === "file") + .map((columns) => Number(columns.at(-3)) * 1024) + .filter((bytes) => Number.isFinite(bytes)), + ); + +const mergeProbes = (probes: DiskUsageProbe[]): DiskUsageProbe => ({ + parts: Object.assign({}, ...probes.map((probe) => probe.parts)), + items: Object.assign({}, ...probes.map((probe) => probe.items)), + itemKinds: probes.flatMap((probe) => probe.itemKinds), +}); + +const emptyProbe = (): DiskUsageProbe => ({ + parts: {}, + items: {}, + itemKinds: [], +}); + +const logProbeFailure = (what: string) => (error: unknown) => { + console.error(`Disk usage: could not read ${what}`, error); + return emptyProbe(); +}; + +// Docker keeps container logs under its root directory, which Dokploy's own +// container does not see, so a throwaway container measures them. +const probeContainerLogs = async (): Promise => { + const [{ DockerRootDir }, containers] = await Promise.all([ + docker.info() as Promise<{ DockerRootDir?: string }>, + docker.listContainers({ all: true }), + ]); + if (!DockerRootDir) { + return emptyProbe(); + } + const mount = `${DockerRootDir}/containers:${CONTAINER_LOGS_MOUNT}:ro`; + const { stdout } = await execAsync( + `docker run --rm --network none -v ${quoteShellArg(mount)} busybox du -k -d 1 ${CONTAINER_LOGS_MOUNT}`, + { timeoutMs: PROBE_TIMEOUT_MS }, + ); + + const names = new Map( + containers.map((container) => [container.Id, getContainerName(container)]), + ); + const items: DiskItems = {}; + let total = 0; + for (const [directory, bytes] of parseDuOutput(stdout)) { + if (directory === CONTAINER_LOGS_MOUNT) { + total = bytes; + continue; + } + const id = path.basename(directory); + addBytes( + items, + diskItemKey("containerLog", names.get(id) ?? id.slice(0, 12)), + bytes, + ); + } + return { + parts: { containerLogs: total }, + items, + itemKinds: ["containerLog"], + }; +}; + +const probeDokployData = async (): Promise => { + const { BASE_PATH } = paths(); + // du exits non-zero for a single unreadable file but still prints the sizes. + const { stdout } = await execAsync( + `du -k -x -d 1 ${quoteShellArg(BASE_PATH)} 2>/dev/null || true`, + { timeoutMs: PROBE_TIMEOUT_MS }, + ); + const sizes = parseDuOutput(stdout); + const total = sizes.get(BASE_PATH); + if (total === undefined) { + return emptyProbe(); + } + const items: DiskItems = {}; + for (const [directory, bytes] of sizes) { + if (directory !== BASE_PATH) { + items[diskItemKey("dokployData", path.basename(directory))] = bytes; + } + } + return { + parts: { dokployData: total }, + items, + itemKinds: ["dokployData"], + }; +}; + +const probeSwapFiles = async (): Promise => { + const procSwaps = await fs.readFile("/proc/swaps", "utf-8").catch(() => null); + if (procSwaps === null) { + return emptyProbe(); + } + return { + parts: { swap: parseSwapFileBytes(procSwaps) }, + items: {}, + itemKinds: [], + }; +}; + +/** Images, container layers and build cache: Docker answers from its metadata. */ +export const probeDockerUsage = async (): Promise => + requestDockerDiskUsage(["image", "container", "build-cache"]) + .then(summarizeDockerDiskUsage) + .catch(logProbeFailure("Docker disk usage")); + +/** + * Everything that walks files on disk: volumes (Docker sizes each one with a + * directory walk), container logs, Dokploy's own data and swap files. + */ +export const probeSlowUsage = async (): Promise => + mergeProbes( + await Promise.all([ + requestDockerDiskUsage(["volume"]) + .then(summarizeDockerDiskUsage) + .catch(logProbeFailure("Docker volume sizes")), + probeContainerLogs().catch(logProbeFailure("container log sizes")), + probeDokployData().catch(logProbeFailure("Dokploy data size")), + probeSwapFiles().catch(logProbeFailure("swap files")), + ]), + ); diff --git a/packages/server/src/services/settings.ts b/packages/server/src/services/settings.ts index 570831e4825..08f042509c0 100644 --- a/packages/server/src/services/settings.ts +++ b/packages/server/src/services/settings.ts @@ -553,6 +553,8 @@ export interface UpdateDiskSpace { requiredBytes: number | null; oldImageCount: number; oldImageBytes: number; + /** The old builds a cleanup removes, newest first. */ + oldImages: DokployImageInfo[]; buildCacheBytes: number; } @@ -616,6 +618,7 @@ export const getUpdateDiskSpace = async (): Promise => { (sum, image) => sum + (image.uniqueSizeBytes ?? image.sizeBytes), 0, ), + oldImages: removable, buildCacheBytes, }; };