diff --git a/apps/dokploy/__test__/monitoring/disk-history.test.ts b/apps/dokploy/__test__/monitoring/disk-history.test.ts new file mode 100644 index 00000000000..b854eea59dc --- /dev/null +++ b/apps/dokploy/__test__/monitoring/disk-history.test.ts @@ -0,0 +1,448 @@ +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import type { DiskUsageProbe } from "@dokploy/server/monitoring/disk-usage"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + monitoringPath: "", + getRootDiskStat: vi.fn(), + probeDockerUsage: vi.fn(), + probeSlowUsage: vi.fn(), +})); + +vi.mock("@dokploy/server/constants", () => ({ + docker: {}, + paths: () => ({ MONITORING_PATH: mocks.monitoringPath }), +})); + +vi.mock("@dokploy/server/monitoring/disk-usage", async (importOriginal) => ({ + ...(await importOriginal< + typeof import("@dokploy/server/monitoring/disk-usage") + >()), + getRootDiskStat: mocks.getRootDiskStat, + probeDockerUsage: mocks.probeDockerUsage, + probeSlowUsage: mocks.probeSlowUsage, +})); + +type DiskHistoryModule = + typeof import("@dokploy/server/monitoring/disk-history"); + +const GB = 1e9; +const MB = 1e6; +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; +const TOTAL = 50 * GB; +const START = Date.UTC(2026, 9, 13, 12, 0, 0); + +const diskUsed = (usedBytes: number) => { + mocks.getRootDiskStat.mockResolvedValue({ + totalBytes: TOTAL, + usedBytes, + availableBytes: TOTAL - usedBytes, + }); +}; + +const dockerProbe = ( + images: number, + buildCache: number, + items: DiskUsageProbe["items"] = {}, +): DiskUsageProbe => ({ + parts: { images, buildCache, containers: 0 }, + items, + itemKinds: ["image", "container"], +}); + +const slowProbe = ( + volumes: number, + items: DiskUsageProbe["items"] = {}, +): DiskUsageProbe => ({ + parts: { volumes, containerLogs: 0, dokployData: 0, swap: 2 * GB }, + items, + itemKinds: ["volume", "containerLog", "dokployData"], +}); + +const readLines = async (file: string) => + ( + await readFile( + path.join(mocks.monitoringPath, "disk-history", file), + "utf-8", + ) + ) + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line)); + +describe("disk history sampler", () => { + let history: DiskHistoryModule; + + beforeEach(async () => { + mocks.monitoringPath = await mkdtemp(path.join(tmpdir(), "disk-history-")); + vi.clearAllMocks(); + vi.useFakeTimers({ toFake: ["Date"] }); + vi.setSystemTime(START); + // The sampler keeps its state on globalThis so that bundled copies share it. + delete (globalThis as { diskHistorySampler?: unknown }).diskHistorySampler; + vi.resetModules(); + history = await import("@dokploy/server/monitoring/disk-history"); + + diskUsed(30 * GB); + mocks.probeDockerUsage.mockResolvedValue(dockerProbe(10 * GB, 5 * GB)); + mocks.probeSlowUsage.mockResolvedValue(slowProbe(8 * GB)); + }); + + afterEach(async () => { + vi.useRealTimers(); + await rm(mocks.monitoringPath, { recursive: true, force: true }); + }); + + it("saves a sample with every part and the item sizes on the first run", async () => { + mocks.probeDockerUsage.mockResolvedValue( + dockerProbe(10 * GB, 5 * GB, { "image:web": 4 * GB }), + ); + mocks.probeSlowUsage.mockResolvedValue( + slowProbe(8 * GB, { "volume:pg-data": 8 * GB }), + ); + + await history.refreshDiskUsage(); + + expect(await readLines("samples.jsonl")).toEqual([ + { + at: START, + totalBytes: TOTAL, + usedBytes: 30 * GB, + images: 10 * GB, + buildCache: 5 * GB, + containers: 0, + volumes: 8 * GB, + containerLogs: 0, + dokployData: 0, + swap: 2 * GB, + }, + ]); + expect(await readLines("items.jsonl")).toEqual([ + { + at: START, + items: { "image:web": 4 * GB, "volume:pg-data": 8 * GB }, + }, + ]); + }); + + it("asks Docker every five minutes and walks the disk every hour", async () => { + await history.refreshDiskUsage(); + + for (let minute = 1; minute <= 60; minute++) { + vi.setSystemTime(START + minute * MINUTE); + await history.refreshDiskUsage(); + } + + expect(mocks.getRootDiskStat).toHaveBeenCalledTimes(61); + expect(mocks.probeDockerUsage).toHaveBeenCalledTimes(13); + expect(mocks.probeSlowUsage).toHaveBeenCalledTimes(2); + expect(await readLines("samples.jsonl")).toHaveLength(13); + expect(await readLines("items.jsonl")).toHaveLength(2); + }); + + it("samples right away when the used space jumps", async () => { + await history.refreshDiskUsage(); + + vi.setSystemTime(START + MINUTE); + diskUsed(30 * GB + 100 * MB); + await history.refreshDiskUsage(); + expect(mocks.probeDockerUsage).toHaveBeenCalledTimes(1); + + vi.setSystemTime(START + 2 * MINUTE); + diskUsed(33 * GB); + mocks.probeDockerUsage.mockResolvedValue(dockerProbe(13 * GB, 5 * GB)); + await history.refreshDiskUsage(); + + expect(mocks.probeDockerUsage).toHaveBeenCalledTimes(2); + // The disk was walked two minutes ago; a jump repeats that after ten. + expect(mocks.probeSlowUsage).toHaveBeenCalledTimes(1); + expect((await readLines("samples.jsonl")).at(-1)).toMatchObject({ + at: START + 2 * MINUTE, + usedBytes: 33 * GB, + images: 13 * GB, + volumes: 8 * GB, + }); + }); + + it("waits fifty times as long as a slow probe took", async () => { + mocks.probeDockerUsage.mockImplementation(async () => { + vi.setSystemTime(Date.now() + 20_000); + return dockerProbe(10 * GB, 5 * GB); + }); + await history.refreshDiskUsage(); + + for (let minute = 1; minute <= 16; minute++) { + vi.setSystemTime(START + minute * MINUTE); + await history.refreshDiskUsage(); + } + expect(mocks.probeDockerUsage).toHaveBeenCalledTimes(1); + + vi.setSystemTime(START + 17 * MINUTE); + await history.refreshDiskUsage(); + expect(mocks.probeDockerUsage).toHaveBeenCalledTimes(2); + }); + + it("keeps the last known value of a part that a probe could not read", async () => { + await history.refreshDiskUsage(); + + vi.setSystemTime(START + 5 * MINUTE); + mocks.probeDockerUsage.mockResolvedValue({ + parts: {}, + items: {}, + itemKinds: [], + }); + await history.refreshDiskUsage(); + + expect((await readLines("samples.jsonl")).at(-1)).toMatchObject({ + at: START + 5 * MINUTE, + images: 10 * GB, + buildCache: 5 * GB, + }); + }); + + it("probes Docker after a cleanup even though it just did", async () => { + await history.refreshDiskUsage(); + + vi.setSystemTime(START + 10_000); + diskUsed(25 * GB); + mocks.probeDockerUsage.mockResolvedValue(dockerProbe(10 * GB, 0)); + await history.refreshDiskUsage({ force: true }); + + expect((await readLines("samples.jsonl")).at(-1)).toMatchObject({ + usedBytes: 25 * GB, + buildCache: 0, + }); + }); + + it("drops samples older than thirty days", async () => { + await history.refreshDiskUsage(); + const samplesFile = path.join( + mocks.monitoringPath, + "disk-history", + "samples.jsonl", + ); + const old = { at: START - 31 * 24 * HOUR, totalBytes: TOTAL, usedBytes: 1 }; + const recent = { at: START - 24 * HOUR, totalBytes: TOTAL, usedBytes: 2 }; + await writeFile( + samplesFile, + `${JSON.stringify(old)}\n${JSON.stringify(recent)}\n{"at":12`, + ); + + vi.setSystemTime(START + 6 * HOUR); + await history.refreshDiskUsage(); + + expect( + (await readLines("samples.jsonl")).map((sample) => sample.at), + ).toEqual([recent.at, START + 6 * HOUR]); + }); + + it("reports the period, what takes the space and what grew", async () => { + mocks.probeDockerUsage.mockResolvedValue( + dockerProbe(10 * GB, 5 * GB, { + "image:web": 4 * GB, + "image:api": 6 * GB, + }), + ); + mocks.probeSlowUsage.mockResolvedValue( + slowProbe(8 * GB, { "volume:pg-data": 8 * GB }), + ); + await history.refreshDiskUsage(); + + vi.setSystemTime(START + 3 * HOUR); + diskUsed(38 * GB); + mocks.probeDockerUsage.mockResolvedValue( + dockerProbe(13 * GB, 9 * GB, { + "image:web": 7 * GB, + "image:api": 6 * GB + 1 * MB, + }), + ); + mocks.probeSlowUsage.mockResolvedValue( + slowProbe(9 * GB, { "volume:pg-data": 9 * GB }), + ); + await history.refreshDiskUsage(); + + const result = await history.getDiskUsageHistory("1d", 180); + + expect(result.current).toMatchObject({ + usedBytes: 38 * GB, + totalBytes: TOTAL, + availableBytes: 12 * GB, + parts: { + images: 13 * GB, + buildCache: 9 * GB, + volumes: 9 * GB, + swap: 2 * GB, + other: 5 * GB, + }, + }); + expect(result.changeSince).toBe(START); + expect(result.usedDeltaBytes).toBe(8 * GB); + expect( + Object.fromEntries( + result.categories.map((row) => [row.category, row.deltaBytes]), + ), + ).toMatchObject({ + images: 3 * GB, + buildCache: 4 * GB, + volumes: 1 * GB, + swap: 0, + other: 0, + }); + // Build cache has no items, so it is listed as a whole; the 1 MB of + // "api" is below what counts as a change. + expect(result.movers).toEqual([ + { + kind: "category", + name: "buildCache", + bytes: 9 * GB, + deltaBytes: 4 * GB, + }, + { kind: "image", name: "web", bytes: 7 * GB, deltaBytes: 3 * GB }, + { kind: "volume", name: "pg-data", bytes: 9 * GB, deltaBytes: 1 * GB }, + ]); + + expect(result.buckets).toHaveLength(24); + expect(result.bucketMs).toBe(HOUR); + const filled = result.buckets.filter((bucket) => bucket.sample); + expect(filled.map((bucket) => bucket.from)).toEqual([ + START, + START + 3 * HOUR, + ]); + expect(result.to).toBe(START + 4 * HOUR); + }); + + it("has no changes to report while there is a single sample", async () => { + const result = await history.getDiskUsageHistory("7d"); + + expect(mocks.probeDockerUsage).toHaveBeenCalledTimes(1); + expect(result.current?.usedBytes).toBe(30 * GB); + expect(result.changeSince).toBeNull(); + expect(result.usedDeltaBytes).toBeNull(); + expect(result.movers).toEqual([]); + expect(result.categories.every((row) => row.deltaBytes === null)).toBe( + true, + ); + }); + + it("answers without data when the disk cannot be read", async () => { + mocks.getRootDiskStat.mockRejectedValue(new Error("no such device")); + vi.spyOn(console, "error").mockImplementation(() => {}); + + const result = await history.getDiskUsageHistory("1d"); + + expect(result.current).toBeNull(); + expect(result.categories).toEqual([]); + expect(result.buckets.every((bucket) => bucket.sample === null)).toBe(true); + }); +}); + +describe("disk history maths", () => { + let history: DiskHistoryModule; + + beforeEach(async () => { + history = await import("@dokploy/server/monitoring/disk-history"); + }); + + it("treats half a percent of the disk, but at least 256 MiB, as a jump", () => { + const { isDiskUsageJump } = history; + + expect(isDiskUsageJump(10 * GB, 10 * GB + 200 * MB, 20 * GB)).toBe(false); + expect(isDiskUsageJump(10 * GB, 10 * GB + 300 * MB, 20 * GB)).toBe(true); + expect(isDiskUsageJump(10 * GB, 10 * GB - 300 * MB, 20 * GB)).toBe(true); + expect(isDiskUsageJump(100 * GB, 104 * GB, 1000 * GB)).toBe(false); + expect(isDiskUsageJump(100 * GB, 106 * GB, 1000 * GB)).toBe(true); + }); + + it("puts the space no part explains into other, never below zero", () => { + const { toDiskUsageBreakdown } = history; + + expect( + toDiskUsageBreakdown({ + at: 1, + totalBytes: TOTAL, + usedBytes: 10 * GB, + images: 6 * GB, + }).parts, + ).toEqual({ + images: 6 * GB, + buildCache: null, + volumes: null, + containerLogs: null, + containers: null, + dokployData: null, + swap: null, + other: 4 * GB, + }); + expect( + toDiskUsageBreakdown({ + at: 1, + totalBytes: TOTAL, + usedBytes: 10 * GB, + images: 12 * GB, + }).parts.other, + ).toBe(0); + }); + + it("starts day-long bars at the viewer's midnight", () => { + const { getDiskHistoryRange } = history; + // 13 Oct 2026 22:30 UTC is 14 Oct 01:30 in UTC+3. + const now = Date.UTC(2026, 9, 13, 22, 30); + + const range = getDiskHistoryRange("30d", now, 180); + + expect(new Date(range.to).toISOString()).toBe("2026-10-14T21:00:00.000Z"); + expect(new Date(range.from).toISOString()).toBe("2026-09-14T21:00:00.000Z"); + expect(range.count).toBe(30); + }); + + it("keeps the fullest sample of every bar", () => { + const { bucketDiskSamples, getDiskHistoryRange } = history; + const range = getDiskHistoryRange("1d", START + 30 * MINUTE, 0); + const sample = (at: number, usedBytes: number) => ({ + at, + totalBytes: TOTAL, + usedBytes, + }); + + const buckets = bucketDiskSamples( + [ + sample(START - 2 * 24 * HOUR, 49 * GB), + sample(START - HOUR + 5 * MINUTE, 20 * GB), + sample(START - HOUR + 10 * MINUTE, 45 * GB), + sample(START - HOUR + 15 * MINUTE, 21 * GB), + sample(START + 5 * MINUTE, 22 * GB), + ], + range, + ); + + expect(buckets).toHaveLength(24); + expect(buckets.at(-2)?.sample?.usedBytes).toBe(45 * GB); + expect(buckets.at(-1)).toMatchObject({ + from: START, + sample: { usedBytes: 22 * GB }, + }); + expect(buckets.slice(0, -2).every((bucket) => bucket.sample === null)).toBe( + true, + ); + }); + + it("lists what shrank after what grew, and what vanished as freed", () => { + const { getDiskMovers } = history; + + expect( + getDiskMovers( + { "image:old": 2 * GB, "volume:data": 1 * GB }, + { "volume:data": 1.5 * GB, "containerLog:web": 3 * GB }, + [{ category: "other", bytes: 5 * GB, deltaBytes: -1 * GB }], + ), + ).toEqual([ + { kind: "containerLog", name: "web", bytes: 3 * GB, deltaBytes: 3 * GB }, + { kind: "volume", name: "data", bytes: 1.5 * GB, deltaBytes: 0.5 * GB }, + { kind: "category", name: "other", bytes: 5 * GB, deltaBytes: -1 * GB }, + { kind: "image", name: "old", bytes: 0, deltaBytes: -2 * GB }, + ]); + }); +}); diff --git a/apps/dokploy/__test__/monitoring/disk-usage-format.test.ts b/apps/dokploy/__test__/monitoring/disk-usage-format.test.ts new file mode 100644 index 00000000000..05d73f8f440 --- /dev/null +++ b/apps/dokploy/__test__/monitoring/disk-usage-format.test.ts @@ -0,0 +1,143 @@ +import { describe, expect, it } from "vitest"; +import { + DISK_CATEGORIES, + DISK_CATEGORY_ORDER, + describeDiskBucket, + formatDiskBucket, + formatDiskBytes, + formatDiskDelta, + formatDiskTimeTick, + getDiskAxisTicks, + getDiskMoverCategory, + getDiskMoverLabel, + getDiskTimeTicks, +} from "@/components/dashboard/monitoring/free/container/disk-usage/disk-usage-format"; + +const GB = 1e9; +const HOUR = 60 * 60 * 1000; +const localTime = (day: number, hour: number) => + new Date(2026, 9, day, hour).getTime(); + +describe("disk usage formatting", () => { + it("counts in decimal units, like Docker does", () => { + expect(formatDiskBytes(48_043_151_360)).toBe("48.0 GB"); + expect(formatDiskBytes(9_603_361_573)).toBe("9.60 GB"); + expect(formatDiskBytes(433_033_216)).toBe("433 MB"); + expect(formatDiskBytes(4096)).toBe("4 KB"); + expect(formatDiskBytes(0)).toBe("0 B"); + }); + + it("signs a change and calls a rounding-sized one no change", () => { + expect(formatDiskDelta(7.6 * GB)).toBe("+7.60 GB"); + expect(formatDiskDelta(-1.2 * GB)).toBe("−1.20 GB"); + expect(formatDiskDelta(300_000)).toBe("no change"); + expect(formatDiskDelta(0)).toBe("no change"); + }); + + it("keeps the stacking order the colours were checked in", () => { + expect(DISK_CATEGORY_ORDER).toEqual([ + "images", + "buildCache", + "volumes", + "containerLogs", + "containers", + "dokployData", + "swap", + "other", + ]); + expect( + new Set(Object.values(DISK_CATEGORIES).map((meta) => meta.color)).size, + ).toBe(8); + }); + + it("names what a bar covers", () => { + expect(describeDiskBucket(HOUR)).toBe("an hour"); + expect(describeDiskBucket(4 * HOUR)).toBe("4 hours"); + expect(describeDiskBucket(24 * HOUR)).toBe("a day"); + expect(formatDiskBucket(localTime(13, 16), 4 * HOUR)).toBe( + "13 Oct, 16:00–20:00", + ); + expect(formatDiskBucket(localTime(13, 0), 24 * HOUR)).toBe("13 Oct"); + }); +}); + +describe("disk chart axes", () => { + it("puts round values under the disk size", () => { + expect(getDiskAxisTicks(52.74 * GB)).toEqual([0, 20 * GB, 40 * GB]); + expect(getDiskAxisTicks(500 * GB)).toEqual([0, 200 * GB, 400 * GB]); + expect(getDiskAxisTicks(0)).toEqual([0]); + }); + + it("drops a value that would sit right under the disk size", () => { + // 100 GB would touch the "Disk size 105 GB" label. + expect(getDiskAxisTicks(105 * GB)).toEqual([0, 50 * GB]); + }); + + it("labels every third hour of a day", () => { + const starts = Array.from({ length: 24 }, (_, hour) => localTime(13, hour)); + + expect( + getDiskTimeTicks(starts, HOUR)?.map((from) => + formatDiskTimeTick(from, HOUR), + ), + ).toEqual([ + "13 Oct", + "03:00", + "06:00", + "09:00", + "12:00", + "15:00", + "18:00", + "21:00", + ]); + }); + + it("labels only the midnights of a week", () => { + const starts = Array.from({ length: 12 }, (_, index) => + localTime(12, index * 4), + ); + + expect( + getDiskTimeTicks(starts, 4 * HOUR)?.map((from) => + formatDiskTimeTick(from, 4 * HOUR), + ), + ).toEqual(["12 Oct", "13 Oct"]); + }); + + it("leaves day-long bars to the chart", () => { + expect(getDiskTimeTicks([localTime(13, 0)], 24 * HOUR)).toBeUndefined(); + expect(formatDiskTimeTick(localTime(13, 0), 24 * HOUR)).toBe("13 Oct"); + }); +}); + +describe("disk movers", () => { + it("labels an item with its kind and takes the colour of its category", () => { + const mover = { + kind: "containerLog" as const, + name: "web", + bytes: 1, + deltaBytes: 1, + }; + + expect(getDiskMoverLabel(mover)).toEqual({ + kind: "Container logs", + name: "web", + }); + expect(getDiskMoverCategory(mover)).toBe("containerLogs"); + }); + + it("shows a whole category by its own name", () => { + const mover = { + kind: "category" as const, + name: "buildCache", + bytes: 1, + deltaBytes: 1, + }; + + expect(getDiskMoverLabel(mover)).toEqual({ + kind: null, + name: "Build cache", + }); + expect(getDiskMoverCategory(mover)).toBe("buildCache"); + }); +}); diff --git a/apps/dokploy/__test__/monitoring/disk-usage.test.ts b/apps/dokploy/__test__/monitoring/disk-usage.test.ts new file mode 100644 index 00000000000..ac72f8e068b --- /dev/null +++ b/apps/dokploy/__test__/monitoring/disk-usage.test.ts @@ -0,0 +1,331 @@ +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + dial: vi.fn(), + info: vi.fn(), + listContainers: vi.fn(), + execAsync: vi.fn(), + readFile: vi.fn(), +})); + +vi.mock("@dokploy/server/constants", () => ({ + docker: { + modem: { dial: mocks.dial }, + info: mocks.info, + listContainers: mocks.listContainers, + }, + paths: () => ({ BASE_PATH: "/etc/dokploy" }), +})); + +vi.mock("@dokploy/server/utils/process/execAsync", () => ({ + execAsync: mocks.execAsync, +})); + +vi.mock("node:fs", async (importOriginal) => { + const original = await importOriginal(); + return { + ...original, + promises: { ...original.promises, readFile: mocks.readFile }, + }; +}); + +const { + parseDuOutput, + parseSwapFileBytes, + probeDockerUsage, + probeSlowUsage, + summarizeDockerDiskUsage, +} = await import("@dokploy/server/monitoring/disk-usage"); + +const GB = 1e9; +const ANONYMOUS = "a".repeat(64); + +const answerDiskUsage = (answers: Record) => { + mocks.dial.mockImplementation( + ( + request: { options: { type: string[] } }, + callback: (error: Error | null, data?: unknown) => void, + ) => { + const answer = answers[request.options.type.join(",")]; + if (answer instanceof Error) { + callback(answer); + return; + } + callback(null, answer); + }, + ); +}; + +describe("summarizeDockerDiskUsage", () => { + it("groups images by repository and counts only what each one owns", () => { + const { parts, items } = summarizeDockerDiskUsage({ + LayersSize: 5 * GB, + Images: [ + { + RepoTags: ["registry.example:5000/shop/web:v2"], + Size: 3 * GB, + SharedSize: 2 * GB, + }, + { + RepoTags: [":"], + RepoDigests: ["registry.example:5000/shop/web@sha256:abc"], + Size: 3 * GB, + SharedSize: 2.5 * GB, + }, + { RepoTags: null, RepoDigests: [], Size: 1 * GB, SharedSize: -1 }, + ], + }); + + expect(parts).toEqual({ images: 5 * GB }); + expect(items).toEqual({ + "image:registry.example:5000/shop/web": 1.5 * GB, + "image:untagged": 1 * GB, + }); + }); + + it("adds up the tasks of one swarm service", () => { + const { parts, items, itemKinds } = summarizeDockerDiskUsage({ + Containers: [ + { + Names: ["/web.1.abc"], + Labels: { "com.docker.swarm.service.name": "web" }, + SizeRw: 10, + }, + { + Names: ["/web.1.def"], + Labels: { "com.docker.swarm.service.name": "web" }, + SizeRw: 5, + }, + { Names: ["/compose-db-1"], Labels: {}, SizeRw: 7 }, + { Id: "0123456789abcdef", Names: [] }, + ], + }); + + expect(parts).toEqual({ containers: 22 }); + expect(items).toEqual({ + "container:web": 15, + "container:compose-db-1": 7, + "container:0123456789ab": 0, + }); + expect(itemKinds).toEqual(["container"]); + }); + + it("folds anonymous volumes into one item", () => { + const { parts, items } = summarizeDockerDiskUsage({ + Volumes: [ + { Name: "pg-data", UsageData: { Size: 4 * GB } }, + { Name: ANONYMOUS, UsageData: { Size: 100 } }, + { Name: "b".repeat(64), UsageData: { Size: 50 } }, + { Name: "unmeasured", UsageData: { Size: -1 } }, + ], + }); + + expect(parts).toEqual({ volumes: 4 * GB + 150 }); + expect(items).toEqual({ + "volume:pg-data": 4 * GB, + "volume:anonymous volumes": 150, + "volume:unmeasured": 0, + }); + }); + + it("leaves out build cache that images already count", () => { + const { parts, items } = summarizeDockerDiskUsage({ + BuildCache: [ + { Size: 6 * GB, Shared: false }, + { Size: 1 * GB, Shared: true }, + { Size: 2 * GB }, + ], + }); + + expect(parts).toEqual({ buildCache: 8 * GB }); + expect(items).toEqual({}); + }); + + it("reads the sections Docker moved under the Usage objects", () => { + const { parts, items, itemKinds } = summarizeDockerDiskUsage({ + ImageUsage: { + TotalSize: 2 * GB, + Items: [{ RepoTags: ["web:latest"], Size: 2 * GB, SharedSize: 0 }], + }, + ContainerUsage: { + TotalSize: 9, + Items: [{ Names: ["/web"], SizeRw: 9 }], + }, + VolumeUsage: { + TotalSize: 3, + Items: [{ Name: "data", UsageData: { Size: 3 } }], + }, + BuildCacheUsage: { TotalSize: 4, Items: [{ Size: 4, Shared: false }] }, + }); + + expect(parts).toEqual({ + images: 2 * GB, + containers: 9, + volumes: 3, + buildCache: 4, + }); + expect(items).toEqual({ + "image:web": 2 * GB, + "container:web": 9, + "volume:data": 3, + }); + expect(itemKinds).toEqual(["image", "container", "volume"]); + }); + + it("says nothing about a section Docker was not asked for", () => { + expect(summarizeDockerDiskUsage({ Images: null })).toEqual({ + parts: {}, + items: {}, + itemKinds: [], + }); + }); +}); + +describe("parseDuOutput", () => { + it("turns kilobytes into bytes per path", () => { + expect([ + ...parseDuOutput("4\t/data/a\n1024\t/data/with space\n2048\t/data\n"), + ]).toEqual([ + ["/data/a", 4096], + ["/data/with space", 1024 * 1024], + ["/data", 2048 * 1024], + ]); + }); + + it("ignores lines that are not sizes", () => { + expect(parseDuOutput("du: cannot read directory\n").size).toBe(0); + }); +}); + +describe("parseSwapFileBytes", () => { + it("counts swap files and skips swap partitions", () => { + const procSwaps = [ + "Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority", + "/swapfile file\t\t2097148\t\t178280\t\t-3", + "/dev/sda2 partition\t4194300\t\t0\t\t-2", + "/mnt/my\\040swap file\t\t1048572\t\t0\t\t-4", + ].join("\n"); + + expect(parseSwapFileBytes(procSwaps)).toBe((2097148 + 1048572) * 1024); + }); + + it("is zero without swap", () => { + expect( + parseSwapFileBytes("Filename\t\t\t\tType\t\tSize\t\tUsed\t\tPriority\n"), + ).toBe(0); + }); +}); + +describe("disk usage probes", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.spyOn(console, "error").mockImplementation(() => {}); + }); + + it("asks Docker only for what it answers from metadata", async () => { + answerDiskUsage({ + "image,container,build-cache": { + LayersSize: 10, + Images: [{ RepoTags: ["web:1"], Size: 10, SharedSize: 0 }], + Containers: [], + BuildCache: [{ Size: 3, Shared: false }], + }, + }); + + await expect(probeDockerUsage()).resolves.toEqual({ + parts: { images: 10, containers: 0, buildCache: 3 }, + items: { "image:web": 10 }, + itemKinds: ["image", "container"], + }); + expect(mocks.dial.mock.calls[0]?.[0]).toMatchObject({ + path: "/system/df?", + method: "GET", + }); + }); + + it("reports nothing when Docker does not answer", async () => { + answerDiskUsage({ + "image,container,build-cache": new Error("docker is down"), + }); + + await expect(probeDockerUsage()).resolves.toEqual({ + parts: {}, + items: {}, + itemKinds: [], + }); + }); + + it("measures volumes, container logs, Dokploy data and swap files", async () => { + const containerId = "c".repeat(64); + answerDiskUsage({ + volume: { Volumes: [{ Name: "pg-data", UsageData: { Size: 500 } }] }, + }); + mocks.info.mockResolvedValue({ DockerRootDir: "/var/lib/docker" }); + mocks.listContainers.mockResolvedValue([ + { + Id: containerId, + Names: ["/web.1.abc"], + Labels: { "com.docker.swarm.service.name": "web" }, + }, + ]); + mocks.execAsync.mockImplementation(async (command: string) => { + if (command.startsWith("docker run")) { + return { + stdout: `100\t/docker-containers/${containerId}\n8\t/docker-containers/${"d".repeat(64)}\n108\t/docker-containers\n`, + stderr: "", + }; + } + return { + stdout: + "40\t/etc/dokploy/logs\n4\t/etc/dokploy/ssh\n50\t/etc/dokploy\n", + stderr: "", + }; + }); + mocks.readFile.mockResolvedValue( + "Filename Type Size Used Priority\n/swapfile file 1024 0 -2\n", + ); + + await expect(probeSlowUsage()).resolves.toEqual({ + parts: { + volumes: 500, + containerLogs: 108 * 1024, + dokployData: 50 * 1024, + swap: 1024 * 1024, + }, + items: { + "volume:pg-data": 500, + "containerLog:web": 100 * 1024, + "containerLog:dddddddddddd": 8 * 1024, + "dokployData:logs": 40 * 1024, + "dokployData:ssh": 4 * 1024, + }, + itemKinds: ["volume", "containerLog", "dokployData"], + }); + + const logsCommand = mocks.execAsync.mock.calls + .map(([command]) => command as string) + .find((command) => command.startsWith("docker run")); + expect(logsCommand).toContain("--network none"); + // shell-quote escapes the colons of the mount. + expect(logsCommand?.replaceAll("\\", "")).toContain( + "-v /var/lib/docker/containers:/docker-containers:ro busybox du", + ); + }); + + it("keeps the parts it could measure when the others fail", async () => { + answerDiskUsage({ volume: new Error("docker is down") }); + mocks.info.mockRejectedValue(new Error("docker is down")); + mocks.listContainers.mockRejectedValue(new Error("docker is down")); + mocks.execAsync.mockResolvedValue({ + stdout: "50\t/etc/dokploy\n", + stderr: "", + }); + mocks.readFile.mockRejectedValue(new Error("no /proc on this system")); + + await expect(probeSlowUsage()).resolves.toEqual({ + parts: { dokployData: 50 * 1024 }, + items: {}, + itemKinds: ["dokployData"], + }); + }); +}); diff --git a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts index 53151907501..6a3a42334ce 100644 --- a/apps/dokploy/__test__/settings/docker-server-boundary.test.ts +++ b/apps/dokploy/__test__/settings/docker-server-boundary.test.ts @@ -50,6 +50,7 @@ const mocks = vi.hoisted(() => ({ readPorts: vi.fn(), recreateDirectory: vi.fn(), reloadDockerResource: vi.fn(), + refreshDiskUsage: vi.fn(), removeOldDokployImages: vi.fn(), removeJob: vi.fn(), schedule: vi.fn(), @@ -76,6 +77,7 @@ const redactWebServerSettings = (settings: T) => settings; vi.mock("@dokploy/server", async () => ({ CLEANUP_CRON_JOB: "0 0 * * *", DEFAULT_UPDATE_DATA: {}, + diskHistoryInputSchema: (await import("zod")).z.any(), dockerDiskUsageDetailLimitSchema: (await import("zod")).z.any(), DOKPLOY_KEEP_IMAGES_MAX: 5, DOKPLOY_KEEP_IMAGES_MIN: 3, @@ -131,6 +133,7 @@ vi.mock("@dokploy/server", async () => ({ readPorts: mocks.readPorts, recreateDirectory: mocks.recreateDirectory, reloadDockerResource: mocks.reloadDockerResource, + refreshDiskUsage: mocks.refreshDiskUsage, removeOldDokployImages: mocks.removeOldDokployImages, sendDockerCleanupNotifications: mocks.sendDockerCleanupNotifications, setupGPUSupport: mocks.setupGPUSupport, @@ -247,6 +250,7 @@ describe("settings Docker server boundary", () => { }); mocks.getAccessibleServerIds.mockResolvedValue(new Set(["server-1"])); mocks.cleanupBuilders.mockResolvedValue(undefined); + mocks.refreshDiskUsage.mockResolvedValue(undefined); mocks.cleanupContainers.mockResolvedValue(undefined); mocks.cleanupImages.mockResolvedValue(undefined); mocks.cleanupSystem.mockResolvedValue(undefined); @@ -465,6 +469,23 @@ describe("settings Docker server boundary", () => { expect(mocks.cleanupBuilders).not.toHaveBeenCalled(); }); + it("samples the disk after a cleanup, and answers even when that fails", async () => { + const space = { availableBytes: 2, oldImageCount: 0 }; + mocks.isServerUpdateRunning.mockReturnValue(false); + mocks.getUpdateDiskSpace.mockResolvedValue(space); + mocks.refreshDiskUsage.mockRejectedValue(new Error("docker is down")); + const consoleError = vi + .spyOn(console, "error") + .mockImplementation(() => {}); + + await expect( + createCaller().freeUpdateDiskSpace({ buildCache: false }), + ).resolves.toEqual(space); + + expect(mocks.refreshDiskUsage).toHaveBeenCalledWith({ force: true }); + consoleError.mockRestore(); + }); + it("clears the build cache only when asked", async () => { mocks.isServerUpdateRunning.mockReturnValue(false); diff --git a/apps/dokploy/__test__/settings/request-log-authorization.test.ts b/apps/dokploy/__test__/settings/request-log-authorization.test.ts index 6def3f05dc4..2648b3cd82e 100644 --- a/apps/dokploy/__test__/settings/request-log-authorization.test.ts +++ b/apps/dokploy/__test__/settings/request-log-authorization.test.ts @@ -93,6 +93,7 @@ const mocks = vi.hoisted(() => ({ vi.mock("@dokploy/server", async () => ({ CLEANUP_CRON_JOB: "0 0 * * *", DEFAULT_UPDATE_DATA: {}, + diskHistoryInputSchema: (await import("zod")).z.any(), dockerDiskUsageDetailLimitSchema: (await import("zod")).z.any(), DOKPLOY_KEEP_IMAGES_MAX: 5, DOKPLOY_KEEP_IMAGES_MIN: 3, diff --git a/apps/dokploy/__test__/settings/update-disk-space.test.ts b/apps/dokploy/__test__/settings/update-disk-space.test.ts index abd9266db86..0a2fcc664a9 100644 --- a/apps/dokploy/__test__/settings/update-disk-space.test.ts +++ b/apps/dokploy/__test__/settings/update-disk-space.test.ts @@ -152,6 +152,12 @@ describe("update disk space", () => { requiredBytes: 4_450_000_000, oldImageCount: 1, oldImageBytes: 1_500_000_000, + oldImages: [ + expect.objectContaining({ + id: "sha256:old", + uniqueSizeBytes: 1_500_000_000, + }), + ], buildCacheBytes: 6_193_000_000, }); }); @@ -166,6 +172,7 @@ describe("update disk space", () => { requiredBytes: null, oldImageCount: 0, oldImageBytes: 0, + oldImages: [], buildCacheBytes: 0, }); }); diff --git a/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-breakdown.tsx b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-breakdown.tsx new file mode 100644 index 00000000000..5928bd1c588 --- /dev/null +++ b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-breakdown.tsx @@ -0,0 +1,162 @@ +import type { DiskUsageHistory } from "@dokploy/server/index"; +import { cn } from "@/lib/utils"; +import { + DISK_CATEGORIES, + formatDiskBytes, + formatDiskDelta, + getDiskMoverCategory, + getDiskMoverLabel, +} from "./disk-usage-format"; + +type Current = NonNullable; + +const Swatch = ({ color }: { color: string }) => ( + +); + +const Delta = ({ bytes }: { bytes: number | null }) => ( + + {bytes === null ? "—" : formatDiskDelta(bytes)} + +); + +export const DiskCompositionBar = ({ + current, + categories, +}: { + current: Current; + categories: DiskUsageHistory["categories"]; +}) => ( +
+ {categories + .filter((row) => (row.bytes ?? 0) > 0) + .map((row) => ( +
+ ))} +
+); + +export const DiskCategoryTable = ({ + history, + current, +}: { + history: DiskUsageHistory; + current: Current; +}) => ( +
+
+ What takes the space + Now + Change +
+
    + {history.categories.map((row) => { + const meta = DISK_CATEGORIES[row.category]; + return ( +
  • +
    + +
    +

    {meta.label}

    +

    + {meta.description} +

    +
    +
    + + {row.bytes === null ? "—" : formatDiskBytes(row.bytes)} + + + + +
  • + ); + })} +
+
+ Used in total + + {formatDiskBytes(current.usedBytes)} + + + + +
+
+); + +export const DiskMoversList = ({ history }: { history: DiskUsageHistory }) => { + if (history.changeSince === null) { + return ( +

+ The history has just started. What grows shows up here after the next + measurements. +

+ ); + } + if (history.movers.length === 0) { + return ( +

+ Nothing changed by more than 10 MB. +

+ ); + } + return ( +
    + {history.movers.map((mover) => { + const label = getDiskMoverLabel(mover); + return ( +
  • +
    + +
    +

    + {label.name} +

    + {label.kind && ( +

    {label.kind}

    + )} +
    +
    +
    + +

    + now {formatDiskBytes(mover.bytes)} +

    +
    +
  • + ); + })} +
+ ); +}; diff --git a/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-card.tsx b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-card.tsx new file mode 100644 index 00000000000..18000a9d1e5 --- /dev/null +++ b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-card.tsx @@ -0,0 +1,30 @@ +import type { DiskHistoryPeriod } from "@dokploy/server/index"; +import { useState } from "react"; +import { FreeDiskSpace } from "@/components/dashboard/settings/web-server/free-disk-space"; +import { api } from "@/utils/api"; +import { DiskUsagePanel } from "./disk-usage-panel"; + +const REFRESH_INTERVAL_MS = 60_000; + +export const DiskUsageCard = () => { + const [period, setPeriod] = useState("1d"); + const { data, isLoading } = api.settings.getDiskUsageHistory.useQuery( + { period, utcOffsetMinutes: -new Date().getTimezoneOffset() }, + { + refetchOnWindowFocus: false, + refetchInterval: REFRESH_INTERVAL_MS, + placeholderData: (previous) => previous, + }, + ); + + return ( + + + + ); +}; diff --git a/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-format.ts b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-format.ts new file mode 100644 index 00000000000..ce68d09a9d2 --- /dev/null +++ b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-format.ts @@ -0,0 +1,186 @@ +import type { + DiskCategory, + DiskHistoryPeriod, + DiskMover, +} from "@dokploy/server/index"; +import { format } from "date-fns"; + +interface DiskCategoryMeta { + label: string; + description: string; + color: string; +} + +// The order is the stacking order, bottom to top. The colors were checked as +// neighbours in exactly this order, so a new category goes through the palette +// validator before it is inserted. +export const DISK_CATEGORIES: Record = { + images: { + label: "Images", + description: "Docker images of your apps, databases and Dokploy itself.", + color: "var(--disk-images)", + }, + buildCache: { + label: "Build cache", + description: + "Layers kept to make the next builds faster, without those that images share. Safe to clear.", + color: "var(--disk-build-cache)", + }, + volumes: { + label: "Volumes", + description: "Data of databases and apps stored in Docker volumes.", + color: "var(--disk-volumes)", + }, + containerLogs: { + label: "Container logs", + description: "Output that Docker stores for every container.", + color: "var(--disk-container-logs)", + }, + containers: { + label: "Container files", + description: "Files written inside containers, outside of volumes.", + color: "var(--disk-containers)", + }, + dokployData: { + label: "Dokploy data", + description: "Deployment logs, source checkouts and backups of Dokploy.", + color: "var(--disk-dokploy-data)", + }, + swap: { + label: "Swap files", + description: "Disk space the system reserved to extend the memory.", + color: "var(--disk-swap)", + }, + other: { + label: "System and other", + description: + "The operating system, system logs and everything Docker does not report.", + color: "var(--disk-other)", + }, +}; + +export const DISK_CATEGORY_ORDER = Object.keys( + DISK_CATEGORIES, +) as DiskCategory[]; + +export const DISK_PERIOD_LABELS: Record = { + "1d": "1 day", + "3d": "3 days", + "7d": "7 days", + "30d": "30 days", +}; + +// Decimal units, the same Docker prints, so the numbers match `docker system df`. +export const formatDiskBytes = (bytes: number) => { + const absolute = Math.abs(bytes); + if (absolute >= 1e9) { + return `${(bytes / 1e9).toFixed(absolute >= 1e10 ? 1 : 2)} GB`; + } + if (absolute >= 1e6) { + return `${Math.round(bytes / 1e6)} MB`; + } + if (absolute >= 1e3) { + return `${Math.round(bytes / 1e3)} KB`; + } + return `${bytes} B`; +}; + +export const formatDiskDelta = (bytes: number) => { + // A change that rounds to zero reads as "no change", not as "+0 MB". + if (Math.abs(bytes) < 5e5) { + return "no change"; + } + return `${bytes > 0 ? "+" : "−"}${formatDiskBytes(Math.abs(bytes))}`; +}; + +export const formatDiskBucket = (from: number, bucketMs: number) => { + const start = new Date(from); + if (bucketMs >= 24 * 60 * 60 * 1000) { + return format(start, "d MMM"); + } + return `${format(start, "d MMM")}, ${format(start, "HH:mm")}–${format(new Date(from + bucketMs), "HH:mm")}`; +}; + +export const describeDiskBucket = (bucketMs: number) => { + const hours = bucketMs / (60 * 60 * 1000); + if (hours >= 24) { + return "a day"; + } + return hours === 1 ? "an hour" : `${hours} hours`; +}; + +const MOVER_KIND_LABELS: Record = { + image: "Images", + volume: "Volume", + container: "Container files", + containerLog: "Container logs", + dokployData: "Dokploy data", + category: null, +}; + +export const getDiskMoverLabel = (mover: DiskMover) => { + if (mover.kind === "category") { + return { + kind: null, + name: DISK_CATEGORIES[mover.name as DiskCategory]?.label ?? mover.name, + }; + } + return { kind: MOVER_KIND_LABELS[mover.kind], name: mover.name }; +}; + +const MOVER_CATEGORIES: Record< + Exclude, + DiskCategory +> = { + image: "images", + volume: "volumes", + container: "containers", + containerLog: "containerLogs", + dokployData: "dokployData", +}; + +export const getDiskMoverCategory = (mover: DiskMover): DiskCategory => + mover.kind === "category" + ? (mover.name as DiskCategory) + : MOVER_CATEGORIES[mover.kind]; + +/** Round axis values (1, 2 or 5 times a power of ten) below the disk size. */ +export const getDiskAxisTicks = (totalBytes: number) => { + if (totalBytes <= 0) { + return [0]; + } + const rough = totalBytes / 4; + const magnitude = 10 ** Math.floor(Math.log10(rough)); + const step = + magnitude * + ([1, 2, 5, 10].find((factor) => factor * magnitude >= rough) ?? 10); + const ticks: number[] = []; + // A value right under the disk size would collide with the label of that line. + for (let value = 0; value <= totalBytes * 0.92; value += step) { + ticks.push(value); + } + return ticks; +}; + +const HOUR_MS = 60 * 60 * 1000; + +/** + * Bars that get a label under the axis: every third hour of a day, noon and + * midnight of three days, midnight of a week. Day-long bars are left to the chart. + */ +export const getDiskTimeTicks = (bucketStarts: number[], bucketMs: number) => { + const hours = bucketMs / HOUR_MS; + if (hours >= 24) { + return undefined; + } + const step = hours <= 1 ? 3 : hours <= 2 ? 12 : 24; + return bucketStarts.filter((from) => new Date(from).getHours() % step === 0); +}; + +/** Midnight carries the date, so the days of a longer period can be told apart. */ +export const formatDiskTimeTick = (from: number, bucketMs: number) => { + const start = new Date(from); + return bucketMs >= 24 * HOUR_MS || start.getHours() === 0 + ? format(start, "d MMM") + : format(start, "HH:mm"); +}; diff --git a/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-history-chart.tsx b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-history-chart.tsx new file mode 100644 index 00000000000..06a8644affe --- /dev/null +++ b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-history-chart.tsx @@ -0,0 +1,178 @@ +import type { DiskCategory, DiskUsageHistory } from "@dokploy/server/index"; +import { + Bar, + BarChart, + CartesianGrid, + ReferenceLine, + XAxis, + YAxis, +} from "recharts"; +import { + type ChartConfig, + ChartContainer, + ChartTooltip, +} from "@/components/ui/chart"; +import { + DISK_CATEGORIES, + DISK_CATEGORY_ORDER, + describeDiskBucket, + formatDiskBucket, + formatDiskBytes, + formatDiskTimeTick, + getDiskAxisTicks, + getDiskTimeTicks, +} from "./disk-usage-format"; + +const chartConfig = Object.fromEntries( + DISK_CATEGORY_ORDER.map((category) => [ + category, + { + label: DISK_CATEGORIES[category].label, + color: DISK_CATEGORIES[category].color, + }, + ]), +) satisfies ChartConfig; + +type ChartRow = { from: number; usedBytes: number | null } & Record< + DiskCategory, + number +>; + +const BucketTooltip = ({ + active, + payload, + bucketMs, + totalBytes, +}: { + active?: boolean; + payload?: { payload?: ChartRow }[]; + bucketMs: number; + totalBytes: number; +}) => { + const row = payload?.[0]?.payload; + if (!active || !row || row.usedBytes === null) { + return null; + } + return ( +
+
{formatDiskBucket(row.from, bucketMs)}
+
+ Used {formatDiskBytes(row.usedBytes)} of {formatDiskBytes(totalBytes)} +
+
+ {DISK_CATEGORY_ORDER.filter((category) => row[category] > 0).map( + (category) => ( +
+ + + {DISK_CATEGORIES[category].label} + + + {formatDiskBytes(row[category])} + +
+ ), + )} +
+
+ ); +}; + +interface Props { + history: DiskUsageHistory; +} + +export const DiskUsageHistoryChart = ({ history }: Props) => { + const totalBytes = + history.current?.totalBytes ?? + history.buckets + .map((bucket) => bucket.sample?.totalBytes ?? 0) + .reduce((latest, bytes) => bytes || latest, 0); + const rows: ChartRow[] = history.buckets.map((bucket) => ({ + from: bucket.from, + usedBytes: bucket.sample?.usedBytes ?? null, + ...(Object.fromEntries( + DISK_CATEGORY_ORDER.map((category) => [ + category, + bucket.sample?.parts[category] ?? 0, + ]), + ) as Record), + })); + + return ( +
+ + + + row.from), + history.bucketMs, + )} + tickFormatter={(from: number) => + formatDiskTimeTick(from, history.bucketMs) + } + /> + `${Math.round(bytes / 1e9)} GB`} + /> + + + } + /> + {DISK_CATEGORY_ORDER.map((category, index) => ( + + ))} + + +

+ Each bar covers {describeDiskBucket(history.bucketMs)} and shows the + fullest moment of it, so a short spike stays visible. +

+
+ ); +}; diff --git a/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-panel.tsx b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-panel.tsx new file mode 100644 index 00000000000..c0d7a4083ee --- /dev/null +++ b/apps/dokploy/components/dashboard/monitoring/free/container/disk-usage/disk-usage-panel.tsx @@ -0,0 +1,128 @@ +import type { + DiskHistoryPeriod, + DiskUsageHistory, +} from "@dokploy/server/index"; +import { format } from "date-fns"; +import { AlertTriangle, Loader2 } from "lucide-react"; +import type { ReactNode } from "react"; +import { Tabs, TabsList, TabsTrigger } from "@/components/ui/tabs"; +import { + DiskCategoryTable, + DiskCompositionBar, + DiskMoversList, +} from "./disk-usage-breakdown"; +import { DISK_PERIOD_LABELS, formatDiskBytes } from "./disk-usage-format"; +import { DiskUsageHistoryChart } from "./disk-usage-history-chart"; + +const ALMOST_FULL_SHARE = 0.9; + +interface Props { + history: DiskUsageHistory | null | undefined; + isLoading: boolean; + period: DiskHistoryPeriod; + onPeriodChange: (period: DiskHistoryPeriod) => void; + /** Rendered under the tables; the page puts the cleanup actions here. */ + children?: ReactNode; +} + +export const DiskUsagePanel = ({ + history, + isLoading, + period, + onPeriodChange, + children, +}: Props) => { + const current = history?.current; + + if (isLoading && !history) { + return ( +
+ +
+ ); + } + if (!history || !current) { + return ( +

+ No disk usage data available. +

+ ); + } + + const usedShare = current.usedBytes / current.totalBytes; + // The first saved sample is later than the period's start while the + // history is younger than the period. + const isShortHistory = + history.changeSince !== null && + history.changeSince > history.from + history.bucketMs; + + return ( +
+
+
+

+ {formatDiskBytes(current.usedBytes)} + + used of {formatDiskBytes(current.totalBytes)} + +

+

+ + {formatDiskBytes(current.availableBytes)} free ·{" "} + {Math.round(usedShare * 100)}% full + + {usedShare >= ALMOST_FULL_SHARE && ( + + + Almost full + + )} +

+
+ onPeriodChange(value as DiskHistoryPeriod)} + > + + {(Object.keys(DISK_PERIOD_LABELS) as DiskHistoryPeriod[]).map( + (option) => ( + + {DISK_PERIOD_LABELS[option]} + + ), + )} + + +
+ + + + +
+ +
+
+ {history.changeSince === null + ? "What changed the most" + : `What changed the most since ${format(new Date(history.changeSince), "d MMM, HH:mm")}`} +
+
+ +
+ {isShortHistory && ( +

+ The history is shorter than {DISK_PERIOD_LABELS[period]}, so the + changes are counted from its first measurement. +

+ )} +
+
+ +

+ 1 GB is 1000 MB here, the same units Docker reports. +

+ + {children} +
+ ); +}; diff --git a/apps/dokploy/components/dashboard/monitoring/free/container/docker-disk-chart.tsx b/apps/dokploy/components/dashboard/monitoring/free/container/docker-disk-chart.tsx deleted file mode 100644 index 987d956d090..00000000000 --- a/apps/dokploy/components/dashboard/monitoring/free/container/docker-disk-chart.tsx +++ /dev/null @@ -1,83 +0,0 @@ -import { format } from "date-fns"; -import { Area, AreaChart, CartesianGrid, YAxis } from "recharts"; -import { - type ChartConfig, - ChartContainer, - ChartTooltip, - ChartTooltipContent, -} from "@/components/ui/chart"; -import type { DockerStatsJSON } from "./show-free-container-monitoring"; - -interface Props { - accumulativeData: DockerStatsJSON["disk"]; - diskTotal: number; -} - -const chartConfig = { - usedGb: { - label: "Used (GB)", - color: "hsl(var(--chart-3))", - }, -} satisfies ChartConfig; - -export const DockerDiskChart = ({ accumulativeData, diskTotal }: Props) => { - const transformedData = accumulativeData.map((item, index) => ({ - time: item.time, - name: `Point ${index + 1}`, - usedGb: +item.value.diskUsage, - totalGb: +item.value.diskTotal, - })); - - return ( - - - - - - - - - - `${value} GB`} - /> - { - const time = payload?.[0]?.payload?.time; - return time ? format(new Date(time), "PPpp") : ""; - }} - formatter={(value) => { - return [`${value} GB`, "Used"]; - }} - /> - } - /> - - - - ); -}; diff --git a/apps/dokploy/components/dashboard/monitoring/free/container/docker-disk-usage-chart.tsx b/apps/dokploy/components/dashboard/monitoring/free/container/docker-disk-usage-chart.tsx index 6860e218f29..6e5ca7dec84 100644 --- a/apps/dokploy/components/dashboard/monitoring/free/container/docker-disk-usage-chart.tsx +++ b/apps/dokploy/components/dashboard/monitoring/free/container/docker-disk-usage-chart.tsx @@ -41,21 +41,22 @@ const chartConfig = { value: { label: "Size", }, + // The same colours as in the disk space card above it on the page. images: { label: "Images", - color: "hsl(var(--chart-1))", + color: "var(--disk-images)", }, containers: { label: "Containers", - color: "hsl(var(--chart-2))", + color: "var(--disk-containers)", }, volumes: { label: "Volumes", - color: "hsl(var(--chart-3))", + color: "var(--disk-volumes)", }, buildCache: { label: "Build Cache", - color: "hsl(var(--chart-4))", + color: "var(--disk-build-cache)", }, } satisfies ChartConfig; diff --git a/apps/dokploy/components/dashboard/monitoring/free/container/show-free-container-monitoring.tsx b/apps/dokploy/components/dashboard/monitoring/free/container/show-free-container-monitoring.tsx index 9e10bfb3a9c..ebad53468f4 100644 --- a/apps/dokploy/components/dashboard/monitoring/free/container/show-free-container-monitoring.tsx +++ b/apps/dokploy/components/dashboard/monitoring/free/container/show-free-container-monitoring.tsx @@ -5,9 +5,9 @@ import { Card, CardContent, CardHeader, CardTitle } from "@/components/ui/card"; import { Progress } from "@/components/ui/progress"; import { api } from "@/utils/api"; import { ContainerResourceUsage } from "../../container-resource-usage/container-resource-usage"; +import { DiskUsageCard } from "./disk-usage/disk-usage-card"; import { DockerBlockChart } from "./docker-block-chart"; import { DockerCpuChart } from "./docker-cpu-chart"; -import { DockerDiskChart } from "./docker-disk-chart"; import { DockerDiskUsageChart } from "./docker-disk-usage-chart"; import { getDockerDiskUsageCardClassName } from "./docker-disk-usage-layout"; import { DockerMemoryChart } from "./docker-memory-chart"; @@ -314,24 +314,12 @@ export const ContainerFreeMonitoring = ({ {appName === "dokploy" && ( - + Disk Space -
- - {`Used: ${currentData.disk.value.diskUsage} GB / Limit: ${currentData.disk.value.diskTotal} GB`} - - - -
+
)} diff --git a/apps/dokploy/components/dashboard/settings/web-server/free-disk-space.tsx b/apps/dokploy/components/dashboard/settings/web-server/free-disk-space.tsx index 8fa7ce9f181..de53f36a822 100644 --- a/apps/dokploy/components/dashboard/settings/web-server/free-disk-space.tsx +++ b/apps/dokploy/components/dashboard/settings/web-server/free-disk-space.tsx @@ -1,21 +1,32 @@ +import { format } from "date-fns"; import { AlertTriangle, HardDrive, Loader2, Trash2 } from "lucide-react"; -import { useState } from "react"; +import { useId, useState } from "react"; import { toast } from "sonner"; import { Button } from "@/components/ui/button"; import { Checkbox } from "@/components/ui/checkbox"; import { Label } from "@/components/ui/label"; import { cn } from "@/lib/utils"; import { api } from "@/utils/api"; -import { formatImageSize } from "./dokploy-image-plan"; +import { formatImageSize, getDokployImageVersion } from "./dokploy-image-plan"; interface Props { /** The last update failed because the disk was full. */ diskFull?: boolean; disabled?: boolean; + /** + * Lists the builds a cleanup removes and leaves out the free space line, + * for a page that already shows the disk. + */ + showOldImages?: boolean; } -export const FreeDiskSpace = ({ diskFull = false, disabled }: Props) => { +export const FreeDiskSpace = ({ + diskFull = false, + disabled, + showOldImages = false, +}: Props) => { const [buildCache, setBuildCache] = useState(false); + const buildCacheId = useId(); const utils = api.useUtils(); const { data: space, isLoading } = api.settings.getUpdateDiskSpace.useQuery( undefined, @@ -58,7 +69,11 @@ export const FreeDiskSpace = ({ diskFull = false, disabled }: Props) => { : "Cleanup finished", ); } - await utils.settings.getDokployImages.invalidate(); + await Promise.all([ + utils.settings.getDokployImages.invalidate(), + utils.settings.getDiskUsageHistory.invalidate(), + utils.settings.getDockerDiskUsage.invalidate(), + ]); } catch (error) { toast.error( error instanceof Error && error.message @@ -77,19 +92,23 @@ export const FreeDiskSpace = ({ diskFull = false, disabled }: Props) => { : "border-border", )} > -
- - - Free disk space:{" "} - - {space.availableBytes !== null - ? formatImageSize(space.availableBytes) - : "unknown"} + {showOldImages ? ( + Free up space + ) : ( +
+ + + Free disk space:{" "} + + {space.availableBytes !== null + ? formatImageSize(space.availableBytes) + : "unknown"} + + {space.totalBytes !== null && + ` of ${formatImageSize(space.totalBytes)}`} - {space.totalBytes !== null && - ` of ${formatImageSize(space.totalBytes)}`} - -
+
+ )} {(diskFull || isLow) && (
@@ -108,17 +127,37 @@ export const FreeDiskSpace = ({ diskFull = false, disabled }: Props) => { : "No old Dokploy builds to remove. The running build and the previous one always stay."} + {showOldImages && space.oldImages.length > 0 && ( +
    + {space.oldImages.map((image) => ( +
  • + + {getDokployImageVersion(image)} + + {" · "} + {format(new Date(image.createdAt), "dd.MM.yyyy HH:mm")} + + + + frees{" "} + {formatImageSize(image.uniqueSizeBytes ?? image.sizeBytes)} + +
  • + ))} +
+ )} + {space.buildCacheBytes > 0 && (
setBuildCache(checked === true)} disabled={disabled || isPending} className="mt-0.5" />
diff --git a/apps/dokploy/server/api/routers/settings.ts b/apps/dokploy/server/api/routers/settings.ts index feb062f64f1..17ed0f9b006 100644 --- a/apps/dokploy/server/api/routers/settings.ts +++ b/apps/dokploy/server/api/routers/settings.ts @@ -16,11 +16,13 @@ import { DEFAULT_UPDATE_DATA, DOKPLOY_KEEP_IMAGES_MAX, DOKPLOY_KEEP_IMAGES_MIN, + diskHistoryInputSchema, dockerDiskUsageDetailLimitSchema, filterProtectedTraefikEntries, findServerById, getAgentHitsUpdateCommand, getAgentHitsUpdateImage, + getDiskUsageHistory, getDockerDiskUsage, getDokployImageKeepCount, getDokployImages, @@ -53,6 +55,7 @@ import { readPorts, recreateDirectory, redactWebServerSettings, + refreshDiskUsage, reloadDockerResource, removeOldDokployImages, resolveDockerDiskUsageDetailLimit, @@ -441,6 +444,14 @@ export const settingsRouter = createTRPCRouter({ resolveDockerDiskUsageDetailLimit(input?.detailLimit), ); }), + getDiskUsageHistory: adminProcedure + .input(diskHistoryInputSchema) + .query(async ({ input }) => { + if (IS_CLOUD) { + return null; + } + return await getDiskUsageHistory(input.period, input.utcOffsetMinutes); + }), saveSSHPrivateKey: adminProcedure .input(apiSaveSSHKey) .mutation(async ({ input, ctx }) => { @@ -817,6 +828,11 @@ export const settingsRouter = createTRPCRouter({ ? "free-update-disk-space-with-build-cache" : "free-update-disk-space", }); + // The page asks for the history right after this; without a new + // sample it would still show the space as taken. + await refreshDiskUsage({ force: true }).catch((error) => { + console.error("Disk usage: sampling after the cleanup failed", error); + }); return await getUpdateDiskSpace(); }), diff --git a/apps/dokploy/server/api/utils/procedure-access.ts b/apps/dokploy/server/api/utils/procedure-access.ts index 0f980437183..25bb6b10baf 100644 --- a/apps/dokploy/server/api/utils/procedure-access.ts +++ b/apps/dokploy/server/api/utils/procedure-access.ts @@ -599,6 +599,7 @@ export const PROCEDURE_ACCESS: Record = { "settings.cleanUnusedImages": "write", "settings.cleanUnusedVolumes": "dangerous", "settings.freeUpdateDiskSpace": "write", + "settings.getDiskUsageHistory": "read", "settings.getDockerDiskUsage": "read", "settings.getDokployCloudIps": "read", "settings.getDokployImageKeepCount": "read", diff --git a/apps/dokploy/server/server.ts b/apps/dokploy/server/server.ts index 6c236a02267..c70e3c48ae7 100644 --- a/apps/dokploy/server/server.ts +++ b/apps/dokploy/server/server.ts @@ -14,6 +14,7 @@ import { initVolumeBackupsCronJobs, sendDokployRestartNotifications, setupDirectories, + startDiskHistorySampler, } from "@dokploy/server"; import { warnAboutInterruptedTraefikDeletion } from "@dokploy/server/services/domain-overview"; import { config } from "dotenv"; @@ -158,6 +159,7 @@ void app.prepare().then(async () => { await initVolumeBackupsCronJobs(); await sendDokployRestartNotifications(); void cleanupOldDokployImages(); + startDiskHistorySampler(); void warnAboutInterruptedTraefikDeletion(); } await initEnterpriseBackupCronJobs(); diff --git a/apps/dokploy/styles/globals.css b/apps/dokploy/styles/globals.css index ae5e6fb8125..4937506f3de 100644 --- a/apps/dokploy/styles/globals.css +++ b/apps/dokploy/styles/globals.css @@ -206,6 +206,14 @@ body[data-scroll-locked] [data-slot="sidebar-inset"] { --chart-3: 197 37% 24%; --chart-4: 43 74% 66%; --chart-5: 27 87% 67%; + --disk-images: #2a78d6; + --disk-build-cache: #eb6834; + --disk-volumes: #1baf7a; + --disk-container-logs: #eda100; + --disk-containers: #e87ba4; + --disk-dokploy-data: #008300; + --disk-swap: #4a3aa7; + --disk-other: #898781; --sidebar-background: 0 0% 98%; --sidebar-foreground: 240 5.3% 26.1%; --sidebar-primary: 240 5.9% 10%; @@ -253,6 +261,14 @@ body[data-scroll-locked] [data-slot="sidebar-inset"] { --chart-3: 30 80% 55%; --chart-4: 280 65% 60%; --chart-2: 340 75% 55%; + --disk-images: #3987e5; + --disk-build-cache: #d95926; + --disk-volumes: #199e70; + --disk-container-logs: #c98500; + --disk-containers: #d55181; + --disk-dokploy-data: #008300; + --disk-swap: #9085e9; + --disk-other: #898781; --sidebar-background: 240 5.9% 10%; --sidebar-foreground: 240 4.8% 95.9%; --sidebar-primary: 224.3 76.3% 48%; diff --git a/docs/agenthits-fork.md b/docs/agenthits-fork.md index 3d3b361d999..3948d75256e 100644 --- a/docs/agenthits-fork.md +++ b/docs/agenthits-fork.md @@ -37,7 +37,9 @@ Fork: off_v0.29.8/Fork_+ - Installer `install-agenthits.sh` ставит форк напрямую из GHCR, создает Docker secrets для runtime secret values и поддерживает `postgres:18+` volume target. - Мониторинг содержит дополнительные улучшения: breakdown ресурсов контейнеров, - детализацию Docker disk usage и отображение swap usage. + детализацию Docker disk usage, отображение swap usage и историю занятого + диска по категориям за день, 3, 7 и 30 дней с очисткой старых образов панели + и кэша сборки. - Добавлена безопасная upsert-логика application environment variables. - Исправлено сопоставление GitHub webhook deploys по owner login. - Ветка содержит security hardening поверх официального `v0.29.8`: проверки @@ -95,7 +97,9 @@ commits, the fork version looks like `off_v0.29.8/Fork_157+abc123def456`. secrets for runtime secret values, and supports the `postgres:18+` volume target. - Monitoring includes extra improvements: container resource breakdown, Docker - disk usage details, and swap usage display. + disk usage details, swap usage display, and a disk usage history by category + for 1, 3, 7 and 30 days with a cleanup of old panel images and the build + cache. - Safe application environment variable upsert logic was added. - GitHub webhook deploy matching was fixed to use owner login. - The branch includes security hardening on top of official `v0.29.8`: access diff --git a/docs/server-maintenance.md b/docs/server-maintenance.md index 7da63ad48cc..40743d3f9a5 100644 --- a/docs/server-maintenance.md +++ b/docs/server-maintenance.md @@ -79,3 +79,21 @@ Docker и Postgres кнопкой не обновляются. Карточка - Traefik: `docker logs --tail 120 dokploy-traefik` - Резервные копии Postgres: `/var/backups/dokploy/`, файлы вида `postgres-<дата>.sql.gz`. Доступны только root, сами не удаляются. - Копии пакетов Docker, если обновляли Docker: `/var/backups/dokploy/docker-rollback-<дата>/`. + +## 6. Диск: что занимает место и как его освободить + +Страница **Monitoring**, карточка **Disk Space**. + +- Панель раз в минуту читает, сколько занято на диске, и раз в 5 минут спрашивает у Docker размеры образов, кэша сборки и файлов контейнеров. Раз в час она измеряет тома, логи контейнеров, папку `/etc/dokploy` и файлы подкачки. Если занятое место изменилось больше чем на 0,5% диска (минимум 256 МБ), замер делается сразу, поэтому короткий всплеск тоже попадает в историю. +- График показывает день, 3, 7 или 30 дней. Каждый столбец показывает самый заполненный момент своего интервала. +- Таблица «What takes the space» в сумме даёт ровно занятое место. Строка «System and other» содержит всё, о чём Docker не сообщает: систему, системные логи, служебные файлы Docker. +- Список «What changed the most» называет конкретные образы, тома и контейнеры, которые выросли или уменьшились с начала периода. +- Все числа в десятичных гигабайтах (1 ГБ = 1000 МБ), как у Docker. Команда `df -h` считает в двоичных, поэтому показывает меньше: 52,7 ГБ здесь и 50G там — один и тот же диск. +- История хранится 30 дней в `/etc/dokploy/monitoring/disk-history/`. Действие **Clean Monitoring** удаляет её вместе с остальными данными мониторинга. + +Блок **Free up space** в той же карточке: + +- удаляет старые образы панели, кроме работающего и предыдущего (он нужен для отката), и показывает их список до удаления; +- по галочке очищает кэш сборки Docker. Это безопасно: ничего не ломается, только следующие сборки приложений идут без кэша и дольше. + +Логи контейнеров панель измеряет временным контейнером `busybox`, которому каталог Docker с контейнерами подключён только для чтения. Если образ `busybox` скачать нельзя, логи попадают в строку «System and other». diff --git a/packages/server/src/index.ts b/packages/server/src/index.ts index 69a0f66036b..6706464e8c3 100644 --- a/packages/server/src/index.ts +++ b/packages/server/src/index.ts @@ -6,6 +6,8 @@ export * from "./db/validations/domain"; export * from "./db/validations/index"; export * from "./lib/auth"; export * from "./lib/logger"; +export * from "./monitoring/disk-history"; +export * from "./monitoring/disk-usage"; export * from "./monitoring/utils"; export * from "./services/account-deletion"; export * from "./services/admin"; diff --git a/packages/server/src/monitoring/disk-history.ts b/packages/server/src/monitoring/disk-history.ts new file mode 100644 index 00000000000..c9e4e916552 --- /dev/null +++ b/packages/server/src/monitoring/disk-history.ts @@ -0,0 +1,498 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { z } from "zod"; +import { paths } from "../constants"; +import { + DISK_PARTS, + type DiskCategory, + type DiskItemKind, + type DiskItems, + type DiskPart, + type DiskParts, + type DiskStat, + type DiskUsageProbe, + getRootDiskStat, + parseDiskItemKey, + probeDockerUsage, + probeSlowUsage, +} from "./disk-usage"; + +export const DISK_HISTORY_PERIODS = ["1d", "3d", "7d", "30d"] as const; +export type DiskHistoryPeriod = (typeof DISK_HISTORY_PERIODS)[number]; + +export const diskHistoryInputSchema = z.object({ + period: z.enum(DISK_HISTORY_PERIODS), + /** Minutes east of UTC, so that day-long bars start at the viewer's midnight. */ + utcOffsetMinutes: z.number().int().min(-840).max(840).default(0), +}); + +const MINUTE = 60_000; +const HOUR = 60 * MINUTE; +const DAY = 24 * HOUR; + +const PERIOD_BUCKETS: Record< + DiskHistoryPeriod, + { bucketMs: number; count: number } +> = { + "1d": { bucketMs: HOUR, count: 24 }, + "3d": { bucketMs: 2 * HOUR, count: 36 }, + "7d": { bucketMs: 4 * HOUR, count: 42 }, + "30d": { bucketMs: DAY, count: 30 }, +}; + +const STAT_INTERVAL_MS = MINUTE; +const PROBE_INTERVAL_MS = 5 * MINUTE; +const SLOW_PROBE_INTERVAL_MS = HOUR; +const SLOW_PROBE_JUMP_GAP_MS = 10 * MINUTE; +// A probe may take at most 2% of the time, however slow the disk is. +const PROBE_COOLDOWN_FACTOR = 50; +const RETENTION_MS = 30 * DAY; +const PRUNE_INTERVAL_MS = 6 * HOUR; +const JUMP_MIN_BYTES = 256 * 1024 ** 2; +const JUMP_DISK_SHARE = 0.005; +const MOVER_MIN_BYTES = 10 * 1024 ** 2; +const MOVERS_LIMIT = 12; + +export interface DiskSample extends DiskParts { + at: number; + totalBytes: number; + usedBytes: number; +} + +interface DiskItemsSnapshot { + at: number; + items: DiskItems; +} + +export interface DiskUsageBreakdown { + at: number; + totalBytes: number; + usedBytes: number; + /** Null for a part that could not be measured; its bytes are in `other`. */ + parts: Record; +} + +export interface DiskCategoryChange { + category: DiskCategory; + bytes: number | null; + deltaBytes: number | null; +} + +export interface DiskMover { + kind: DiskItemKind | "category"; + /** Item name, or the category key when the category has no items. */ + name: string; + bytes: number; + deltaBytes: number; +} + +export interface DiskUsageHistory { + period: DiskHistoryPeriod; + from: number; + to: number; + bucketMs: number; + current: (DiskUsageBreakdown & { availableBytes: number }) | null; + /** One entry per bar: the fullest sample of that interval, or null without data. */ + buckets: { from: number; sample: DiskUsageBreakdown | null }[]; + /** Time of the sample the changes are measured from. */ + changeSince: number | null; + usedDeltaBytes: number | null; + categories: DiskCategoryChange[]; + movers: DiskMover[]; +} + +/** Used space that changed this much since the last saved sample is worth a new one. */ +export const isDiskUsageJump = ( + previousUsedBytes: number, + usedBytes: number, + totalBytes: number, +) => + Math.abs(usedBytes - previousUsedBytes) >= + Math.max(JUMP_MIN_BYTES, totalBytes * JUMP_DISK_SHARE); + +export const toDiskUsageBreakdown = ( + sample: DiskSample, +): DiskUsageBreakdown => { + const parts = Object.fromEntries( + DISK_PARTS.map((part) => [part, sample[part] ?? null]), + ) as Record; + const known = Object.values(parts).reduce( + (total, bytes) => total + (bytes ?? 0), + 0, + ); + return { + at: sample.at, + totalBytes: sample.totalBytes, + usedBytes: sample.usedBytes, + // Docker's data may live on another disk; the rest cannot go below zero. + parts: { ...parts, other: Math.max(sample.usedBytes - known, 0) }, + }; +}; + +/** The bars of a period, aligned to the viewer's clock; the last one holds `now`. */ +export const getDiskHistoryRange = ( + period: DiskHistoryPeriod, + now: number, + utcOffsetMinutes: number, +) => { + const { bucketMs, count } = PERIOD_BUCKETS[period]; + const offsetMs = utcOffsetMinutes * MINUTE; + const lastBucketFrom = + Math.floor((now + offsetMs) / bucketMs) * bucketMs - offsetMs; + return { + from: lastBucketFrom - (count - 1) * bucketMs, + to: lastBucketFrom + bucketMs, + bucketMs, + count, + }; +}; + +/** Keeps the fullest sample of each bar, so a short spike stays visible. */ +export const bucketDiskSamples = ( + samples: DiskSample[], + range: ReturnType, +) => { + const fullest: (DiskSample | null)[] = Array.from( + { length: range.count }, + () => null, + ); + for (const sample of samples) { + const index = Math.floor((sample.at - range.from) / range.bucketMs); + if (index < 0 || index >= range.count) { + continue; + } + const current = fullest[index]; + if (!current || sample.usedBytes >= current.usedBytes) { + fullest[index] = sample; + } + } + return fullest.map((sample, index) => ({ + from: range.from + index * range.bucketMs, + sample: sample ? toDiskUsageBreakdown(sample) : null, + })); +}; + +export const getDiskCategoryChanges = ( + start: DiskUsageBreakdown | null, + end: DiskUsageBreakdown, +): DiskCategoryChange[] => + ([...DISK_PARTS, "other"] as DiskCategory[]).map((category) => { + const bytes = end.parts[category]; + const startBytes = start?.parts[category] ?? null; + return { + category, + bytes, + deltaBytes: + bytes !== null && startBytes !== null ? bytes - startBytes : null, + }; + }); + +// Parts without single items; each one is listed among the movers as a whole. +const WHOLE_CATEGORY_MOVERS: DiskCategory[] = ["buildCache", "swap", "other"]; + +/** What grew or shrank the most between two moments, biggest growth first. */ +export const getDiskMovers = ( + startItems: DiskItems, + endItems: DiskItems, + categories: DiskCategoryChange[], +): DiskMover[] => { + const movers: DiskMover[] = []; + for (const key of new Set([ + ...Object.keys(startItems), + ...Object.keys(endItems), + ])) { + const bytes = endItems[key] ?? 0; + movers.push({ + ...parseDiskItemKey(key), + bytes, + deltaBytes: bytes - (startItems[key] ?? 0), + }); + } + for (const change of categories) { + if ( + WHOLE_CATEGORY_MOVERS.includes(change.category) && + change.bytes !== null && + change.deltaBytes !== null + ) { + movers.push({ + kind: "category", + name: change.category, + bytes: change.bytes, + deltaBytes: change.deltaBytes, + }); + } + } + return movers + .filter((mover) => Math.abs(mover.deltaBytes) >= MOVER_MIN_BYTES) + .sort((a, b) => b.deltaBytes - a.deltaBytes) + .slice(0, MOVERS_LIMIT); +}; + +const getHistoryFiles = () => { + const directory = path.join(paths().MONITORING_PATH, "disk-history"); + return { + directory, + samples: path.join(directory, "samples.jsonl"), + items: path.join(directory, "items.jsonl"), + }; +}; + +const readJsonLines = async ( + file: string, + since = 0, +): Promise => { + const content = await fs.readFile(file, "utf-8").catch(() => ""); + const rows: T[] = []; + for (const line of content.split("\n")) { + if (!line) { + continue; + } + try { + const row = JSON.parse(line) as T; + if (row.at >= since) { + rows.push(row); + } + } catch { + // A line cut short by a restart or a full disk. + } + } + return rows; +}; + +const appendJsonLine = async ( + file: string, + row: T, +) => { + await fs.mkdir(path.dirname(file), { recursive: true }); + // A full disk can cut the previous line short; the leading newline keeps + // this row from being glued to it and lost with it. + await fs.appendFile(file, `\n${JSON.stringify(row)}\n`); +}; + +const pruneJsonLines = async (file: string, since: number) => { + const content = await fs.readFile(file, "utf-8").catch(() => null); + if (content === null) { + return; + } + const kept = content.split("\n").filter((line) => { + const at = Number(line.match(/^\{"at":(\d+)/)?.[1]); + return Number.isFinite(at) && at >= since; + }); + // Written next to the file and renamed, so a crash cannot leave half of it. + const temporary = `${file}.tmp`; + await fs.writeFile(temporary, kept.length > 0 ? `${kept.join("\n")}\n` : ""); + await fs.rename(temporary, file); +}; + +interface ProbeRun { + at: number; + durationMs: number; +} + +interface SamplerState { + stat: (DiskStat & { at: number }) | null; + parts: DiskParts; + items: DiskItems; + /** Last sample written to disk; jumps are measured against it. */ + savedSample: DiskSample | null; + probe: ProbeRun | null; + slowProbe: ProbeRun | null; + prunedAt: number; + running: Promise | null; + timer: ReturnType | null; +} + +// Like the Docker client: the module is evaluated once per bundled copy, and +// two samplers would write every sample twice. +const globalForDiskHistory = globalThis as unknown as { + diskHistorySampler?: SamplerState; +}; + +if (!globalForDiskHistory.diskHistorySampler) { + globalForDiskHistory.diskHistorySampler = { + stat: null, + parts: {}, + items: {}, + savedSample: null, + probe: null, + slowProbe: null, + prunedAt: 0, + running: null, + timer: null, + }; +} + +const state = globalForDiskHistory.diskHistorySampler; + +const isProbeDue = (last: ProbeRun | null, now: number, intervalMs: number) => { + if (!last) { + return true; + } + // The timer fires a few milliseconds early or late; without the slack a + // probe due on this tick would wait for the next one. + const elapsed = now - last.at + STAT_INTERVAL_MS / 2; + return ( + elapsed >= intervalMs && elapsed >= last.durationMs * PROBE_COOLDOWN_FACTOR + ); +}; + +const applyProbe = (probe: DiskUsageProbe) => { + state.parts = { ...state.parts, ...probe.parts }; + const kept = Object.entries(state.items).filter( + ([key]) => !probe.itemKinds.includes(parseDiskItemKey(key).kind), + ); + state.items = { ...Object.fromEntries(kept), ...probe.items }; +}; + +const timeProbe = async ( + now: number, + run: () => Promise, +): Promise => { + const startedAt = Date.now(); + applyProbe(await run()); + return { at: now, durationMs: Date.now() - startedAt }; +}; + +const sampleDiskUsage = async (now: number, force: boolean) => { + const stat = await getRootDiskStat(); + state.stat = { ...stat, at: now }; + + const saved = state.savedSample; + const jumped = + !!saved && + isDiskUsageJump(saved.usedBytes, stat.usedBytes, stat.totalBytes); + const probeDue = + force || + isProbeDue(state.probe, now, jumped ? STAT_INTERVAL_MS : PROBE_INTERVAL_MS); + if (!probeDue) { + return; + } + const slowProbeDue = isProbeDue( + state.slowProbe, + now, + jumped ? SLOW_PROBE_JUMP_GAP_MS : SLOW_PROBE_INTERVAL_MS, + ); + + state.probe = await timeProbe(now, probeDockerUsage); + if (slowProbeDue) { + state.slowProbe = await timeProbe(now, probeSlowUsage); + } + + const files = getHistoryFiles(); + const sample: DiskSample = { + at: now, + totalBytes: stat.totalBytes, + usedBytes: stat.usedBytes, + ...state.parts, + }; + await appendJsonLine(files.samples, sample); + state.savedSample = sample; + if (slowProbeDue) { + // Saved together with a sample, so both describe the same moment. + await appendJsonLine(files.items, { at: now, items: state.items }); + } + + if (now - state.prunedAt >= PRUNE_INTERVAL_MS) { + state.prunedAt = now; + await pruneJsonLines(files.samples, now - RETENTION_MS); + await pruneJsonLines(files.items, now - RETENTION_MS); + } +}; + +/** + * Reads the disk and, when it is time, Docker's usage, and saves a sample. With + * `force` the Docker probe runs now, e.g. right after a cleanup. + */ +export const refreshDiskUsage = async ({ force = false } = {}) => { + // A forced refresh must not settle for a run that started before the cleanup. + if (state.running && force) { + await state.running.catch(() => {}); + } + state.running ??= sampleDiskUsage(Date.now(), force).finally(() => { + state.running = null; + }); + await state.running; +}; + +/** Samples the disk every minute for as long as the process lives. */ +export const startDiskHistorySampler = () => { + if (state.timer) { + return; + } + const tick = () => { + refreshDiskUsage().catch((error) => { + console.error("Disk usage: sampling failed", error); + }); + }; + state.timer = setInterval(tick, STAT_INTERVAL_MS); + state.timer.unref(); + tick(); +}; + +const getCurrentSample = async (now: number): Promise => { + // Without the sampler (development) a query is what keeps the numbers fresh. + if (!state.stat || now - state.stat.at > 2 * STAT_INTERVAL_MS) { + await refreshDiskUsage().catch((error) => { + console.error("Disk usage: sampling failed", error); + }); + } + if (!state.stat) { + return null; + } + return { + at: state.stat.at, + totalBytes: state.stat.totalBytes, + usedBytes: state.stat.usedBytes, + ...state.parts, + }; +}; + +export const getDiskUsageHistory = async ( + period: DiskHistoryPeriod, + utcOffsetMinutes = 0, + now = Date.now(), +): Promise => { + const currentSample = await getCurrentSample(now); + const files = getHistoryFiles(); + const range = getDiskHistoryRange(period, now, utcOffsetMinutes); + const [samples, snapshots] = await Promise.all([ + readJsonLines(files.samples, range.from), + readJsonLines(files.items, range.from), + ]); + const buckets = bucketDiskSamples( + currentSample ? [...samples, currentSample] : samples, + range, + ); + + const current = currentSample ? toDiskUsageBreakdown(currentSample) : null; + // Item sizes are saved less often than samples; starting from the first + // saved set keeps the category changes and the movers on the same moment. + const startItems = snapshots[0]; + const startSample = + samples.find((sample) => sample.at === startItems?.at) ?? samples[0]; + const start = + startSample && current && startSample.at < current.at + ? toDiskUsageBreakdown(startSample) + : null; + const categories = current ? getDiskCategoryChanges(start, current) : []; + + return { + period, + from: range.from, + to: range.to, + bucketMs: range.bucketMs, + current: + current && state.stat + ? { ...current, availableBytes: state.stat.availableBytes } + : null, + buckets, + changeSince: start?.at ?? null, + usedDeltaBytes: + start && current ? current.usedBytes - start.usedBytes : null, + categories, + movers: + start && startItems?.at === start.at + ? getDiskMovers(startItems.items, state.items, categories) + : [], + }; +}; diff --git a/packages/server/src/monitoring/disk-usage.ts b/packages/server/src/monitoring/disk-usage.ts new file mode 100644 index 00000000000..db1acf09ac8 --- /dev/null +++ b/packages/server/src/monitoring/disk-usage.ts @@ -0,0 +1,381 @@ +import { promises as fs } from "node:fs"; +import path from "node:path"; +import { docker, paths } from "../constants"; +import { quoteShellArg } from "../utils/filesystem/safe-path"; +import { execAsync } from "../utils/process/execAsync"; + +/** What the disk usage is split into; the rest of the used space is "other". */ +export const DISK_PARTS = [ + "images", + "buildCache", + "volumes", + "containerLogs", + "containers", + "dokployData", + "swap", +] as const; + +export type DiskPart = (typeof DISK_PARTS)[number]; +export type DiskCategory = DiskPart | "other"; +export type DiskParts = Partial>; + +export type DiskItemKind = + | "image" + | "volume" + | "container" + | "containerLog" + | "dokployData"; + +/** Sizes of single images, volumes and so on, keyed by `:`. */ +export type DiskItems = Record; + +export interface DiskUsageProbe { + parts: DiskParts; + items: DiskItems; + /** Kinds this probe measured; items of other kinds keep their last value. */ + itemKinds: DiskItemKind[]; +} + +export interface DiskStat { + totalBytes: number; + usedBytes: number; + availableBytes: number; +} + +const PROBE_TIMEOUT_MS = 60_000; +const ANONYMOUS_VOLUME_NAME = /^[a-f0-9]{64}$/; +const CONTAINER_LOGS_MOUNT = "/docker-containers"; + +export const diskItemKey = (kind: DiskItemKind, name: string) => + `${kind}:${name}`; + +export const parseDiskItemKey = (key: string) => { + const separator = key.indexOf(":"); + return { + kind: key.slice(0, separator) as DiskItemKind, + name: key.slice(separator + 1), + }; +}; + +// The same numbers `df /` prints. Dokploy's container root is an overlay on the +// host disk, so this is the host's disk even without /var/lib/docker mounted. +export const getRootDiskStat = async (): Promise => { + const stat = await fs.statfs("/"); + return { + totalBytes: stat.blocks * stat.bsize, + usedBytes: (stat.blocks - stat.bfree) * stat.bsize, + availableBytes: stat.bavail * stat.bsize, + }; +}; + +interface DockerUsageSection { + TotalSize?: number; + Items?: T[] | null; +} + +interface DockerImageUsage { + Id?: string; + RepoTags?: string[] | null; + RepoDigests?: string[] | null; + Size?: number; + SharedSize?: number; +} + +interface DockerContainerUsage { + Id?: string; + Names?: string[] | null; + Labels?: Record | null; + SizeRw?: number; +} + +interface DockerVolumeUsage { + Name?: string; + UsageData?: { Size?: number } | null; +} + +interface DockerBuildCacheUsage { + Size?: number; + Shared?: boolean; +} + +// Docker 29 reports each section twice: the legacy top-level lists and the +// `*Usage` objects that replace them in later API versions. +export interface DockerDiskUsageResponse { + LayersSize?: number; + Images?: DockerImageUsage[] | null; + Containers?: DockerContainerUsage[] | null; + Volumes?: DockerVolumeUsage[] | null; + BuildCache?: DockerBuildCacheUsage[] | null; + ImageUsage?: DockerUsageSection | null; + ContainerUsage?: DockerUsageSection | null; + VolumeUsage?: DockerUsageSection | null; + BuildCacheUsage?: DockerUsageSection | null; +} + +type DockerUsageType = "image" | "container" | "volume" | "build-cache"; + +const getImageRepository = (image: DockerImageUsage) => { + const reference = + image.RepoTags?.find((tag) => !tag.startsWith("")) ?? + image.RepoDigests?.[0]; + if (!reference) { + return "untagged"; + } + const withoutDigest = reference.split("@")[0] ?? reference; + const lastSlash = withoutDigest.lastIndexOf("/"); + const lastColon = withoutDigest.lastIndexOf(":"); + // A colon before the last slash is a registry port, not a tag. + return lastColon > lastSlash + ? withoutDigest.slice(0, lastColon) + : withoutDigest; +}; + +// Swarm gives every task its own container name, so a redeploy would look like +// one container vanishing and another one appearing. +const getContainerName = (container: DockerContainerUsage) => + container.Labels?.["com.docker.swarm.service.name"] ?? + container.Names?.[0]?.replace(/^\//, "") ?? + container.Id?.slice(0, 12) ?? + "unknown"; + +const addBytes = (items: DiskItems, key: string, bytes: number) => { + items[key] = (items[key] ?? 0) + bytes; +}; + +const sum = (values: number[]) => + values.reduce((total, value) => total + value, 0); + +/** + * Turns a `GET /system/df` answer into part totals and per-item sizes. Only the + * sections Docker was asked for are present, so the others stay out of the result. + */ +export const summarizeDockerDiskUsage = ( + usage: DockerDiskUsageResponse, +): DiskUsageProbe => { + const parts: DiskParts = {}; + const items: DiskItems = {}; + const itemKinds: DiskItemKind[] = []; + + const images = usage.ImageUsage?.Items ?? usage.Images; + if (images) { + itemKinds.push("image"); + for (const image of images) { + const size = image.Size ?? 0; + const shared = image.SharedSize ?? -1; + // Docker answers -1 when it did not work out the shared part. + const unique = shared >= 0 ? Math.max(size - shared, 0) : size; + addBytes(items, diskItemKey("image", getImageRepository(image)), unique); + } + parts.images = + usage.ImageUsage?.TotalSize ?? + usage.LayersSize ?? + sum(images.map((image) => image.Size ?? 0)); + } + + const containers = usage.ContainerUsage?.Items ?? usage.Containers; + if (containers) { + itemKinds.push("container"); + for (const container of containers) { + addBytes( + items, + diskItemKey("container", getContainerName(container)), + container.SizeRw ?? 0, + ); + } + parts.containers = + usage.ContainerUsage?.TotalSize ?? + sum(containers.map((container) => container.SizeRw ?? 0)); + } + + const volumes = usage.VolumeUsage?.Items ?? usage.Volumes; + if (volumes) { + itemKinds.push("volume"); + for (const volume of volumes) { + const name = volume.Name ?? "unknown"; + addBytes( + items, + diskItemKey( + "volume", + ANONYMOUS_VOLUME_NAME.test(name) ? "anonymous volumes" : name, + ), + Math.max(volume.UsageData?.Size ?? 0, 0), + ); + } + parts.volumes = + usage.VolumeUsage?.TotalSize ?? + sum(volumes.map((volume) => Math.max(volume.UsageData?.Size ?? 0, 0))); + } + + const buildCache = usage.BuildCacheUsage?.Items ?? usage.BuildCache; + if (buildCache) { + // Shared records are image layers, and the images part already counts them. + parts.buildCache = sum( + buildCache + .filter((record) => !record.Shared) + .map((record) => record.Size ?? 0), + ); + } + + return { parts, items, itemKinds }; +}; + +const requestDockerDiskUsage = (types: DockerUsageType[]) => + new Promise((resolve, reject) => { + docker.modem.dial( + { + // The modem appends the query only to a path that ends with "?". + path: "/system/df?", + method: "GET", + options: { type: types }, + statusCodes: { 200: true, 500: "server error" }, + }, + (error, data) => { + if (error) { + reject(error); + return; + } + resolve((data ?? {}) as DockerDiskUsageResponse); + }, + ); + }); + +/** Parses `du -k` output into bytes per path. */ +export const parseDuOutput = (stdout: string) => { + const sizes = new Map(); + for (const line of stdout.split("\n")) { + const match = line.match(/^(\d+)\s+(.+)$/); + if (match) { + sizes.set(match[2] as string, Number(match[1]) * 1024); + } + } + return sizes; +}; + +/** Bytes of the swap files listed in /proc/swaps; swap partitions take no disk space. */ +export const parseSwapFileBytes = (procSwaps: string) => + sum( + procSwaps + .split("\n") + .slice(1) + .map((line) => line.trim().split(/\s+/)) + // The file name may hold spaces, so the columns are read from the end. + .filter((columns) => columns.at(-4) === "file") + .map((columns) => Number(columns.at(-3)) * 1024) + .filter((bytes) => Number.isFinite(bytes)), + ); + +const mergeProbes = (probes: DiskUsageProbe[]): DiskUsageProbe => ({ + parts: Object.assign({}, ...probes.map((probe) => probe.parts)), + items: Object.assign({}, ...probes.map((probe) => probe.items)), + itemKinds: probes.flatMap((probe) => probe.itemKinds), +}); + +const emptyProbe = (): DiskUsageProbe => ({ + parts: {}, + items: {}, + itemKinds: [], +}); + +const logProbeFailure = (what: string) => (error: unknown) => { + console.error(`Disk usage: could not read ${what}`, error); + return emptyProbe(); +}; + +// Docker keeps container logs under its root directory, which Dokploy's own +// container does not see, so a throwaway container measures them. +const probeContainerLogs = async (): Promise => { + const [{ DockerRootDir }, containers] = await Promise.all([ + docker.info() as Promise<{ DockerRootDir?: string }>, + docker.listContainers({ all: true }), + ]); + if (!DockerRootDir) { + return emptyProbe(); + } + const mount = `${DockerRootDir}/containers:${CONTAINER_LOGS_MOUNT}:ro`; + const { stdout } = await execAsync( + `docker run --rm --network none -v ${quoteShellArg(mount)} busybox du -k -d 1 ${CONTAINER_LOGS_MOUNT}`, + { timeoutMs: PROBE_TIMEOUT_MS }, + ); + + const names = new Map( + containers.map((container) => [container.Id, getContainerName(container)]), + ); + const items: DiskItems = {}; + let total = 0; + for (const [directory, bytes] of parseDuOutput(stdout)) { + if (directory === CONTAINER_LOGS_MOUNT) { + total = bytes; + continue; + } + const id = path.basename(directory); + addBytes( + items, + diskItemKey("containerLog", names.get(id) ?? id.slice(0, 12)), + bytes, + ); + } + return { + parts: { containerLogs: total }, + items, + itemKinds: ["containerLog"], + }; +}; + +const probeDokployData = async (): Promise => { + const { BASE_PATH } = paths(); + // du exits non-zero for a single unreadable file but still prints the sizes. + const { stdout } = await execAsync( + `du -k -x -d 1 ${quoteShellArg(BASE_PATH)} 2>/dev/null || true`, + { timeoutMs: PROBE_TIMEOUT_MS }, + ); + const sizes = parseDuOutput(stdout); + const total = sizes.get(BASE_PATH); + if (total === undefined) { + return emptyProbe(); + } + const items: DiskItems = {}; + for (const [directory, bytes] of sizes) { + if (directory !== BASE_PATH) { + items[diskItemKey("dokployData", path.basename(directory))] = bytes; + } + } + return { + parts: { dokployData: total }, + items, + itemKinds: ["dokployData"], + }; +}; + +const probeSwapFiles = async (): Promise => { + const procSwaps = await fs.readFile("/proc/swaps", "utf-8").catch(() => null); + if (procSwaps === null) { + return emptyProbe(); + } + return { + parts: { swap: parseSwapFileBytes(procSwaps) }, + items: {}, + itemKinds: [], + }; +}; + +/** Images, container layers and build cache: Docker answers from its metadata. */ +export const probeDockerUsage = async (): Promise => + requestDockerDiskUsage(["image", "container", "build-cache"]) + .then(summarizeDockerDiskUsage) + .catch(logProbeFailure("Docker disk usage")); + +/** + * Everything that walks files on disk: volumes (Docker sizes each one with a + * directory walk), container logs, Dokploy's own data and swap files. + */ +export const probeSlowUsage = async (): Promise => + mergeProbes( + await Promise.all([ + requestDockerDiskUsage(["volume"]) + .then(summarizeDockerDiskUsage) + .catch(logProbeFailure("Docker volume sizes")), + probeContainerLogs().catch(logProbeFailure("container log sizes")), + probeDokployData().catch(logProbeFailure("Dokploy data size")), + probeSwapFiles().catch(logProbeFailure("swap files")), + ]), + ); diff --git a/packages/server/src/services/settings.ts b/packages/server/src/services/settings.ts index 570831e4825..08f042509c0 100644 --- a/packages/server/src/services/settings.ts +++ b/packages/server/src/services/settings.ts @@ -553,6 +553,8 @@ export interface UpdateDiskSpace { requiredBytes: number | null; oldImageCount: number; oldImageBytes: number; + /** The old builds a cleanup removes, newest first. */ + oldImages: DokployImageInfo[]; buildCacheBytes: number; } @@ -616,6 +618,7 @@ export const getUpdateDiskSpace = async (): Promise => { (sum, image) => sum + (image.uniqueSizeBytes ?? image.sizeBytes), 0, ), + oldImages: removable, buildCacheBytes, }; };