diff --git a/apps/dokploy/__test__/backup/backup-failure-output.test.ts b/apps/dokploy/__test__/backup/backup-failure-output.test.ts new file mode 100644 index 00000000000..d7e7e0392b3 --- /dev/null +++ b/apps/dokploy/__test__/backup/backup-failure-output.test.ts @@ -0,0 +1,232 @@ +import { spawnSync } from "node:child_process"; +import { + chmodSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { ExecError } from "@dokploy/server/utils/process/ExecError"; +import { REDACTED_SECRET_VALUE } from "@dokploy/server/utils/security/redaction"; +import { parse } from "shell-quote"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + execAsync: vi.fn(), + execAsyncRemote: vi.fn(), +})); + +vi.mock("@dokploy/server/lib/logger", () => ({ + logger: { + error: vi.fn(), + info: vi.fn(), + }, +})); + +vi.mock("@dokploy/server/utils/process/execAsync", () => ({ + execAsync: mocks.execAsync, + execAsyncRemote: mocks.execAsyncRemote, +})); + +const { buildRcloneS3Command, getBackupCommand, runBackupCommand } = + await import("@dokploy/server/utils/backups/utils"); + +const destination = { + accessKey: "access-key", + additionalFlags: [], + bucket: "bucket", + endpoint: "https://s3.example.test", + provider: "AWS", + region: "us-east-1", + secretAccessKey: "s3-secret", +}; + +const postgresBackup = { + backupId: "backup-1", + backupType: "database", + database: "appdb", + databaseType: "postgres", + destinationId: "destination-1", + postgresId: "postgres-1", + postgres: { appName: "postgres-app", databaseUser: "postgres" }, + prefix: "daily", +} as never; + +const rcloneFailureOutput = [ + "Failed to rcat: SignatureDoesNotMatch: status code: 403", + "RCLONE_CONFIG_DOKPLOYS3_SECRET_ACCESS_KEY=leaked-secret", +].join("\n"); + +let tempDir: string; + +beforeEach(() => { + vi.clearAllMocks(); + tempDir = mkdtempSync(join(tmpdir(), "dokploy-backup-failure-")); +}); + +afterEach(() => { + rmSync(tempDir, { force: true, recursive: true }); +}); + +const writeExecutable = (path: string, content: string) => { + writeFileSync(path, content); + chmodSync(path, 0o755); +}; + +const runBackupScript = (rcatExitCode: number) => { + const binDir = join(tempDir, "bin"); + const callLog = join(tempDir, "calls.log"); + const logPath = join(tempDir, "backup.log"); + mkdirSync(binDir); + writeFileSync(callLog, ""); + writeFileSync(logPath, ""); + writeExecutable( + join(binDir, "docker"), + `#!/bin/sh +echo "docker $1" >> "$CALL_LOG" +case "$1" in + ps) echo "container-1" ;; + exec) printf 'dump-bytes' ;; +esac +`, + ); + writeExecutable( + join(binDir, "rclone"), + `#!/bin/sh +echo "rclone $1" >> "$CALL_LOG" +if [ "$1" = "rcat" ]; then + cat > /dev/null + if [ ${rcatExitCode} -ne 0 ]; then +${rcloneFailureOutput + .split("\n") + .map((line) => `\t\techo '${line}' >&2`) + .join("\n")} + exit ${rcatExitCode} + fi +fi +`, + ); + + const target = "dokploys3:bucket/postgres-app/daily/backup.sql.gz"; + const script = getBackupCommand( + postgresBackup, + buildRcloneS3Command("rcat", destination, [target]), + buildRcloneS3Command("deletefile", destination, [target]), + logPath, + ); + const result = spawnSync("/bin/bash", ["-c", script], { + encoding: "utf8", + env: { + CALL_LOG: callLog, + NODE_ENV: "test", + PATH: `${binDir}:/usr/bin:/bin`, + }, + }); + + return { + calls: readFileSync(callLog, "utf8").trim().split("\n"), + log: readFileSync(logPath, "utf8"), + result, + }; +}; + +describe("backup script", () => { + it("dumps the database and uploads it once", () => { + const { calls, log, result } = runBackupScript(0); + + expect(result.status).toBe(0); + expect(calls).toEqual(["docker ps", "docker exec", "rclone rcat"]); + expect(log).toContain("✅ Backup uploaded to S3 successfully"); + expect(log).toContain("Backup done ✅"); + }); + + it("prints failure output to stderr instead of the log and removes the partial upload", () => { + const { calls, log, result } = runBackupScript(1); + + expect(result.status).toBe(1); + expect(calls).toEqual([ + "docker ps", + "docker exec", + "rclone rcat", + "rclone deletefile", + ]); + expect(log).toContain("❌ Error: Backup failed"); + expect(log).not.toContain("SignatureDoesNotMatch"); + expect(log).not.toContain("leaked-secret"); + expect(result.stderr).toContain("SignatureDoesNotMatch"); + }); +}); + +describe("runBackupCommand", () => { + const failedBackup = (serverId?: string) => + new ExecError("Command execution failed", { + command: "backup script", + stderr: `${rcloneFailureOutput}\n`, + exitCode: 1, + serverId, + }); + + it("appends the masked failure output to a local deployment log", async () => { + const logPath = join(tempDir, "backup.log"); + writeFileSync(logPath, "[date] ❌ Error: Backup failed\n"); + mocks.execAsync.mockRejectedValue(failedBackup()); + + await expect( + runBackupCommand("backup script", logPath, null), + ).rejects.toBeInstanceOf(ExecError); + + expect(mocks.execAsync).toHaveBeenCalledWith("backup script", { + shell: "/bin/bash", + }); + const log = readFileSync(logPath, "utf8"); + expect(log).toContain( + "Error: Failed to rcat: SignatureDoesNotMatch: status code: 403", + ); + expect(log).toContain(REDACTED_SECRET_VALUE); + expect(log).not.toContain("leaked-secret"); + }); + + it("appends the masked failure output to a remote deployment log", async () => { + mocks.execAsyncRemote + .mockRejectedValueOnce(failedBackup("server-1")) + .mockResolvedValueOnce({ stdout: "", stderr: "" }); + + await expect( + runBackupCommand( + "backup script", + "/etc/dokploy/logs/app.log", + "server-1", + ), + ).rejects.toBeInstanceOf(ExecError); + + expect(mocks.execAsyncRemote).toHaveBeenCalledTimes(2); + const [serverId, appendCommand] = mocks.execAsyncRemote.mock.calls[1] as [ + string, + string, + ]; + expect(serverId).toBe("server-1"); + const args = parse(appendCommand); + expect(args.slice(0, 2)).toEqual(["printf", "%s\\n"]); + expect(args[2]).toContain( + "Error: Failed to rcat: SignatureDoesNotMatch: status code: 403", + ); + expect(args[2]).toContain(REDACTED_SECRET_VALUE); + expect(appendCommand).not.toContain("leaked-secret"); + expect(args.slice(3)).toEqual([{ op: ">>" }, "/etc/dokploy/logs/app.log"]); + }); + + it("keeps the original error when there is no command output", async () => { + const logPath = join(tempDir, "backup.log"); + writeFileSync(logPath, ""); + mocks.execAsync.mockRejectedValue(new Error("spawn failed")); + + await expect(runBackupCommand("backup script", logPath)).rejects.toThrow( + "spawn failed", + ); + + expect(readFileSync(logPath, "utf8")).toBe(""); + }); +}); diff --git a/apps/dokploy/__test__/backup/rclone-command-boundary.test.ts b/apps/dokploy/__test__/backup/rclone-command-boundary.test.ts index d17a0c87670..dd2b85f6999 100644 --- a/apps/dokploy/__test__/backup/rclone-command-boundary.test.ts +++ b/apps/dokploy/__test__/backup/rclone-command-boundary.test.ts @@ -317,13 +317,11 @@ describe("destination rclone command boundary", () => { const command = mocks.execAsyncRemote.mock.calls[0]?.[1] as string; expect(command).toContain("UPLOAD_OUTPUT="); - expect(command).toContain( - "Error: Backup command failed. Check server logs for details.", - ); - expect(command).toContain( - "Error: Upload command failed. Check server logs for details.", - ); + expect(command).toContain(`printf '%s\\n' "$UPLOAD_OUTPUT" >&2;`); expect(command).not.toContain('echo "Error: $UPLOAD_OUTPUT"'); + expect(command).not.toMatch(/\$UPLOAD_OUTPUT"?\s*>>/); + expect(command.match(/pg_dump/g)).toHaveLength(1); + expect(command.match(/\brcat\b/g)).toHaveLength(1); }); it("wraps retention rclone delete commands after xargs", async () => { diff --git a/apps/dokploy/__test__/backup/restore-security.test.ts b/apps/dokploy/__test__/backup/restore-security.test.ts index 131e962e277..3fca4240a11 100644 --- a/apps/dokploy/__test__/backup/restore-security.test.ts +++ b/apps/dokploy/__test__/backup/restore-security.test.ts @@ -1,4 +1,6 @@ +import * as schema from "@dokploy/server/db/schema"; import { TRPCError } from "@trpc/server"; +import { drizzle } from "drizzle-orm/postgres-js"; import { parse } from "shell-quote"; import { beforeEach, describe, expect, it, vi } from "vitest"; @@ -324,6 +326,40 @@ const emit = (log: string) => emittedLogs.push(log); const parseShellArgs = (command: string) => parse(command).filter((part): part is string => typeof part === "string"); +const POSTGRES_MAX_FUNCTION_ARGS = 100; + +const maxJsonBuildArrayArgs = (sqlText: string) => { + let max = 0; + for (const match of sqlText.matchAll(/json_build_array\(/g)) { + let depth = 1; + let args = 1; + let inIdentifier = false; + for ( + let index = match.index + match[0].length; + index < sqlText.length && depth > 0; + index++ + ) { + const char = sqlText[index]; + if (char === '"') { + inIdentifier = !inIdentifier; + continue; + } + if (inIdentifier) { + continue; + } + if (char === "(") { + depth++; + } else if (char === ")") { + depth--; + } else if (char === "," && depth === 1) { + args++; + } + } + max = Math.max(max, args); + } + return max; +}; + describe("backup destination ownership boundary", () => { beforeEach(() => { vi.clearAllMocks(); @@ -784,6 +820,62 @@ describe("backup restore route boundary", () => { expect(mocks.execAsyncRemote).not.toHaveBeenCalled(); }); + it("keeps backup file listing queries within the Postgres function argument limit", async () => { + const relationalDb = drizzle.mock({ schema }); + const compiledQueries: string[] = []; + mocks.findRestoreBackups.mockImplementation(async (config) => { + compiledQueries.push( + relationalDb.query.backups.findMany(config).toSQL().sql, + ); + return [ + { + appName: "backup-one", + backupType: "database", + databaseType: "postgres", + destinationId: "destination-1", + postgresId: "postgres-1", + prefix: "daily", + postgres: { appName: "app-one" }, + }, + ]; + }); + mocks.findVolumeBackupSchedules.mockImplementation(async (config) => { + compiledQueries.push( + relationalDb.query.volumeBackups.findMany(config).toSQL().sql, + ); + return [ + { + appName: "volume-backup-one", + applicationId: "application-1", + prefix: "volumes", + application: { appName: "web-app" }, + }, + ]; + }); + mocks.execAsync.mockResolvedValue({ stdout: "[]" }); + + await expect( + createCaller().listBackupFiles({ + destinationId: "destination-1", + search: "", + }), + ).resolves.toEqual([]); + + expect(compiledQueries).toHaveLength(2); + for (const sqlText of compiledQueries) { + expect(sqlText).toContain("json_build_array("); + expect(maxJsonBuildArrayArgs(sqlText)).toBeLessThanOrEqual( + POSTGRES_MAX_FUNCTION_ARGS, + ); + } + const listedArgs = mocks.execAsync.mock.calls.map(([command]) => + parseShellArgs((command as string).replace(/\s+2>\/dev\/null$/, "")), + ); + expect(listedArgs).toHaveLength(2); + expect(listedArgs[0]).toContain(":s3:dokploy-backups/app-one/daily/"); + expect(listedArgs[1]).toContain(":s3:dokploy-backups/web-app/volumes/"); + }); + it("requires backup management permission before remote backup file listing", async () => { mocks.getAccessibleServerIds.mockResolvedValue(new Set(["server-1"])); mocks.checkPermission.mockImplementation(async (_ctx, permissions) => { diff --git a/apps/dokploy/__test__/destination/destination-router-assigned-server.test.ts b/apps/dokploy/__test__/destination/destination-router-assigned-server.test.ts index f354dfdeac9..b5bd4a0f41e 100644 --- a/apps/dokploy/__test__/destination/destination-router-assigned-server.test.ts +++ b/apps/dokploy/__test__/destination/destination-router-assigned-server.test.ts @@ -10,13 +10,16 @@ const mocks = vi.hoisted(() => ({ execAsyncRemote: vi.fn(), findDestinationById: vi.fn(), getAccessibleServerIds: vi.fn(), + isCloud: true, removeDestinationById: vi.fn(), assertDestinationEndpointAllowed: vi.fn(), updateDestinationById: vi.fn(), })); vi.mock("@dokploy/server", () => ({ - IS_CLOUD: true, + get IS_CLOUD() { + return mocks.isCloud; + }, createDestination: mocks.createDestination, execAsync: mocks.execAsync, execAsyncRemote: mocks.execAsyncRemote, @@ -83,6 +86,7 @@ const createCaller = () => describe("destination router assigned-server boundary", () => { beforeEach(() => { vi.clearAllMocks(); + mocks.isCloud = true; mocks.checkPermission.mockResolvedValue(undefined); mocks.destinationFindMany.mockResolvedValue([]); mocks.execAsync.mockResolvedValue({ stdout: "", stderr: "" }); @@ -203,4 +207,114 @@ describe("destination router assigned-server boundary", () => { expect.stringContaining("rclone ls"), ); }); + + it("tests a saved destination with the stored secret when the edit form sends the placeholder", async () => { + mocks.isCloud = false; + + await expect( + createCaller().testConnection({ + ...destinationInput, + serverId: undefined, + secretAccessKey: REDACTED_SECRET_VALUE, + destinationId: "destination-1", + }), + ).resolves.toBeUndefined(); + + expect(mocks.findDestinationById).toHaveBeenCalledWith("destination-1"); + const command = mocks.execAsync.mock.calls[0]?.[0] as string; + expect(command).toContain( + "RCLONE_CONFIG_DOKPLOYS3_SECRET_ACCESS_KEY=stored-secret", + ); + expect(command).not.toContain(REDACTED_SECRET_VALUE); + }); + + it("uses a newly entered secret instead of the saved one", async () => { + mocks.isCloud = false; + + await expect( + createCaller().testConnection({ + ...destinationInput, + serverId: undefined, + secretAccessKey: "new-secret", + destinationId: "destination-1", + }), + ).resolves.toBeUndefined(); + + expect(mocks.findDestinationById).not.toHaveBeenCalled(); + expect(mocks.execAsync).toHaveBeenCalledWith( + expect.stringContaining( + "RCLONE_CONFIG_DOKPLOYS3_SECRET_ACCESS_KEY=new-secret", + ), + ); + }); + + it("rejects the placeholder secret without a destinationId before running rclone", async () => { + mocks.isCloud = false; + + await expect( + createCaller().testConnection({ + ...destinationInput, + serverId: undefined, + secretAccessKey: REDACTED_SECRET_VALUE, + }), + ).rejects.toMatchObject({ + code: "BAD_REQUEST", + message: expect.stringContaining("destinationId"), + }); + + expect(mocks.findDestinationById).not.toHaveBeenCalled(); + expect(mocks.execAsync).not.toHaveBeenCalled(); + }); + + it("does not use the stored secret of another organization's destination", async () => { + mocks.isCloud = false; + mocks.findDestinationById.mockResolvedValue({ + destinationId: "destination-2", + secretAccessKey: "other-org-secret", + organizationId: "org-2", + }); + + await expect( + createCaller().testConnection({ + ...destinationInput, + serverId: undefined, + secretAccessKey: REDACTED_SECRET_VALUE, + destinationId: "destination-2", + }), + ).rejects.toMatchObject({ code: "UNAUTHORIZED" }); + + expect(mocks.execAsync).not.toHaveBeenCalled(); + }); + + it("checks cloud server access before loading the stored secret", async () => { + mocks.getAccessibleServerIds.mockResolvedValue(new Set(["server-2"])); + + await expect( + createCaller().testConnection({ + ...destinationInput, + secretAccessKey: REDACTED_SECRET_VALUE, + destinationId: "destination-1", + }), + ).rejects.toMatchObject({ code: "UNAUTHORIZED" }); + + expect(mocks.findDestinationById).not.toHaveBeenCalled(); + expect(mocks.execAsyncRemote).not.toHaveBeenCalled(); + }); + + it("tests a saved destination on an accessible cloud server with the stored secret", async () => { + await expect( + createCaller().testConnection({ + ...destinationInput, + secretAccessKey: REDACTED_SECRET_VALUE, + destinationId: "destination-1", + }), + ).resolves.toBeUndefined(); + + expect(mocks.execAsyncRemote).toHaveBeenCalledWith( + "server-1", + expect.stringContaining( + "RCLONE_CONFIG_DOKPLOYS3_SECRET_ACCESS_KEY=stored-secret", + ), + ); + }); }); diff --git a/apps/dokploy/__test__/security/secret-redaction.test.ts b/apps/dokploy/__test__/security/secret-redaction.test.ts index 012c19f7512..1cd816292d2 100644 --- a/apps/dokploy/__test__/security/secret-redaction.test.ts +++ b/apps/dokploy/__test__/security/secret-redaction.test.ts @@ -309,6 +309,28 @@ describe("shared secret redaction helpers", () => { } }); + it("keeps the argument after a negated password flag while masking real passwords", () => { + expect( + redactSensitiveText( + "pg_dump -Fc -h localhost -U postgres --no-password appdb | gzip", + ), + ).toContain("--no-password appdb |"); + + const messages = [ + "psql --password hunter2 -d appdb", + "tool --db-password=hunter2", + "cli -password hunter2", + "ALTER USER app WITH PASSWORD 'hunter2'", + "CREATE USER 'app'@'%' IDENTIFIED BY 'hunter2'", + ]; + for (const message of messages) { + const redacted = redactSensitiveText(message); + + expect(redacted).not.toContain("hunter2"); + expect(redacted).toContain(REDACTED_SECRET_VALUE); + } + }); + it("stores only redacted command output on ExecError", () => { const error = new ExecError( "Command failed: npm run build TOKEN=build-secret", diff --git a/apps/dokploy/components/dashboard/settings/destination/handle-destinations.tsx b/apps/dokploy/components/dashboard/settings/destination/handle-destinations.tsx index d1542dc8038..3823b34e44d 100644 --- a/apps/dokploy/components/dashboard/settings/destination/handle-destinations.tsx +++ b/apps/dokploy/components/dashboard/settings/destination/handle-destinations.tsx @@ -199,6 +199,7 @@ export const HandleDestinations = ({ destinationId }: Props) => { const connectionString = `:s3,provider=${provider},access_key_id=${accessKey},secret_access_key=${secretKey},endpoint=${endpoint}${region ? `,region=${region}` : ""}:${bucket}`; await testConnection({ + destinationId, provider, accessKey, bucket, diff --git a/apps/dokploy/server/api/routers/backup.ts b/apps/dokploy/server/api/routers/backup.ts index 7b035928699..54a1e753f00 100644 --- a/apps/dokploy/server/api/routers/backup.ts +++ b/apps/dokploy/server/api/routers/backup.ts @@ -518,11 +518,14 @@ const getAccessibleBackupListingPrefixes = async ( } } + // The application table has more than 100 columns, and Drizzle packs every + // selected column of a nested relation into one json_build_array() call, + // which Postgres rejects (54023). Only appName is needed for the prefix. const volumeBackupSchedules = await db.query.volumeBackups.findMany({ where: eq(volumeBackups.destinationId, destinationId), with: { - application: true, - compose: true, + application: { columns: { appName: true } }, + compose: { columns: { appName: true } }, }, }); diff --git a/apps/dokploy/server/api/routers/destination.ts b/apps/dokploy/server/api/routers/destination.ts index 488ba0225e6..7c7a3221027 100644 --- a/apps/dokploy/server/api/routers/destination.ts +++ b/apps/dokploy/server/api/routers/destination.ts @@ -27,6 +27,7 @@ import { apiCreateDestination, apiFindOneDestination, apiRemoveDestination, + apiTestDestinationConnection, apiUpdateDestination, destinations, } from "@/server/db/schema"; @@ -86,7 +87,7 @@ export const destinationRouter = createTRPCRouter({ } }), testConnection: withPermission("destination", "create") - .input(apiCreateDestination) + .input(apiTestDestinationConnection) .mutation(async ({ input, ctx }) => { if (IS_CLOUD && !input.serverId) { throw new TRPCError({ @@ -98,8 +99,28 @@ export const destinationRouter = createTRPCRouter({ await assertDestinationServerAccess(ctx, input.serverId); } + const { destinationId, ...connectionInput } = input; + if (isRedactedSecretValue(connectionInput.secretAccessKey)) { + if (!destinationId) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: + "The secret access key is hidden. Enter it again or pass destinationId to test the saved destination.", + }); + } + const destination = await findDestinationById(destinationId); + if (destination.organizationId !== ctx.session.activeOrganizationId) { + throw new TRPCError({ + code: "UNAUTHORIZED", + message: "You are not allowed to access this destination", + }); + } + connectionInput.secretAccessKey = destination.secretAccessKey; + } + try { - const destinationInput = await normalizeDestinationEndpointInput(input); + const destinationInput = + await normalizeDestinationEndpointInput(connectionInput); const rcloneCommand = buildRcloneS3Command("ls", destinationInput, [ "--retries", "1", diff --git a/packages/server/src/db/schema/destination.ts b/packages/server/src/db/schema/destination.ts index 11d9f6997c3..beb26b6029a 100644 --- a/packages/server/src/db/schema/destination.ts +++ b/packages/server/src/db/schema/destination.ts @@ -77,6 +77,10 @@ export const apiCreateDestination = createSchema serverId: z.string().optional(), }); +export const apiTestDestinationConnection = apiCreateDestination.extend({ + destinationId: z.string().min(1).optional(), +}); + export const apiFindOneDestination = z.object({ destinationId: z.string().min(1), }); diff --git a/packages/server/src/utils/backups/compose.ts b/packages/server/src/utils/backups/compose.ts index 0fc2468bb4f..86ba82ca8dd 100644 --- a/packages/server/src/utils/backups/compose.ts +++ b/packages/server/src/utils/backups/compose.ts @@ -8,7 +8,6 @@ import { findDestinationById } from "@dokploy/server/services/destination"; import { findEnvironmentById } from "@dokploy/server/services/environment"; import { findProjectById } from "@dokploy/server/services/project"; import { sendDatabaseBackupNotifications } from "../notifications/database-backup"; -import { execAsync, execAsyncRemote } from "../process/execAsync"; import { assertRcloneS3DestinationAllowed, buildRcloneS3Command, @@ -16,6 +15,7 @@ import { getBackupTimestamp, getRcloneS3Destination, normalizeS3Path, + runBackupCommand, } from "./utils"; export const runComposeBackup = async ( @@ -57,13 +57,7 @@ export const runComposeBackup = async ( rcloneDeleteCommand, deployment.logPath, ); - if (compose.serverId) { - await execAsyncRemote(compose.serverId, backupCommand); - } else { - await execAsync(backupCommand, { - shell: "/bin/bash", - }); - } + await runBackupCommand(backupCommand, deployment.logPath, compose.serverId); await sendDatabaseBackupNotifications({ applicationName: name, diff --git a/packages/server/src/utils/backups/libsql.ts b/packages/server/src/utils/backups/libsql.ts index 45fb14b2fa9..5a32dc94065 100644 --- a/packages/server/src/utils/backups/libsql.ts +++ b/packages/server/src/utils/backups/libsql.ts @@ -8,7 +8,6 @@ import { findEnvironmentById } from "@dokploy/server/services/environment"; import type { Libsql } from "@dokploy/server/services/libsql"; import { findProjectById } from "@dokploy/server/services/project"; import { sendDatabaseBackupNotifications } from "../notifications/database-backup"; -import { execAsync, execAsyncRemote } from "../process/execAsync"; import { assertRcloneS3DestinationAllowed, buildRcloneS3Command, @@ -16,6 +15,7 @@ import { getBackupTimestamp, getRcloneS3Destination, normalizeS3Path, + runBackupCommand, } from "./utils"; export const runLibsqlBackup = async ( @@ -56,13 +56,7 @@ export const runLibsqlBackup = async ( rcloneDeleteCommand, deployment.logPath, ); - if (libsql.serverId) { - await execAsyncRemote(libsql.serverId, backupCommand); - } else { - await execAsync(backupCommand, { - shell: "/bin/bash", - }); - } + await runBackupCommand(backupCommand, deployment.logPath, libsql.serverId); await sendDatabaseBackupNotifications({ applicationName: name, diff --git a/packages/server/src/utils/backups/mariadb.ts b/packages/server/src/utils/backups/mariadb.ts index 73a051a6b5e..35a56807fe1 100644 --- a/packages/server/src/utils/backups/mariadb.ts +++ b/packages/server/src/utils/backups/mariadb.ts @@ -8,7 +8,6 @@ import { findEnvironmentById } from "@dokploy/server/services/environment"; import type { Mariadb } from "@dokploy/server/services/mariadb"; import { findProjectById } from "@dokploy/server/services/project"; import { sendDatabaseBackupNotifications } from "../notifications/database-backup"; -import { execAsync, execAsyncRemote } from "../process/execAsync"; import { assertRcloneS3DestinationAllowed, buildRcloneS3Command, @@ -16,6 +15,7 @@ import { getBackupTimestamp, getRcloneS3Destination, normalizeS3Path, + runBackupCommand, } from "./utils"; export const runMariadbBackup = async ( @@ -55,13 +55,7 @@ export const runMariadbBackup = async ( rcloneDeleteCommand, deployment.logPath, ); - if (mariadb.serverId) { - await execAsyncRemote(mariadb.serverId, backupCommand); - } else { - await execAsync(backupCommand, { - shell: "/bin/bash", - }); - } + await runBackupCommand(backupCommand, deployment.logPath, mariadb.serverId); await sendDatabaseBackupNotifications({ applicationName: name, diff --git a/packages/server/src/utils/backups/mongo.ts b/packages/server/src/utils/backups/mongo.ts index 2ea1bf71c9c..529fea64e01 100644 --- a/packages/server/src/utils/backups/mongo.ts +++ b/packages/server/src/utils/backups/mongo.ts @@ -8,7 +8,6 @@ import { findEnvironmentById } from "@dokploy/server/services/environment"; import type { Mongo } from "@dokploy/server/services/mongo"; import { findProjectById } from "@dokploy/server/services/project"; import { sendDatabaseBackupNotifications } from "../notifications/database-backup"; -import { execAsync, execAsyncRemote } from "../process/execAsync"; import { assertRcloneS3DestinationAllowed, buildRcloneS3Command, @@ -16,6 +15,7 @@ import { getBackupTimestamp, getRcloneS3Destination, normalizeS3Path, + runBackupCommand, } from "./utils"; export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => { @@ -53,13 +53,7 @@ export const runMongoBackup = async (mongo: Mongo, backup: BackupSchedule) => { deployment.logPath, ); - if (mongo.serverId) { - await execAsyncRemote(mongo.serverId, backupCommand); - } else { - await execAsync(backupCommand, { - shell: "/bin/bash", - }); - } + await runBackupCommand(backupCommand, deployment.logPath, mongo.serverId); await sendDatabaseBackupNotifications({ applicationName: name, diff --git a/packages/server/src/utils/backups/mysql.ts b/packages/server/src/utils/backups/mysql.ts index 98ae21336a0..62987b304dd 100644 --- a/packages/server/src/utils/backups/mysql.ts +++ b/packages/server/src/utils/backups/mysql.ts @@ -8,7 +8,6 @@ import { findEnvironmentById } from "@dokploy/server/services/environment"; import type { MySql } from "@dokploy/server/services/mysql"; import { findProjectById } from "@dokploy/server/services/project"; import { sendDatabaseBackupNotifications } from "../notifications/database-backup"; -import { execAsync, execAsyncRemote } from "../process/execAsync"; import { assertRcloneS3DestinationAllowed, buildRcloneS3Command, @@ -16,6 +15,7 @@ import { getBackupTimestamp, getRcloneS3Destination, normalizeS3Path, + runBackupCommand, } from "./utils"; export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => { @@ -54,13 +54,7 @@ export const runMySqlBackup = async (mysql: MySql, backup: BackupSchedule) => { deployment.logPath, ); - if (mysql.serverId) { - await execAsyncRemote(mysql.serverId, backupCommand); - } else { - await execAsync(backupCommand, { - shell: "/bin/bash", - }); - } + await runBackupCommand(backupCommand, deployment.logPath, mysql.serverId); await sendDatabaseBackupNotifications({ applicationName: name, projectName: project.name, diff --git a/packages/server/src/utils/backups/postgres.ts b/packages/server/src/utils/backups/postgres.ts index 617c604cbf6..165fc4da2c5 100644 --- a/packages/server/src/utils/backups/postgres.ts +++ b/packages/server/src/utils/backups/postgres.ts @@ -8,7 +8,6 @@ import { findEnvironmentById } from "@dokploy/server/services/environment"; import type { Postgres } from "@dokploy/server/services/postgres"; import { findProjectById } from "@dokploy/server/services/project"; import { sendDatabaseBackupNotifications } from "../notifications/database-backup"; -import { execAsync, execAsyncRemote } from "../process/execAsync"; import { assertRcloneS3DestinationAllowed, buildRcloneS3Command, @@ -16,6 +15,7 @@ import { getBackupTimestamp, getRcloneS3Destination, normalizeS3Path, + runBackupCommand, } from "./utils"; export const runPostgresBackup = async ( @@ -56,13 +56,11 @@ export const runPostgresBackup = async ( rcloneDeleteCommand, deployment.logPath, ); - if (postgres.serverId) { - await execAsyncRemote(postgres.serverId, backupCommand); - } else { - await execAsync(backupCommand, { - shell: "/bin/bash", - }); - } + await runBackupCommand( + backupCommand, + deployment.logPath, + postgres.serverId, + ); await sendDatabaseBackupNotifications({ applicationName: name, diff --git a/packages/server/src/utils/backups/utils.ts b/packages/server/src/utils/backups/utils.ts index 896608eaaba..8f5deeb023e 100644 --- a/packages/server/src/utils/backups/utils.ts +++ b/packages/server/src/utils/backups/utils.ts @@ -1,3 +1,4 @@ +import { appendFile } from "node:fs/promises"; import { assertRcloneAdditionalFlagsAllowed } from "@dokploy/server/db/validations/destination"; import { logger } from "@dokploy/server/lib/logger"; import type { BackupSchedule } from "@dokploy/server/services/backup"; @@ -6,6 +7,11 @@ import { assertDestinationEndpointAllowed, normalizeDestinationEndpointUrl, } from "@dokploy/server/utils/destination/endpoint"; +import { ExecError } from "@dokploy/server/utils/process/ExecError"; +import { + execAsync, + execAsyncRemote, +} from "@dokploy/server/utils/process/execAsync"; import { normalizeRestoreDatabaseName, normalizeRestoreServiceName, @@ -437,7 +443,7 @@ export const generateBackupCommand = (backup: BackupSchedule) => { export const getBackupCommand = ( backup: BackupSchedule, rcloneCommand: string, - _rcloneDeleteCommand: string, + rcloneDeleteCommand: string, logPath: string, ) => { if (!isBackupScheduleTargetBound(backup)) { @@ -477,17 +483,8 @@ export const getBackupCommand = ( UPLOAD_OUTPUT=$({ ${backupCommand} | ${rcloneCommand}; } 2>&1 >/dev/null) || { echo "[$(date)] ❌ Error: Backup failed" >> ${logPath}; - echo "Error: Backup command failed. Check server logs for details." >> ${logPath}; - exit 1; - } - - echo "[$(date)] ✅ backup completed successfully" >> ${logPath}; - echo "[$(date)] Starting upload to S3..." >> ${logPath}; - - # Run the upload command and capture the exit status - UPLOAD_OUTPUT=$(${backupCommand} | ${rcloneCommand} 2>&1 >/dev/null) || { - echo "[$(date)] ❌ Error: Upload to S3 failed" >> ${logPath}; - echo "Error: Upload command failed. Check server logs for details." >> ${logPath}; + ${rcloneDeleteCommand} >/dev/null 2>&1 || true; + printf '%s\\n' "$UPLOAD_OUTPUT" >&2; exit 1; }; @@ -495,3 +492,53 @@ export const getBackupCommand = ( echo "Backup done ✅" >> ${logPath}; `; }; + +const appendBackupErrorOutput = async ( + error: unknown, + logPath: string, + serverId?: string | null, +) => { + const output = + error instanceof ExecError ? redactSensitiveText(error.stderr)?.trim() : ""; + if (!output) { + return; + } + + const logLine = `Error: ${output}`; + try { + if (serverId) { + await execAsyncRemote( + serverId, + `printf '%s\\n' ${quoteShellArgument(logLine)} >> ${quoteShellArgument(logPath)}`, + ); + } else { + await appendFile(logPath, `${logLine}\n`); + } + } catch (appendError) { + logger.error( + { logPath, error: redactSensitiveText(String(appendError)) }, + "Failed to append backup error output to the deployment log", + ); + } +}; + +// The backup script never writes command output to the log itself: it prints +// it to stderr, and it is masked here before it reaches the deployment log. +export const runBackupCommand = async ( + command: string, + logPath: string, + serverId?: string | null, +) => { + try { + if (serverId) { + await execAsyncRemote(serverId, command); + } else { + await execAsync(command, { + shell: "/bin/bash", + }); + } + } catch (error) { + await appendBackupErrorOutput(error, logPath, serverId); + throw error; + } +}; diff --git a/packages/server/src/utils/security/redaction.ts b/packages/server/src/utils/security/redaction.ts index 20bec645829..a55be3f9ef4 100644 --- a/packages/server/src/utils/security/redaction.ts +++ b/packages/server/src/utils/security/redaction.ts @@ -335,16 +335,18 @@ export function redactSensitiveText(value: string | null | undefined) { `$1${REDACTED_SECRET_VALUE}`, ); + // Negated flags such as pg_dump's `--no-password` take no value, so the word + // after them is a positional argument (the database name), not a secret. redacted = redacted.replace( new RegExp( - `(\\s--?[a-z0-9-]*${SENSITIVE_KEY_PATTERN}[a-z0-9-]*(?:=|\\s+))("[^"]*"|'[^']*'|[^\\s;&|]+)`, + `(\\s(?!--?no-)--?[a-z0-9-]*${SENSITIVE_KEY_PATTERN}[a-z0-9-]*(?:=|\\s+))("[^"]*"|'[^']*'|[^\\s;&|]+)`, "gi", ), `$1${REDACTED_SECRET_VALUE}`, ); redacted = redacted.replace( - /(\b(?:PASSWORD|IDENTIFIED\s+BY)\s+)('[^']*'|"[^"]*"|[^\s;&|]+)/gi, + /(\b(?