fix: repair s3 backup connection test, file listing and failure logs

apps/dokploy/__test__/backup/backup-failure-output.test.ts

@@ -0,0 +1,232 @@
+import { spawnSync } from "node:child_process";
+import {
+	chmodSync,
+	mkdirSync,
+	mkdtempSync,
+	readFileSync,
+	rmSync,
+	writeFileSync,
+} from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { ExecError } from "@dokploy/server/utils/process/ExecError";
+import { REDACTED_SECRET_VALUE } from "@dokploy/server/utils/security/redaction";
+import { parse } from "shell-quote";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+
+const mocks = vi.hoisted(() => ({
+	execAsync: vi.fn(),
+	execAsyncRemote: vi.fn(),
+}));
+
+vi.mock("@dokploy/server/lib/logger", () => ({
+	logger: {
+		error: vi.fn(),
+		info: vi.fn(),
+	},
+}));
+
+vi.mock("@dokploy/server/utils/process/execAsync", () => ({
+	execAsync: mocks.execAsync,
+	execAsyncRemote: mocks.execAsyncRemote,
+}));
+
+const { buildRcloneS3Command, getBackupCommand, runBackupCommand } =
+	await import("@dokploy/server/utils/backups/utils");
+
+const destination = {
+	accessKey: "access-key",
+	additionalFlags: [],
+	bucket: "bucket",
+	endpoint: "https://s3.example.test",
+	provider: "AWS",
+	region: "us-east-1",
+	secretAccessKey: "s3-secret",
+};
+
+const postgresBackup = {
+	backupId: "backup-1",
+	backupType: "database",
+	database: "appdb",
+	databaseType: "postgres",
+	destinationId: "destination-1",
+	postgresId: "postgres-1",
+	postgres: { appName: "postgres-app", databaseUser: "postgres" },
+	prefix: "daily",
+} as never;
+
+const rcloneFailureOutput = [
+	"Failed to rcat: SignatureDoesNotMatch: status code: 403",
+	"RCLONE_CONFIG_DOKPLOYS3_SECRET_ACCESS_KEY=leaked-secret",
+].join("\n");
+
+let tempDir: string;
+
+beforeEach(() => {
+	vi.clearAllMocks();
+	tempDir = mkdtempSync(join(tmpdir(), "dokploy-backup-failure-"));
+});
+
+afterEach(() => {
+	rmSync(tempDir, { force: true, recursive: true });
+});
+
+const writeExecutable = (path: string, content: string) => {
+	writeFileSync(path, content);
+	chmodSync(path, 0o755);
+};
+
+const runBackupScript = (rcatExitCode: number) => {
+	const binDir = join(tempDir, "bin");
+	const callLog = join(tempDir, "calls.log");
+	const logPath = join(tempDir, "backup.log");
+	mkdirSync(binDir);
+	writeFileSync(callLog, "");
+	writeFileSync(logPath, "");
+	writeExecutable(
+		join(binDir, "docker"),
+		`#!/bin/sh
+echo "docker $1" >> "$CALL_LOG"
+case "$1" in
+	ps) echo "container-1" ;;
+	exec) printf 'dump-bytes' ;;
+esac
+`,
+	);
+	writeExecutable(
+		join(binDir, "rclone"),
+		`#!/bin/sh
+echo "rclone $1" >> "$CALL_LOG"
+if [ "$1" = "rcat" ]; then
+	cat > /dev/null
+	if [ ${rcatExitCode} -ne 0 ]; then
+${rcloneFailureOutput
+	.split("\n")
+	.map((line) => `\t\techo '${line}' >&2`)
+	.join("\n")}
+		exit ${rcatExitCode}
+	fi
+fi
+`,
+	);
+
+	const target = "dokploys3:bucket/postgres-app/daily/backup.sql.gz";
+	const script = getBackupCommand(
+		postgresBackup,
+		buildRcloneS3Command("rcat", destination, [target]),
+		buildRcloneS3Command("deletefile", destination, [target]),
+		logPath,
+	);
+	const result = spawnSync("/bin/bash", ["-c", script], {
+		encoding: "utf8",
+		env: {
+			CALL_LOG: callLog,
+			NODE_ENV: "test",
+			PATH: `${binDir}:/usr/bin:/bin`,
+		},
+	});
+
+	return {
+		calls: readFileSync(callLog, "utf8").trim().split("\n"),
+		log: readFileSync(logPath, "utf8"),
+		result,
+	};
+};
+
+describe("backup script", () => {
+	it("dumps the database and uploads it once", () => {
+		const { calls, log, result } = runBackupScript(0);
+
+		expect(result.status).toBe(0);
+		expect(calls).toEqual(["docker ps", "docker exec", "rclone rcat"]);
+		expect(log).toContain("✅ Backup uploaded to S3 successfully");
+		expect(log).toContain("Backup done ✅");
+	});
+
+	it("prints failure output to stderr instead of the log and removes the partial upload", () => {
+		const { calls, log, result } = runBackupScript(1);
+
+		expect(result.status).toBe(1);
+		expect(calls).toEqual([
+			"docker ps",
+			"docker exec",
+			"rclone rcat",
+			"rclone deletefile",
+		]);
+		expect(log).toContain("❌ Error: Backup failed");
+		expect(log).not.toContain("SignatureDoesNotMatch");
+		expect(log).not.toContain("leaked-secret");
+		expect(result.stderr).toContain("SignatureDoesNotMatch");
+	});
+});
+
+describe("runBackupCommand", () => {
+	const failedBackup = (serverId?: string) =>
+		new ExecError("Command execution failed", {
+			command: "backup script",
+			stderr: `${rcloneFailureOutput}\n`,
+			exitCode: 1,
+			serverId,
+		});
+
+	it("appends the masked failure output to a local deployment log", async () => {
+		const logPath = join(tempDir, "backup.log");
+		writeFileSync(logPath, "[date] ❌ Error: Backup failed\n");
+		mocks.execAsync.mockRejectedValue(failedBackup());
+
+		await expect(
+			runBackupCommand("backup script", logPath, null),
+		).rejects.toBeInstanceOf(ExecError);
+
+		expect(mocks.execAsync).toHaveBeenCalledWith("backup script", {
+			shell: "/bin/bash",
+		});
+		const log = readFileSync(logPath, "utf8");
+		expect(log).toContain(
+			"Error: Failed to rcat: SignatureDoesNotMatch: status code: 403",
+		);
+		expect(log).toContain(REDACTED_SECRET_VALUE);
+		expect(log).not.toContain("leaked-secret");
+	});
+
+	it("appends the masked failure output to a remote deployment log", async () => {
+		mocks.execAsyncRemote
+			.mockRejectedValueOnce(failedBackup("server-1"))
+			.mockResolvedValueOnce({ stdout: "", stderr: "" });
+
+		await expect(
+			runBackupCommand(
+				"backup script",
+				"/etc/dokploy/logs/app.log",
+				"server-1",
+			),
+		).rejects.toBeInstanceOf(ExecError);
+
+		expect(mocks.execAsyncRemote).toHaveBeenCalledTimes(2);
+		const [serverId, appendCommand] = mocks.execAsyncRemote.mock.calls[1] as [
+			string,
+			string,
+		];
+		expect(serverId).toBe("server-1");
+		const args = parse(appendCommand);
+		expect(args.slice(0, 2)).toEqual(["printf", "%s\\n"]);
+		expect(args[2]).toContain(
+			"Error: Failed to rcat: SignatureDoesNotMatch: status code: 403",
+		);
+		expect(args[2]).toContain(REDACTED_SECRET_VALUE);
+		expect(appendCommand).not.toContain("leaked-secret");
+		expect(args.slice(3)).toEqual([{ op: ">>" }, "/etc/dokploy/logs/app.log"]);
+	});
+
+	it("keeps the original error when there is no command output", async () => {
+		const logPath = join(tempDir, "backup.log");
+		writeFileSync(logPath, "");
+		mocks.execAsync.mockRejectedValue(new Error("spawn failed"));
+
+		await expect(runBackupCommand("backup script", logPath)).rejects.toThrow(
+			"spawn failed",
+		);
+
+		expect(readFileSync(logPath, "utf8")).toBe("");
+	});
+});

apps/dokploy/__test__/backup/rclone-command-boundary.test.ts

@@ -317,13 +317,11 @@ describe("destination rclone command boundary", () => {
 		const command = mocks.execAsyncRemote.mock.calls[0]?.[1] as string;
 
 		expect(command).toContain("UPLOAD_OUTPUT=");
-		expect(command).toContain(
-			"Error: Backup command failed. Check server logs for details.",
-		);
-		expect(command).toContain(
-			"Error: Upload command failed. Check server logs for details.",
-		);
+		expect(command).toContain(`printf '%s\\n' "$UPLOAD_OUTPUT" >&2;`);
 		expect(command).not.toContain('echo "Error: $UPLOAD_OUTPUT"');
+		expect(command).not.toMatch(/\$UPLOAD_OUTPUT"?\s*>>/);
+		expect(command.match(/pg_dump/g)).toHaveLength(1);
+		expect(command.match(/\brcat\b/g)).toHaveLength(1);
 	});
 
 	it("wraps retention rclone delete commands after xargs", async () => {

apps/dokploy/__test__/backup/restore-security.test.ts

@@ -1,4 +1,6 @@
+import * as schema from "@dokploy/server/db/schema";
 import { TRPCError } from "@trpc/server";
+import { drizzle } from "drizzle-orm/postgres-js";
 import { parse } from "shell-quote";
 import { beforeEach, describe, expect, it, vi } from "vitest";
 
@@ -324,6 +326,40 @@ const emit = (log: string) => emittedLogs.push(log);
 const parseShellArgs = (command: string) =>
 	parse(command).filter((part): part is string => typeof part === "string");
 
+const POSTGRES_MAX_FUNCTION_ARGS = 100;
+
+const maxJsonBuildArrayArgs = (sqlText: string) => {
+	let max = 0;
+	for (const match of sqlText.matchAll(/json_build_array\(/g)) {
+		let depth = 1;
+		let args = 1;
+		let inIdentifier = false;
+		for (
+			let index = match.index + match[0].length;
+			index < sqlText.length && depth > 0;
+			index++
+		) {
+			const char = sqlText[index];
+			if (char === '"') {
+				inIdentifier = !inIdentifier;
+				continue;
+			}
+			if (inIdentifier) {
+				continue;
+			}
+			if (char === "(") {
+				depth++;
+			} else if (char === ")") {
+				depth--;
+			} else if (char === "," && depth === 1) {
+				args++;
+			}
+		}
+		max = Math.max(max, args);
+	}
+	return max;
+};
+
 describe("backup destination ownership boundary", () => {
 	beforeEach(() => {
 		vi.clearAllMocks();
@@ -784,6 +820,62 @@ describe("backup restore route boundary", () => {
 		expect(mocks.execAsyncRemote).not.toHaveBeenCalled();
 	});
 
+	it("keeps backup file listing queries within the Postgres function argument limit", async () => {
+		const relationalDb = drizzle.mock({ schema });
+		const compiledQueries: string[] = [];
+		mocks.findRestoreBackups.mockImplementation(async (config) => {
+			compiledQueries.push(
+				relationalDb.query.backups.findMany(config).toSQL().sql,
+			);
+			return [
+				{
+					appName: "backup-one",
+					backupType: "database",
+					databaseType: "postgres",
+					destinationId: "destination-1",
+					postgresId: "postgres-1",
+					prefix: "daily",
+					postgres: { appName: "app-one" },
+				},
+			];
+		});
+		mocks.findVolumeBackupSchedules.mockImplementation(async (config) => {
+			compiledQueries.push(
+				relationalDb.query.volumeBackups.findMany(config).toSQL().sql,
+			);
+			return [
+				{
+					appName: "volume-backup-one",
+					applicationId: "application-1",
+					prefix: "volumes",
+					application: { appName: "web-app" },
+				},
+			];
+		});
+		mocks.execAsync.mockResolvedValue({ stdout: "[]" });
+
+		await expect(
+			createCaller().listBackupFiles({
+				destinationId: "destination-1",
+				search: "",
+			}),
+		).resolves.toEqual([]);
+
+		expect(compiledQueries).toHaveLength(2);
+		for (const sqlText of compiledQueries) {
+			expect(sqlText).toContain("json_build_array(");
+			expect(maxJsonBuildArrayArgs(sqlText)).toBeLessThanOrEqual(
+				POSTGRES_MAX_FUNCTION_ARGS,
+			);
+		}
+		const listedArgs = mocks.execAsync.mock.calls.map(([command]) =>
+			parseShellArgs((command as string).replace(/\s+2>\/dev\/null$/, "")),
+		);
+		expect(listedArgs).toHaveLength(2);
+		expect(listedArgs[0]).toContain(":s3:dokploy-backups/app-one/daily/");
+		expect(listedArgs[1]).toContain(":s3:dokploy-backups/web-app/volumes/");
+	});
+
 	it("requires backup management permission before remote backup file listing", async () => {
 		mocks.getAccessibleServerIds.mockResolvedValue(new Set(["server-1"]));
 		mocks.checkPermission.mockImplementation(async (_ctx, permissions) => {