diff --git a/apps/dokploy/__test__/backup/rclone-command-boundary.test.ts b/apps/dokploy/__test__/backup/rclone-command-boundary.test.ts index dd2b85f6999..45c2f883e28 100644 --- a/apps/dokploy/__test__/backup/rclone-command-boundary.test.ts +++ b/apps/dokploy/__test__/backup/rclone-command-boundary.test.ts @@ -195,19 +195,36 @@ describe("destination rclone command boundary", () => { additionalFlags: dangerousDestination.additionalFlags, serverId: "none", }), - ).resolves.toBeUndefined(); - - const command = mocks.execAsync.mock.calls[0]?.[0] as string; - const args = parseShellArgs(command); - const rcloneIndex = args.indexOf("rclone"); + ).resolves.toEqual({ + read: { ok: true }, + write: { ok: true }, + delete: { ok: true }, + }); - expect(args.slice(rcloneIndex, rcloneIndex + 2)).toEqual(["rclone", "ls"]); - expectS3CredentialsAsEnvironment(command, args); - expect(args).toContain(`dokploys3:${dangerousDestination.bucket}`); - expect(command).not.toContain( - '--s3-access-key-id="AKIA; touch /tmp/access"', + const commands = mocks.execAsync.mock.calls.map( + ([command]) => command as string, ); - expect(command).not.toContain('":s3:bucket$(id);touch"'); + expect(commands).toHaveLength(3); + for (const [index, operation] of ["ls", "rcat", "deletefile"].entries()) { + const command = commands[index] ?? ""; + const args = parseShellArgs(command); + const rcloneIndex = args.indexOf("rclone"); + + expect(args.slice(rcloneIndex, rcloneIndex + 2)).toEqual([ + "rclone", + operation, + ]); + expectS3CredentialsAsEnvironment(command, args); + expect(args.at(-1)).toMatch( + operation === "ls" + ? `dokploys3:${dangerousDestination.bucket}` + : `dokploys3:${dangerousDestination.bucket}/dokploy-connection-test/`, + ); + expect(command).not.toContain( + '--s3-access-key-id="AKIA; touch /tmp/access"', + ); + expect(command).not.toContain('":s3:bucket$(id);touch"'); + } }); it("quotes destination fields in scheduled backup upload rclone commands", async () => { diff --git a/apps/dokploy/__test__/destination/destination-connection-report.test.ts b/apps/dokploy/__test__/destination/destination-connection-report.test.ts new file mode 100644 index 00000000000..fbc19b4b971 --- /dev/null +++ b/apps/dokploy/__test__/destination/destination-connection-report.test.ts @@ -0,0 +1,117 @@ +import { describeRcloneFailure } from "@dokploy/server/utils/destination/connection-test"; +import { + type DestinationConnectionTestReport, + formatDestinationConnectionTestReport, + isDestinationConnectionTestPassed, + parseDestinationConnectionTestReport, +} from "@dokploy/server/utils/destination/connection-test-report"; +import { describe, expect, it } from "vitest"; + +const readOnlyKeyReport: DestinationConnectionTestReport = { + read: { ok: true }, + write: { + ok: false, + reason: "403 AccessDenied (the key cannot upload objects)", + }, + delete: { ok: false, skipped: true, reason: "no test object was uploaded" }, +}; + +describe("destination connection test report", () => { + it("formats one line per check and parses it back for the dialog", () => { + const message = formatDestinationConnectionTestReport(readOnlyKeyReport); + + expect(message).toBe( + [ + "Read: OK", + "Write: FAILED - 403 AccessDenied (the key cannot upload objects)", + "Delete: SKIPPED - no test object was uploaded", + ].join("\n"), + ); + expect(parseDestinationConnectionTestReport(message)).toEqual( + readOnlyKeyReport, + ); + expect(isDestinationConnectionTestPassed(readOnlyKeyReport)).toBe(false); + expect( + isDestinationConnectionTestPassed({ + read: { ok: true }, + write: { ok: true }, + delete: { ok: true }, + }), + ).toBe(true); + }); + + it("does not treat other errors as a check report", () => { + expect( + parseDestinationConnectionTestReport( + "S3 endpoint host is not allowed in cloud deployments", + ), + ).toBeNull(); + expect( + parseDestinationConnectionTestReport("Read: OK\nWrite: OK"), + ).toBeNull(); + }); +}); + +describe("rclone failure reasons", () => { + it("summarizes S3 status codes and error codes per operation", () => { + expect( + describeRcloneFailure( + { + stderr: + "Failed to rcat: InvalidAccessKeyId: The AWS Access Key Id you provided does not exist in our records.\n\tstatus code: 403, request id: tx01", + }, + "write", + [], + ), + ).toBe("403 InvalidAccessKeyId (the access key ID is not recognized)"); + expect( + describeRcloneFailure( + { + stderr: + "operation error S3: ListObjectsV2, https response error StatusCode: 404, api error NoSuchBucket: The specified bucket does not exist", + }, + "read", + [], + ), + ).toBe("404 NoSuchBucket (the bucket does not exist)"); + expect( + describeRcloneFailure( + { stderr: "https response error StatusCode: 403" }, + "delete", + [], + ), + ).toBe("403 (the key cannot delete objects)"); + }); + + it("never echoes the command line of a failed local exec", () => { + expect( + describeRcloneFailure( + new Error( + "Command execution failed: Command failed: RCLONE_CONFIG_DOKPLOYS3_SECRET_ACCESS_KEY=abc rclone ls dokploys3:bucket", + ), + "read", + [], + ), + ).toBe("rclone could not complete the request"); + }); + + it("keeps connection errors that are not tied to the command", () => { + expect( + describeRcloneFailure( + new Error("SSH connection error: connect ECONNREFUSED"), + "read", + [], + ), + ).toBe("SSH connection error: connect ECONNREFUSED"); + expect( + describeRcloneFailure( + { + stderr: + "2026/10/06 03:00:01 ERROR : : error reading source root directory: dial tcp: lookup s3.example.com: no such host\n2026/10/06 03:00:01 Failed to ls: dial tcp: lookup s3.example.com: no such host", + }, + "read", + [], + ), + ).toBe("Failed to ls: dial tcp: lookup s3.example.com: no such host"); + }); +}); diff --git a/apps/dokploy/__test__/destination/destination-connection-test.test.ts b/apps/dokploy/__test__/destination/destination-connection-test.test.ts new file mode 100644 index 00000000000..d353fded2e4 --- /dev/null +++ b/apps/dokploy/__test__/destination/destination-connection-test.test.ts @@ -0,0 +1,318 @@ +import { execFileSync } from "node:child_process"; +import { + existsSync, + mkdtempSync, + readFileSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { REDACTED_SECRET_VALUE } from "@dokploy/server/utils/security/redaction"; +import { parse } from "shell-quote"; +import { beforeEach, describe, expect, it, vi } from "vitest"; + +const mocks = vi.hoisted(() => ({ + assertDestinationEndpointAllowed: vi.fn(), + checkPermission: vi.fn(), + execAsync: vi.fn(), + execAsyncRemote: vi.fn(), + findDestinationById: vi.fn(), + getAccessibleServerIds: vi.fn(), + isCloud: false, +})); + +vi.mock("@dokploy/server", () => ({ + get IS_CLOUD() { + return mocks.isCloud; + }, + createDestination: vi.fn(), + execAsync: mocks.execAsync, + execAsyncRemote: mocks.execAsyncRemote, + findDestinationById: mocks.findDestinationById, + getAccessibleServerIds: mocks.getAccessibleServerIds, + removeDestinationById: vi.fn(), + updateDestinationById: vi.fn(), +})); + +vi.mock("@dokploy/server/db", () => ({ + db: { query: { destinations: { findMany: vi.fn() } } }, +})); + +vi.mock("@dokploy/server/services/permission", () => ({ + checkPermission: mocks.checkPermission, +})); + +vi.mock("@dokploy/server/utils/destination/endpoint", () => ({ + assertDestinationEndpointAllowed: mocks.assertDestinationEndpointAllowed, + normalizeDestinationEndpointUrl: (endpoint: string) => endpoint, +})); + +vi.mock("@/server/api/utils/audit", () => ({ + audit: vi.fn(), +})); + +const { destinationRouter } = await import( + "../../server/api/routers/destination" +); + +const SECRET = "super-secret-access-key-value"; +const ACCESS_KEY = "AKIAEXAMPLEACCESSKEY"; +const TEST_OBJECT_PATTERN = + /^dokploy-connection-test\/\d{4}-\d{2}-\d{2}T\d{2}-\d{2}-\d{2}-\d{3}Z-[0-9a-f]{8}\.txt$/; + +const destinationInput = { + name: "Test", + provider: "AWS", + accessKey: ACCESS_KEY, + secretAccessKey: SECRET, + bucket: "bucket", + region: "us-east-1", + endpoint: "https://s3.example.com", + additionalFlags: [], +}; + +const createCaller = () => + destinationRouter.createCaller({ + db: {}, + req: {}, + res: {}, + session: { + userId: "user-1", + activeOrganizationId: "org-1", + }, + user: { + id: "user-1", + role: "member", + }, + } as never); + +// Mirrors what execAsync throws for a failed local rclone call: the message +// repeats the command line and stderr carries rclone's own log output. +const rcloneFailure = (operation: string, stderrLines: string[]) => + Object.assign( + new Error( + `Command execution failed: Command failed: RCLONE_CONFIG_DOKPLOYS3_ACCESS_KEY_ID=${ACCESS_KEY} RCLONE_CONFIG_DOKPLOYS3_SECRET_ACCESS_KEY=${SECRET} rclone ${operation}`, + ), + { stderr: stderrLines.join("\n") }, + ); + +const accessDenied = (s3Operation: string) => + rcloneFailure("rcat", [ + `2026/10/06 03:00:01 ERROR : dokploy-connection-test/object.txt: Failed to copy: operation error S3: ${s3Operation}, https response error StatusCode: 403, RequestID: tx01, HostID: host01, api error AccessDenied: Access Denied.`, + `2026/10/06 03:00:01 Failed to ${s3Operation}: operation error S3: ${s3Operation}, https response error StatusCode: 403, RequestID: tx01, HostID: host01, api error AccessDenied: Access Denied.`, + ]); + +const executedCommands = () => + mocks.execAsync.mock.calls.map(([command]) => command as string); + +const lastArgument = (command = "") => { + const last = parse(command).at(-1); + return typeof last === "string" ? last : undefined; +}; + +const getTestObject = (command?: string) => + lastArgument(command)?.replace(/^dokploys3:bucket\//, ""); + +const getBadRequestMessage = async (promise: Promise) => { + const error = await promise.then( + () => { + throw new Error("Expected the destination test to fail"); + }, + (rejection: unknown) => rejection, + ); + expect(error).toMatchObject({ code: "BAD_REQUEST" }); + return (error as Error).message; +}; + +describe("destination connection test checks read, write and delete access", () => { + beforeEach(() => { + vi.clearAllMocks(); + mocks.isCloud = false; + mocks.checkPermission.mockResolvedValue(undefined); + mocks.execAsync.mockResolvedValue({ stdout: "", stderr: "" }); + mocks.execAsyncRemote.mockResolvedValue({ stdout: "", stderr: "" }); + mocks.assertDestinationEndpointAllowed.mockImplementation( + async (endpoint: string) => endpoint, + ); + mocks.getAccessibleServerIds.mockResolvedValue(new Set(["server-1"])); + }); + + it("lists, uploads and deletes one test object when every check passes", async () => { + await expect( + createCaller().testConnection(destinationInput), + ).resolves.toEqual({ + read: { ok: true }, + write: { ok: true }, + delete: { ok: true }, + }); + + const [read, write, remove] = executedCommands(); + expect(executedCommands()).toHaveLength(3); + expect(read).toContain("rclone ls "); + expect(lastArgument(read)).toBe("dokploys3:bucket"); + expect(write).toContain("rclone rcat "); + expect(remove).toContain("rclone deletefile "); + + const writtenObject = getTestObject(write); + expect(writtenObject).toMatch(TEST_OBJECT_PATTERN); + expect(getTestObject(remove)).toBe(writtenObject); + for (const command of [read, write, remove]) { + expect(command).toContain("--timeout 10s"); + expect(command).toContain("--contimeout 5s"); + } + }); + + it("runs every check on the selected server in cloud mode", async () => { + mocks.isCloud = true; + + await expect( + createCaller().testConnection({ + ...destinationInput, + serverId: "server-1", + }), + ).resolves.toEqual({ + read: { ok: true }, + write: { ok: true }, + delete: { ok: true }, + }); + + expect(mocks.execAsync).not.toHaveBeenCalled(); + expect( + mocks.execAsyncRemote.mock.calls.map(([serverId, command]) => [ + serverId, + (command as string).match(/rclone (\w+)/)?.[1], + ]), + ).toEqual([ + ["server-1", "ls"], + ["server-1", "rcat"], + ["server-1", "deletefile"], + ]); + }); + + it("reports a read-only key as Read OK, Write FAILED and Delete SKIPPED without leaking keys", async () => { + mocks.execAsync + .mockResolvedValueOnce({ stdout: "", stderr: "" }) + .mockRejectedValueOnce(accessDenied("PutObject")); + + const message = await getBadRequestMessage( + createCaller().testConnection(destinationInput), + ); + + expect(message.split("\n")).toEqual([ + "Read: OK", + "Write: FAILED - 403 AccessDenied (the key cannot upload objects)", + "Delete: SKIPPED - no test object was uploaded", + ]); + expect(message).not.toContain(SECRET); + expect(message).not.toContain(ACCESS_KEY); + expect(executedCommands()).toHaveLength(2); + }); + + it("keeps testing writes when listing is forbidden", async () => { + mocks.execAsync.mockRejectedValueOnce( + rcloneFailure("ls", [ + "2026/10/06 03:00:01 Failed to ls: AccessDenied: Access Denied", + "\tstatus code: 403, request id: tx01, host id: host01", + ]), + ); + + const message = await getBadRequestMessage( + createCaller().testConnection(destinationInput), + ); + + expect(message.split("\n")).toEqual([ + "Read: FAILED - 403 AccessDenied (the key cannot list objects in the bucket)", + "Write: OK", + "Delete: OK", + ]); + expect(executedCommands()).toHaveLength(3); + }); + + it("names the leftover test object when the upload works but the delete is denied", async () => { + mocks.execAsync + .mockResolvedValueOnce({ stdout: "", stderr: "" }) + .mockResolvedValueOnce({ stdout: "", stderr: "" }) + .mockRejectedValueOnce(accessDenied("DeleteObject")); + + const message = await getBadRequestMessage( + createCaller().testConnection(destinationInput), + ); + const writtenObject = getTestObject(executedCommands()[1]); + + expect(writtenObject).toMatch(TEST_OBJECT_PATTERN); + expect(message.split("\n")).toEqual([ + "Read: OK", + "Write: OK", + `Delete: FAILED - 403 AccessDenied (the key cannot delete objects); remove the leftover test object bucket/${writtenObject} manually`, + ]); + expect(message).not.toContain(SECRET); + }); + + it("masks keys that appear in unrecognized rclone errors", async () => { + mocks.execAsync + .mockResolvedValueOnce({ stdout: "", stderr: "" }) + .mockRejectedValueOnce( + rcloneFailure("rcat", [ + `2026/10/06 03:00:01 ERROR : upload rejected for ${ACCESS_KEY} with secret ${SECRET}`, + ]), + ); + + const message = await getBadRequestMessage( + createCaller().testConnection(destinationInput), + ); + + expect(message).toContain("Write: FAILED - upload rejected for"); + expect(message).toContain(REDACTED_SECRET_VALUE); + expect(message).not.toContain(SECRET); + expect(message).not.toContain(ACCESS_KEY); + expect(message).not.toContain("RCLONE_CONFIG"); + }); + + it("keeps the test object commands shell-safe for hostile bucket names", async () => { + const bucket = + "b'; touch pwned-a; echo `touch pwned-b` $(touch pwned-c) \"x"; + await createCaller().testConnection({ ...destinationInput, bucket }); + const [, write, remove] = executedCommands(); + + const workdir = mkdtempSync(path.join(tmpdir(), "dokploy-rclone-")); + try { + const argsFile = path.join(workdir, "args"); + const stdinFile = path.join(workdir, "stdin"); + writeFileSync( + path.join(workdir, "rclone"), + `#!/bin/sh\nprintf '%s\\n' "$@" > "${argsFile}"\ncat > "${stdinFile}"\n`, + { mode: 0o755 }, + ); + const runInShell = (command: string) => { + execFileSync("/bin/sh", ["-c", command], { + cwd: workdir, + env: { NODE_ENV: "test", PATH: `${workdir}:/usr/bin:/bin` }, + input: "", + }); + return readFileSync(argsFile, "utf8").trimEnd().split("\n"); + }; + + const writeArgs = runInShell(write ?? ""); + expect(writeArgs[0]).toBe("rcat"); + const target = writeArgs.at(-1) ?? ""; + expect(target.startsWith(`dokploys3:${bucket}/`)).toBe(true); + expect(target.slice(`dokploys3:${bucket}/`.length)).toMatch( + TEST_OBJECT_PATTERN, + ); + expect(readFileSync(stdinFile, "utf8")).toContain( + "Dokploy S3 destination connection test", + ); + + const deleteArgs = runInShell(remove ?? ""); + expect(deleteArgs[0]).toBe("deletefile"); + expect(deleteArgs.at(-1)).toBe(target); + + for (const marker of ["pwned-a", "pwned-b", "pwned-c"]) { + expect(existsSync(path.join(workdir, marker))).toBe(false); + } + } finally { + rmSync(workdir, { recursive: true, force: true }); + } + }); +}); diff --git a/apps/dokploy/__test__/destination/destination-router-assigned-server.test.ts b/apps/dokploy/__test__/destination/destination-router-assigned-server.test.ts index b5bd4a0f41e..156b841a404 100644 --- a/apps/dokploy/__test__/destination/destination-router-assigned-server.test.ts +++ b/apps/dokploy/__test__/destination/destination-router-assigned-server.test.ts @@ -68,6 +68,12 @@ const destinationInput = { serverId: "server-1", }; +const allChecksPassed = { + read: { ok: true }, + write: { ok: true }, + delete: { ok: true }, +}; + const createCaller = () => destinationRouter.createCaller({ db: {}, @@ -200,7 +206,7 @@ describe("destination router assigned-server boundary", () => { it("allows cloud connection tests on accessible servers", async () => { await expect( createCaller().testConnection(destinationInput), - ).resolves.toBeUndefined(); + ).resolves.toEqual(allChecksPassed); expect(mocks.execAsyncRemote).toHaveBeenCalledWith( "server-1", @@ -218,7 +224,7 @@ describe("destination router assigned-server boundary", () => { secretAccessKey: REDACTED_SECRET_VALUE, destinationId: "destination-1", }), - ).resolves.toBeUndefined(); + ).resolves.toEqual(allChecksPassed); expect(mocks.findDestinationById).toHaveBeenCalledWith("destination-1"); const command = mocks.execAsync.mock.calls[0]?.[0] as string; @@ -238,7 +244,7 @@ describe("destination router assigned-server boundary", () => { secretAccessKey: "new-secret", destinationId: "destination-1", }), - ).resolves.toBeUndefined(); + ).resolves.toEqual(allChecksPassed); expect(mocks.findDestinationById).not.toHaveBeenCalled(); expect(mocks.execAsync).toHaveBeenCalledWith( @@ -308,7 +314,7 @@ describe("destination router assigned-server boundary", () => { secretAccessKey: REDACTED_SECRET_VALUE, destinationId: "destination-1", }), - ).resolves.toBeUndefined(); + ).resolves.toEqual(allChecksPassed); expect(mocks.execAsyncRemote).toHaveBeenCalledWith( "server-1", diff --git a/apps/dokploy/components/dashboard/settings/destination/handle-destinations.tsx b/apps/dokploy/components/dashboard/settings/destination/handle-destinations.tsx index 3823b34e44d..bca36b4d043 100644 --- a/apps/dokploy/components/dashboard/settings/destination/handle-destinations.tsx +++ b/apps/dokploy/components/dashboard/settings/destination/handle-destinations.tsx @@ -2,13 +2,28 @@ import { ADDITIONAL_FLAG_ERROR, ADDITIONAL_FLAG_REGEX, } from "@dokploy/server/db/validations/destination"; +import { + DESTINATION_ACCESS_CHECK_LABELS, + DESTINATION_ACCESS_CHECKS, + type DestinationConnectionTestReport, + isDestinationConnectionTestPassed, + parseDestinationConnectionTestReport, +} from "@dokploy/server/utils/destination/connection-test-report"; import { standardSchemaResolver as zodResolver } from "@hookform/resolvers/standard-schema"; -import { PenBoxIcon, PlusIcon, Trash2 } from "lucide-react"; +import { + CheckCircle2, + MinusCircle, + PenBoxIcon, + PlusIcon, + Trash2, + XCircle, +} from "lucide-react"; import { useEffect, useState } from "react"; import { useFieldArray, useForm } from "react-hook-form"; import { toast } from "sonner"; import { z } from "zod"; import { AlertBlock } from "@/components/shared/alert-block"; +import { Alert, AlertDescription, AlertTitle } from "@/components/ui/alert"; import { Button } from "@/components/ui/button"; import { Dialog, @@ -68,6 +83,58 @@ interface Props { destinationId?: string; } +const CHECK_STATUS_DISPLAY = { + ok: { icon: CheckCircle2, label: "OK", className: "text-green-500" }, + failed: { icon: XCircle, label: "Failed", className: "text-red-500" }, + skipped: { + icon: MinusCircle, + label: "Skipped", + className: "text-muted-foreground", + }, +}; + +const ConnectionTestResults = ({ + report, +}: { + report: DestinationConnectionTestReport; +}) => { + const passed = isDestinationConnectionTestPassed(report); + return ( + + + {passed + ? "Read, write and delete access confirmed" + : "Some S3 permissions are missing"} + + +
    + {DESTINATION_ACCESS_CHECKS.map((name) => { + const check = report[name]; + const display = + CHECK_STATUS_DISPLAY[ + check.ok ? "ok" : check.skipped ? "skipped" : "failed" + ]; + return ( +
  • + + + + {DESTINATION_ACCESS_CHECK_LABELS[name]}: {display.label} + + {check.reason ? ` - ${check.reason}` : null} + +
    • + ); + })} +
+ + + ); +}; + export const HandleDestinations = ({ destinationId }: Props) => { const [open, setOpen] = useState(false); const utils = api.useUtils(); @@ -91,8 +158,16 @@ export const HandleDestinations = ({ destinationId }: Props) => { mutateAsync: testConnection, isPending: isPendingConnection, error: connectionError, - isError: isErrorConnection, + data: connectionResult, } = api.destination.testConnection.useMutation(); + const connectionReport = + connectionResult ?? + (connectionError + ? parseDestinationConnectionTestReport(connectionError.message) + : null); + const connectionErrorMessage = connectionReport + ? undefined + : connectionError?.message; const form = useForm({ defaultValues: { @@ -212,9 +287,17 @@ export const HandleDestinations = ({ destinationId }: Props) => { form.getValues("additionalFlags")?.map((f) => f.value) ?? [], }) .then(() => { - toast.success("Connection Success"); + toast.success("Connection Success", { + description: "Read, write and delete access confirmed", + }); }) .catch((e) => { + if (parseDestinationConnectionTestReport(e.message)) { + toast.error("Destination test failed", { + description: "See the check results in the dialog.", + }); + return; + } toast.error("Error connecting to provider", { description: `${e.message}\n\nTry manually: rclone ls ${connectionString}`, }); @@ -250,9 +333,9 @@ export const HandleDestinations = ({ destinationId }: Props) => { guarantee secure and efficient storage. - {(isError || isErrorConnection) && ( + {(isError || connectionErrorMessage) && ( - {connectionError?.message || error?.message} + {connectionErrorMessage || error?.message} )} @@ -433,6 +516,10 @@ export const HandleDestinations = ({ destinationId }: Props) => { + {connectionReport && ( + + )} + + IS_CLOUD + ? execAsyncRemote(destinationInput.serverId || "", command) + : execAsync(command), + ); } catch (error) { throw new TRPCError({ code: "BAD_REQUEST", message: error instanceof Error - ? error?.message + ? redactSensitiveText(error.message) : "Error connecting to bucket", cause: error, }); } + + if (!isDestinationConnectionTestPassed(report)) { + throw new TRPCError({ + code: "BAD_REQUEST", + message: formatDestinationConnectionTestReport(report), + }); + } + return report; }), one: withPermission("destination", "read") .input(apiFindOneDestination) diff --git a/packages/server/src/utils/destination/connection-test-report.ts b/packages/server/src/utils/destination/connection-test-report.ts new file mode 100644 index 00000000000..057d90cfc89 --- /dev/null +++ b/packages/server/src/utils/destination/connection-test-report.ts @@ -0,0 +1,77 @@ +export const DESTINATION_ACCESS_CHECKS = ["read", "write", "delete"] as const; + +export type DestinationAccessCheckName = + (typeof DESTINATION_ACCESS_CHECKS)[number]; + +export type DestinationAccessCheckResult = { + ok: boolean; + skipped?: boolean; + reason?: string; +}; + +export type DestinationConnectionTestReport = Record< + DestinationAccessCheckName, + DestinationAccessCheckResult +>; + +export const DESTINATION_ACCESS_CHECK_LABELS: Record< + DestinationAccessCheckName, + string +> = { + read: "Read", + write: "Write", + delete: "Delete", +}; + +const REPORT_LINE_PATTERN = + /^(Read|Write|Delete): (OK|FAILED|SKIPPED)(?: - (.+))?$/; + +export const isDestinationConnectionTestPassed = ( + report: DestinationConnectionTestReport, +) => DESTINATION_ACCESS_CHECKS.every((name) => report[name].ok); + +// The UI parses this text back with parseDestinationConnectionTestReport, so +// a failed test can stay a plain tRPC error for API and MCP clients. +export const formatDestinationConnectionTestReport = ( + report: DestinationConnectionTestReport, +) => + DESTINATION_ACCESS_CHECKS.map((name) => { + const check = report[name]; + const label = DESTINATION_ACCESS_CHECK_LABELS[name]; + if (check.ok) { + return `${label}: OK`; + } + const status = check.skipped ? "SKIPPED" : "FAILED"; + return check.reason + ? `${label}: ${status} - ${check.reason}` + : `${label}: ${status}`; + }).join("\n"); + +export const parseDestinationConnectionTestReport = ( + message: string, +): DestinationConnectionTestReport | null => { + const report: Partial = {}; + for (const line of message.split("\n")) { + const match = REPORT_LINE_PATTERN.exec(line.trim()); + if (!match) { + continue; + } + const [, label, status, reason] = match; + const name = DESTINATION_ACCESS_CHECKS.find( + (check) => DESTINATION_ACCESS_CHECK_LABELS[check] === label, + ); + if (!name) { + continue; + } + report[name] = { + ok: status === "OK", + ...(status === "SKIPPED" ? { skipped: true } : {}), + ...(reason ? { reason } : {}), + }; + } + + const { read, write, delete: deleteCheck } = report; + return read && write && deleteCheck + ? { read, write, delete: deleteCheck } + : null; +}; diff --git a/packages/server/src/utils/destination/connection-test.ts b/packages/server/src/utils/destination/connection-test.ts new file mode 100644 index 00000000000..f3bf36e2932 --- /dev/null +++ b/packages/server/src/utils/destination/connection-test.ts @@ -0,0 +1,202 @@ +import { randomBytes } from "node:crypto"; +import { + buildRcloneS3Command, + getBackupTimestamp, + getRcloneS3Destination, + type RcloneS3Destination, +} from "@dokploy/server/utils/backups/utils"; +import { + REDACTED_SECRET_VALUE, + redactSensitiveText, +} from "@dokploy/server/utils/security/redaction"; +import { quoteShellArgs } from "@dokploy/server/utils/shell"; +import type { + DestinationAccessCheckName, + DestinationAccessCheckResult, + DestinationConnectionTestReport, +} from "./connection-test-report"; + +export const DESTINATION_TEST_OBJECT_PREFIX = "dokploy-connection-test"; + +const TEST_OBJECT_CONTENT = + "Dokploy S3 destination connection test object. It is safe to delete."; + +const RCLONE_TEST_FLAGS = [ + "--retries", + "1", + "--low-level-retries", + "1", + "--timeout", + "10s", + "--contimeout", + "5s", +]; + +const MAX_REASON_LENGTH = 300; +const MIN_MASKED_SECRET_LENGTH = 4; + +const ACCESS_DENIED_HINTS: Record = { + read: "the key cannot list objects in the bucket", + write: "the key cannot upload objects", + delete: "the key cannot delete objects", +}; + +const S3_ERROR_HINTS: Record = { + InvalidAccessKeyId: "the access key ID is not recognized", + SignatureDoesNotMatch: + "the secret access key does not match the access key ID", + NoSuchBucket: "the bucket does not exist", +}; + +const ACCESS_DENIED_CODES = new Set([ + "AccessDenied", + "AllAccessDisabled", + "Forbidden", +]); + +const KNOWN_S3_ERROR_CODE_PATTERN = + /\b(AccessDenied|AllAccessDisabled|Forbidden|InvalidAccessKeyId|SignatureDoesNotMatch|NoSuchBucket)\b/; + +const RCLONE_LOG_PREFIX_PATTERN = + /^\d{4}\/\d{2}\/\d{2} \d{2}:\d{2}:\d{2}\s+(?:[A-Z]+\s*:\s*)?/; + +export type DestinationCommandRunner = (command: string) => Promise; + +export const createDestinationTestObjectPath = () => + `${DESTINATION_TEST_OBJECT_PREFIX}/${getBackupTimestamp()}-${randomBytes(4).toString("hex")}.txt`; + +export const buildDestinationTestCommands = ( + destination: RcloneS3Destination, + objectPath: string, +): Record => { + const testObject = getRcloneS3Destination(destination, objectPath); + return { + read: buildRcloneS3Command("ls", destination, [ + ...RCLONE_TEST_FLAGS, + getRcloneS3Destination(destination), + ]), + write: `${quoteShellArgs(["printf", "%s", TEST_OBJECT_CONTENT])} | ${buildRcloneS3Command( + "rcat", + destination, + [...RCLONE_TEST_FLAGS, testObject], + )}`, + delete: buildRcloneS3Command("deletefile", destination, [ + ...RCLONE_TEST_FLAGS, + testObject, + ]), + }; +}; + +const toSingleLine = (text: string) => text.replace(/\s+/g, " ").trim(); + +const truncate = (text: string) => + text.length > MAX_REASON_LENGTH + ? `${text.slice(0, MAX_REASON_LENGTH - 3)}...` + : text; + +const maskSecrets = (text: string, secrets: readonly string[]) => { + let masked = text; + for (const secret of secrets) { + if (secret.length >= MIN_MASKED_SECRET_LENGTH) { + masked = masked.split(secret).join(REDACTED_SECRET_VALUE); + } + } + return redactSensitiveText(masked); +}; + +const getFailureOutput = (error: unknown) => { + if (typeof error === "string") { + return error; + } + if (!error || typeof error !== "object") { + return ""; + } + const { stderr, message } = error as { stderr?: unknown; message?: unknown }; + if (typeof stderr === "string" && stderr.trim()) { + return stderr; + } + // Local exec errors repeat the whole command line, so only messages from + // other failures (SSH, missing server key) are worth showing. + if (typeof message === "string" && !message.includes("Command failed:")) { + return message; + } + return ""; +}; + +export const describeRcloneFailure = ( + error: unknown, + check: DestinationAccessCheckName, + secrets: readonly string[], +) => { + const output = maskSecrets(getFailureOutput(error), secrets); + const status = /status ?code:?\s*(\d{3})/i.exec(output)?.[1]; + const code = + /api error ([A-Za-z]+)/.exec(output)?.[1] ?? + KNOWN_S3_ERROR_CODE_PATTERN.exec(output)?.[1]; + + if (status || code) { + const summary = [status, code].filter(Boolean).join(" "); + const hint = + (code && S3_ERROR_HINTS[code]) || + (status === "403" || (code && ACCESS_DENIED_CODES.has(code)) + ? ACCESS_DENIED_HINTS[check] + : undefined); + return hint ? `${summary} (${hint})` : summary; + } + + const lastLine = output + .split("\n") + .map((line) => line.replace(RCLONE_LOG_PREFIX_PATTERN, "").trim()) + .filter(Boolean) + .at(-1); + return lastLine + ? truncate(toSingleLine(lastLine)) + : "rclone could not complete the request"; +}; + +export const runDestinationConnectionTest = async ( + destination: RcloneS3Destination, + run: DestinationCommandRunner, +): Promise => { + const objectPath = createDestinationTestObjectPath(); + const commands = buildDestinationTestCommands(destination, objectPath); + const secrets = [destination.secretAccessKey, destination.accessKey]; + + const runCheck = async ( + check: DestinationAccessCheckName, + ): Promise => { + try { + await run(commands[check]); + return { ok: true }; + } catch (error) { + return { + ok: false, + reason: describeRcloneFailure(error, check, secrets), + }; + } + }; + + const read = await runCheck("read"); + const write = await runCheck("write"); + if (!write.ok) { + return { + read, + write, + delete: { + ok: false, + skipped: true, + reason: "no test object was uploaded", + }, + }; + } + + const deleteCheck = await runCheck("delete"); + if (!deleteCheck.ok) { + const leftover = toSingleLine(`${destination.bucket}/${objectPath}`); + deleteCheck.reason = maskSecrets( + `${deleteCheck.reason}; remove the leftover test object ${leftover} manually`, + secrets, + ); + } + return { read, write, delete: deleteCheck }; +};