diff --git a/apps/dokploy/__test__/api/openapi-document.test.ts b/apps/dokploy/__test__/api/openapi-document.test.ts index 22bfbf3f5b5..473ea487676 100644 --- a/apps/dokploy/__test__/api/openapi-document.test.ts +++ b/apps/dokploy/__test__/api/openapi-document.test.ts @@ -24,19 +24,23 @@ describe("OpenAPI document", () => { ); }); - it("serves blind env management at GET /env.keys and POST /env.apply", () => { + it("serves blind env management at GET /env.keys, GET /env.layout and POST /env.apply", () => { const paths = generate().paths ?? {}; const keys = paths["/env.keys"]?.get as | { operationId?: string; parameters?: { name: string; in: string }[] } | undefined; + const layout = paths["/env.layout"]?.get as + | { operationId?: string; parameters?: { name: string; in: string }[] } + | undefined; const apply = paths["/env.apply"]?.post as | { operationId?: string; requestBody?: unknown } | undefined; expect( Object.keys(paths).filter((path) => path.startsWith("/env.")), - ).toEqual(["/env.keys", "/env.apply"]); + ).toEqual(["/env.keys", "/env.layout", "/env.apply"]); expect(Object.keys(paths["/env.keys"] ?? {})).toEqual(["get"]); + expect(Object.keys(paths["/env.layout"] ?? {})).toEqual(["get"]); expect(Object.keys(paths["/env.apply"] ?? {})).toEqual(["post"]); expect(keys?.operationId).toBe("env-keys"); expect(keys?.parameters).toEqual([ @@ -44,6 +48,12 @@ describe("OpenAPI document", () => { expect.objectContaining({ name: "id", in: "query", required: true }), expect.objectContaining({ name: "field", in: "query" }), ]); + expect(layout?.operationId).toBe("env-layout"); + expect(layout?.parameters?.map(({ name }) => name)).toEqual([ + "scope", + "id", + "field", + ]); expect(apply?.operationId).toBe("env-apply"); expect( Object.keys( @@ -63,6 +73,7 @@ describe("OpenAPI document", () => { "remove", "generate", "copy", + "layout", "dryRun", "expectedRevision", ]); @@ -95,7 +106,23 @@ describe("OpenAPI document", () => { } ).responses[200].content["application/json"].schema; const keys = responseSchema(paths["/env.keys"]?.get); + const layout = responseSchema(paths["/env.layout"]?.get); const apply = responseSchema(paths["/env.apply"]?.post); + const layoutItems = (schema: unknown) => + ( + schema as { + properties: { + layout: { + items: { + oneOf: { + properties: object; + additionalProperties: boolean; + }[]; + }; + }; + }; + } + ).properties.layout.items.oneOf; expect(Object.keys(keys.properties)).toEqual([ "scope", @@ -117,14 +144,34 @@ describe("OpenAPI document", () => { "revision", "dryRun", "variables", + "layout", ]); + expect(Object.keys(layout.properties)).toEqual([ + "scope", + "id", + "field", + "revision", + "variables", + "layout", + ]); + for (const items of [layoutItems(layout), layoutItems(apply)]) { + expect(items.map(({ properties }) => Object.keys(properties))).toEqual([ + ["type", "name"], + ["type", "ref", "text"], + ["type", "ref", "kind", "variable"], + ["type"], + ]); + for (const item of items) { + expect(item.additionalProperties).toBe(false); + } + } expect(Object.keys(apply.properties.variables.items.properties)).toEqual([ "name", "action", "source", "secret", ]); - for (const schema of [keys, apply]) { + for (const schema of [keys, layout, apply]) { expect(schema.additionalProperties).toBe(false); expect(schema.properties.variables.items.additionalProperties).toBe( false, diff --git a/apps/dokploy/__test__/env/blind-env-router.test.ts b/apps/dokploy/__test__/env/blind-env-router.test.ts index bdadfe48029..5f869ec34b3 100644 --- a/apps/dokploy/__test__/env/blind-env-router.test.ts +++ b/apps/dokploy/__test__/env/blind-env-router.test.ts @@ -1179,6 +1179,293 @@ describe("env.applyChanges", () => { }); }); +describe("env layout", () => { + const appLayout = [ + { type: "comment", ref: "c1", text: "App settings" }, + { type: "hidden", ref: "h1", kind: "overridden", variable: "SERVICE_URL" }, + { type: "variable", name: "SERVICE_TOKEN" }, + { type: "variable", name: "EMPTY_FLAG" }, + { type: "variable", name: "SERVICE_URL" }, + ]; + const sectioned = [ + { type: "comment", text: "=== Web ===" }, + { type: "hidden", ref: "h1" }, + { type: "variable", name: "SERVICE_URL" }, + { type: "variable", name: "EMPTY_FLAG" }, + { type: "blank" }, + { type: "comment", text: "=== Auth ===" }, + { type: "comment", ref: "c1", text: "Signs sessions" }, + { type: "variable", name: "SERVICE_TOKEN" }, + ] as const; + const sectionedEnv = `# === Web ===\nSERVICE_URL=https://app.example.test\nSERVICE_URL=https://override.example.test\nEMPTY_FLAG=\n\n# === Auth ===\n# Signs sessions\nSERVICE_TOKEN=${SENTINEL}\n`; + + it("reads the layout with names, comments and a revision but never a value", async () => { + const result = await createCaller().layout({ + scope: "application", + id: "app-1", + }); + + expect(result).toEqual({ + scope: "application", + id: "app-1", + field: "env", + revision: getApplicationEnvRevision("app-1", storedEnv(applications)), + variables: [ + { name: "SERVICE_URL", secret: false, empty: false }, + { name: "SERVICE_TOKEN", secret: true, empty: false }, + { name: "EMPTY_FLAG", secret: false, empty: true }, + ], + layout: appLayout, + }); + expect(JSON.stringify(result)).not.toContain(SENTINEL); + expect(JSON.stringify(result)).not.toContain("example.test"); + expect( + mocks.checkServicePermissionAndAccess, + ).toHaveBeenCalledExactlyOnceWith(ctxMatcher, "app-1", { + envVars: ["read"], + }); + expect(fake.locks).toEqual([]); + }); + + it("checks the project variable permission before showing a layout", async () => { + mocks.checkPermission.mockRejectedValueOnce( + new TRPCError({ code: "UNAUTHORIZED", message: "denied" }), + ); + + await expectTrpcError( + createCaller().layout({ scope: "project", id: "project-1" }), + "UNAUTHORIZED", + ); + expect(mocks.checkPermission).toHaveBeenCalledExactlyOnceWith(ctxMatcher, { + projectEnvVars: ["read"], + }); + }); + + it("rearranges variables into sections under the row lock and audits once", async () => { + const before = storedEnv(applications); + const result = await createCaller().applyChanges({ + scope: "application", + id: "app-1", + layout: [...sectioned], + expectedRevision: getApplicationEnvRevision("app-1", before), + }); + + const after = storedEnv(applications); + expect(after).toBe(sectionedEnv); + expect(parse(after ?? "")).toEqual(parse(before ?? "")); + expect(result).toEqual({ + scope: "application", + id: "app-1", + field: "env", + changed: true, + revision: getApplicationEnvRevision("app-1", after), + dryRun: false, + variables: [], + layout: [ + { type: "comment", ref: "c1", text: "=== Web ===" }, + { + type: "hidden", + ref: "h1", + kind: "overridden", + variable: "SERVICE_URL", + }, + { type: "variable", name: "SERVICE_URL" }, + { type: "variable", name: "EMPTY_FLAG" }, + { type: "blank" }, + { type: "comment", ref: "c2", text: "=== Auth ===" }, + { type: "comment", ref: "c3", text: "Signs sessions" }, + { type: "variable", name: "SERVICE_TOKEN" }, + ], + }); + expect(JSON.stringify(result)).not.toContain(SENTINEL); + expect(fake.locks).toEqual([{ table: "application", strength: "update" }]); + expect(mocks.audit).toHaveBeenCalledExactlyOnceWith(ctxMatcher, { + action: "update", + resourceType: "application", + resourceId: "app-1", + resourceName: "app-one", + }); + expect( + mocks.checkServicePermissionAndAccess, + ).toHaveBeenCalledExactlyOnceWith(ctxMatcher, "app-1", { + envVars: ["write"], + }); + }); + + it("previews a layout on a dry run without writing or auditing", async () => { + const before = storedEnv(applications); + const result = await createCaller().applyChanges({ + scope: "application", + id: "app-1", + layout: [...sectioned], + dryRun: true, + expectedRevision: getApplicationEnvRevision("app-1", before), + }); + + expect(result.changed).toBe(true); + expect(result.dryRun).toBe(true); + expect(result.revision).toBe(getApplicationEnvRevision("app-1", before)); + expect(result.layout?.map(({ type }) => type)).toEqual([ + "comment", + "hidden", + "variable", + "variable", + "blank", + "comment", + "comment", + "variable", + ]); + expect(storedEnv(applications)).toBe(before); + expect(mocks.audit).not.toHaveBeenCalled(); + }); + + it("does not write or audit a layout that is sent back unchanged", async () => { + const before = storedEnv(applications); + const result = await createCaller().applyChanges({ + scope: "application", + id: "app-1", + layout: appLayout as never, + expectedRevision: getApplicationEnvRevision("app-1", before), + }); + + expect(result).toMatchObject({ changed: false, layout: appLayout }); + expect(storedEnv(applications)).toBe(before); + expect(mocks.audit).not.toHaveBeenCalled(); + }); + + it("rearranges build args of an application and variables of a project", async () => { + await createCaller().applyChanges({ + scope: "application", + id: "app-1", + field: "buildArgs", + layout: [ + { type: "comment", text: "Build" }, + { type: "variable", name: "BUILD_MODE" }, + ], + expectedRevision: getEnvFieldRevision( + "application", + "app-1", + "buildArgs", + "BUILD_MODE=release\n", + ), + }); + await createCaller().applyChanges({ + scope: "project", + id: "project-1", + layout: [ + { type: "variable", name: "SHARED_TOKEN" }, + { type: "variable", name: "SHARED_URL" }, + ], + expectedRevision: getEnvFieldRevision( + "project", + "project-1", + "env", + storedEnv(projects), + ), + }); + + expect(storedEnv(applications, "buildArgs")).toBe( + "# Build\nBUILD_MODE=release\n", + ); + expect(storedEnv(projects)).toBe( + `SHARED_TOKEN=${SENTINEL}\nSHARED_URL=https://shared.example.test\n`, + ); + }); + + it("rejects a layout combined with value changes or without a revision", async () => { + const before = storedEnv(applications); + + const combined = await expectTrpcError( + createCaller().applyChanges({ + scope: "application", + id: "app-1", + set: { NEW_NAME: "1" }, + layout: [...sectioned], + expectedRevision: getApplicationEnvRevision("app-1", before), + }), + "BAD_REQUEST", + ); + const unpinned = await expectTrpcError( + createCaller().applyChanges({ + scope: "application", + id: "app-1", + layout: [...sectioned], + }), + "BAD_REQUEST", + ); + const empty = await expectTrpcError( + createCaller().applyChanges({ + scope: "application", + id: "app-1", + layout: [{ type: "comment" }, ...sectioned], + expectedRevision: getApplicationEnvRevision("app-1", before), + }), + "BAD_REQUEST", + ); + + expect(combined.message).toContain("layout cannot be combined"); + expect(unpinned.message).toContain("expectedRevision"); + expect(empty.message).toContain("A comment needs a ref, a text or both"); + expect(storedEnv(applications)).toBe(before); + expect(mocks.checkServicePermissionAndAccess).not.toHaveBeenCalled(); + }); + + it("rejects a stale revision without writing", async () => { + const before = storedEnv(applications); + + await expectTrpcError( + createCaller().applyChanges({ + scope: "application", + id: "app-1", + layout: [...sectioned], + expectedRevision: "stale", + }), + "CONFLICT", + ); + expect(storedEnv(applications)).toBe(before); + }); + + it("rejects a layout that drops a variable, naming only names", async () => { + const before = storedEnv(applications); + + const error = await expectTrpcError( + createCaller().applyChanges({ + scope: "application", + id: "app-1", + layout: sectioned.filter( + (item) => item.type !== "variable" || item.name !== "SERVICE_TOKEN", + ), + expectedRevision: getApplicationEnvRevision("app-1", before), + }), + "BAD_REQUEST", + ); + + expect(error.message).toContain( + "The layout leaves out variables: SERVICE_TOKEN.", + ); + expect(error.message).not.toContain(SENTINEL); + expect(storedEnv(applications)).toBe(before); + expect(mocks.audit).not.toHaveBeenCalled(); + }); + + it("rejects comment text that carries a redacted placeholder", async () => { + const before = storedEnv(applications); + + const error = await expectTrpcError( + createCaller().applyChanges({ + scope: "application", + id: "app-1", + layout: [{ type: "comment", text: "old [REDACTED]" }, ...sectioned], + expectedRevision: getApplicationEnvRevision("app-1", before), + }), + "BAD_REQUEST", + ); + + expect(error.message).toContain("layout[0]"); + expect(storedEnv(applications)).toBe(before); + }); +}); + describe("REST routes", () => { it("serves GET /env.keys from query parameters", async () => { const response = await callRest("GET", "env.keys", { @@ -1268,6 +1555,32 @@ describe("REST routes", () => { expect(JSON.stringify(response)).not.toContain(values.NEW_TOKEN); }); + it("serves GET /env.layout and a layout through POST /env.apply", async () => { + const read = await callRest("GET", "env.layout", { + scope: "postgres", + id: "postgres-1", + }); + const layout = (read.body as { layout: unknown[]; revision: string }) + .layout; + const applied = await callRest("POST", "env.apply", { + scope: "postgres", + id: "postgres-1", + layout: [{ type: "comment", text: "Replication" }, ...layout.reverse()], + expectedRevision: (read.body as { revision: string }).revision, + }); + + expect(read.status).toBe(200); + expect(layout).toEqual([ + { type: "variable", name: "REPLICATION_PASSWORD" }, + { type: "variable", name: "POSTGRES_INITDB_ARGS" }, + ]); + expect(applied.status).toBe(200); + expect(storedEnv(postgres)).toBe( + `# Replication\nREPLICATION_PASSWORD=${OTHER_SENTINEL}\nPOSTGRES_INITDB_ARGS=--data-checksums\n`, + ); + expect(JSON.stringify([read, applied])).not.toContain(OTHER_SENTINEL); + }); + it("maps errors to HTTP statuses without echoing values", async () => { const conflict = await callRest("POST", "env.apply", { scope: "application", diff --git a/apps/dokploy/__test__/env/layout.test.ts b/apps/dokploy/__test__/env/layout.test.ts new file mode 100644 index 00000000000..d2e9d1a33ce --- /dev/null +++ b/apps/dokploy/__test__/env/layout.test.ts @@ -0,0 +1,313 @@ +import { + applyEnvLayout, + type EnvLayoutInputItem, + readEnvLayout, +} from "@dokploy/server/utils/env-layout"; +import { parse } from "dotenv"; +import { describe, expect, it } from "vitest"; + +const SENTINEL = "sentinel-value-0001"; +const KEY_LINE = + "MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7VJTUt9Us8cKj="; + +const ENV = `# Production settings +SERVICE_URL=https://app.example.test # public address +# OLD_TOKEN=${SENTINEL} +# Signs the session cookies +AUTH_SECRET=${SENTINEL} + +SERVICE_KEY=-----BEGIN PRIVATE KEY----- +${KEY_LINE} +-----END PRIVATE KEY----- +NAME=first +MULTI_LINE="first line +second line" +NAME=second +`; + +describe("readEnvLayout", () => { + it("lists variables, comments and blank lines in file order without values", () => { + const layout = readEnvLayout(ENV); + + expect(layout).toEqual([ + { type: "comment", ref: "c1", text: "Production settings" }, + { type: "variable", name: "SERVICE_URL" }, + { type: "hidden", ref: "h1", kind: "comment", variable: "OLD_TOKEN" }, + { type: "comment", ref: "c2", text: "Signs the session cookies" }, + { type: "variable", name: "AUTH_SECRET" }, + { type: "blank" }, + { type: "variable", name: "SERVICE_KEY" }, + { type: "hidden", ref: "h2", kind: "overridden", variable: "NAME" }, + { type: "variable", name: "MULTI_LINE" }, + { type: "variable", name: "NAME" }, + ]); + const text = JSON.stringify(layout); + expect(text).not.toContain(SENTINEL); + expect(text).not.toContain(KEY_LINE); + expect(text).not.toContain("public address"); + }); + + it("withholds comments that could hold a value", () => { + const hidden = [ + `# export API_TOKEN = ${SENTINEL}`, + `# DATABASE_PASSWORD: ${SENTINEL}`, + "# admin password: hunter2", + "# пароль=hunter2", + "# mirror at https://deploy:hunter2@git.example.test/repo", + "# -----BEGIN OPENSSH PRIVATE KEY-----", + "# Authorization: Bearer abc", + `# old one was ${SENTINEL}`, + "# 0123456789abcdef0123", + ]; + const shown = [ + "# === Database ===", + "# TODO: rotate before release", + "# Secret used to sign sessions", + "#no space after the hash", + "# Docs: https://docs.example.test/env", + ]; + + const layout = readEnvLayout([...hidden, ...shown, "LAST=1"].join("\n")); + + expect(layout.slice(0, hidden.length)).toEqual( + hidden.map((_, index) => + expect.objectContaining({ ref: `h${index + 1}` }), + ), + ); + expect(layout[0]).toEqual({ + type: "hidden", + ref: "h1", + kind: "comment", + variable: "API_TOKEN", + }); + expect(layout[1]).toEqual({ + type: "hidden", + ref: "h2", + kind: "comment", + variable: "DATABASE_PASSWORD", + }); + expect(layout.slice(hidden.length, -1)).toEqual([ + { type: "comment", ref: "c1", text: "=== Database ===" }, + { type: "comment", ref: "c2", text: "TODO: rotate before release" }, + { type: "comment", ref: "c3", text: "Secret used to sign sessions" }, + { type: "comment", ref: "c4", text: "no space after the hash" }, + { + type: "comment", + ref: "c5", + text: "Docs: https://docs.example.test/env", + }, + ]); + const text = JSON.stringify(layout); + for (const secret of [SENTINEL, "hunter2", "0123456789abcdef0123"]) { + expect(text).not.toContain(secret); + } + }); + + it("hides lines dotenv cannot read when no variable comes right before them", () => { + expect( + readEnvLayout(`${KEY_LINE} +-----END PRIVATE KEY----- + +FIRST=1 +# note +stray text`), + ).toEqual([ + { type: "hidden", ref: "h1", kind: "unreadable" }, + { type: "blank" }, + { type: "variable", name: "FIRST" }, + { type: "comment", ref: "c1", text: "note" }, + { type: "hidden", ref: "h2", kind: "unreadable" }, + ]); + }); + + it("treats a missing field as an empty layout", () => { + expect(readEnvLayout(null)).toEqual([]); + expect(readEnvLayout(undefined)).toEqual([]); + expect(readEnvLayout("")).toEqual([]); + }); + + it("refuses a text where an assignment shares a line with other text", () => { + expect(() => readEnvLayout("# note
NAME=1\n")).toThrow( + "cannot be shown or rearranged safely", + ); + }); +}); + +describe("applyEnvLayout", () => { + const sections: EnvLayoutInputItem[] = [ + { type: "comment", ref: "c1" }, + { type: "blank" }, + { type: "comment", text: "=== Web ===" }, + { type: "variable", name: "SERVICE_URL" }, + { type: "hidden", ref: "h2" }, + { type: "variable", name: "NAME" }, + { type: "blank" }, + { type: "comment", text: "=== Auth ===" }, + { type: "comment", ref: "c2", text: "Signs session cookies" }, + { type: "variable", name: "AUTH_SECRET" }, + { type: "hidden", ref: "h1" }, + { type: "variable", name: "SERVICE_KEY" }, + { type: "comment", text: "" }, + { type: "variable", name: "MULTI_LINE" }, + ]; + + it("arranges variables into sections and keeps every value byte for byte", () => { + const result = applyEnvLayout(ENV, sections); + + expect(result.changed).toBe(true); + expect(result.env).toBe(`# Production settings + +# === Web === +SERVICE_URL=https://app.example.test # public address +NAME=first +NAME=second + +# === Auth === +# Signs session cookies +AUTH_SECRET=${SENTINEL} +# OLD_TOKEN=${SENTINEL} +SERVICE_KEY=-----BEGIN PRIVATE KEY----- +${KEY_LINE} +-----END PRIVATE KEY----- +# +MULTI_LINE="first line +second line" +`); + expect(parse(result.env)).toEqual(parse(ENV)); + expect(result.layout).toEqual([ + { type: "comment", ref: "c1", text: "Production settings" }, + { type: "blank" }, + { type: "comment", ref: "c2", text: "=== Web ===" }, + { type: "variable", name: "SERVICE_URL" }, + { type: "hidden", ref: "h1", kind: "overridden", variable: "NAME" }, + { type: "variable", name: "NAME" }, + { type: "blank" }, + { type: "comment", ref: "c3", text: "=== Auth ===" }, + { type: "comment", ref: "c4", text: "Signs session cookies" }, + { type: "variable", name: "AUTH_SECRET" }, + { type: "hidden", ref: "h2", kind: "comment", variable: "OLD_TOKEN" }, + { type: "variable", name: "SERVICE_KEY" }, + { type: "comment", ref: "c5", text: "" }, + { type: "variable", name: "MULTI_LINE" }, + ]); + expect(JSON.stringify(result.layout)).not.toContain(SENTINEL); + }); + + it("keeps the text untouched when the layout is sent back unchanged", () => { + const env = "# Header\r\n#Tight comment\r\nFIRST=1\r\n\r\nSECOND=2\r\n"; + const layout = readEnvLayout(env); + + expect(applyEnvLayout(env, layout)).toEqual({ + env, + changed: false, + layout, + }); + }); + + it("rewrites a comment only when its text changes", () => { + const env = "# Indented note\nFIRST=1\n"; + + expect(readEnvLayout(env)[0]).toEqual({ + type: "comment", + ref: "c1", + text: " Indented note", + }); + expect( + applyEnvLayout(env, [ + { type: "comment", ref: "c1", text: " Indented note" }, + { type: "variable", name: "FIRST" }, + ]).changed, + ).toBe(false); + expect( + applyEnvLayout(env, [ + { type: "comment", ref: "c1", text: "New note" }, + { type: "variable", name: "FIRST" }, + ]).env, + ).toBe("# New note\nFIRST=1\n"); + }); + + it("drops a shown comment that the layout leaves out", () => { + expect( + applyEnvLayout("# old header\nFIRST=1", [ + { type: "variable", name: "FIRST" }, + ]).env, + ).toBe("FIRST=1"); + }); + + it("starts an empty text with comments only", () => { + expect( + applyEnvLayout("", [{ type: "comment", text: "=== Settings ===" }]), + ).toEqual({ + env: "# === Settings ===", + changed: true, + layout: [{ type: "comment", ref: "c1", text: "=== Settings ===" }], + }); + }); + + it("names every problem with names and refs only", () => { + const attempt = () => + applyEnvLayout(ENV, [ + { type: "variable", name: "SERVICE_URL" }, + { type: "variable", name: "SERVICE_URL" }, + { type: "variable", name: "UNKNOWN_NAME" }, + { type: "comment", ref: "c1" }, + { type: "comment", ref: "c1" }, + { type: "comment", ref: "c9" }, + { type: "hidden", ref: "h7" }, + ]); + + expect(attempt).toThrow( + "The layout names variables that do not exist: UNKNOWN_NAME. " + + "The layout places variables more than once: SERVICE_URL. " + + "The layout leaves out variables: AUTH_SECRET, SERVICE_KEY, MULTI_LINE, NAME. Every variable has to be placed; delete one with remove instead. " + + "The layout refers to lines that do not exist: c9, h7. " + + "The layout places lines more than once: c1. " + + "The layout leaves out hidden lines: h1, h2. Their text is not shown, so they can be moved but not dropped.", + ); + const message = (() => { + try { + attempt(); + } catch (error) { + return (error as Error).message; + } + })(); + expect(message).not.toContain(SENTINEL); + }); + + it("keeps an earlier assignment of a name above the one that counts", () => { + const env = "NAME=first\nOTHER=1\nNAME=second\n"; + + expect(readEnvLayout(env)).toEqual([ + { type: "hidden", ref: "h1", kind: "overridden", variable: "NAME" }, + { type: "variable", name: "OTHER" }, + { type: "variable", name: "NAME" }, + ]); + expect( + applyEnvLayout(env, [ + { type: "variable", name: "OTHER" }, + { type: "hidden", ref: "h1" }, + { type: "variable", name: "NAME" }, + ]).env, + ).toBe("OTHER=1\nNAME=first\nNAME=second\n"); + expect(() => + applyEnvLayout(env, [ + { type: "variable", name: "NAME" }, + { type: "hidden", ref: "h1" }, + { type: "variable", name: "OTHER" }, + ]), + ).toThrow( + "These hidden lines are earlier assignments of a variable and have to stay above it: h1 (NAME).", + ); + }); + + it("refuses a layout after which dotenv would read another value", () => { + const env = 'LATER=1 "\nOPEN="value\n'; + + expect(() => + applyEnvLayout(env, [ + { type: "variable", name: "OPEN" }, + { type: "variable", name: "LATER" }, + ]), + ).toThrow("Environment variables cannot be rearranged safely: LATER, OPEN"); + }); +}); diff --git a/apps/dokploy/server/api/routers/env.ts b/apps/dokploy/server/api/routers/env.ts index 687ffbb2ed8..f2366d3f181 100644 --- a/apps/dokploy/server/api/routers/env.ts +++ b/apps/dokploy/server/api/routers/env.ts @@ -4,11 +4,13 @@ import { apiEnvApplyResponse, apiEnvKeys, apiEnvKeysResponse, + apiEnvLayoutResponse, } from "@dokploy/server/db/schema"; import { applyEnvVariableChanges, findEnvVariablesOwner, listEnvVariableKeys, + listEnvVariableLayout, } from "@dokploy/server/services/env-variables"; import { checkEnvironmentAccess, @@ -88,6 +90,13 @@ export const envRouter = createTRPCRouter({ await assertEnvAccess(ctx, input.scope, input.id, "read"); return await listEnvVariableKeys(input); }), + layout: protectedProcedure + .input(apiEnvKeys) + .output(apiEnvLayoutResponse) + .query(async ({ input, ctx }) => { + await assertEnvAccess(ctx, input.scope, input.id, "read"); + return await listEnvVariableLayout(input); + }), applyChanges: protectedProcedure // tRPC reserves `apply` as a procedure name, so the REST route and its // operation id are pinned here to keep the HTTP API at POST /env.apply. diff --git a/apps/dokploy/server/api/utils/procedure-access.ts b/apps/dokploy/server/api/utils/procedure-access.ts index a6f31dcdd5c..2dfca49a448 100644 --- a/apps/dokploy/server/api/utils/procedure-access.ts +++ b/apps/dokploy/server/api/utils/procedure-access.ts @@ -278,6 +278,7 @@ export const PROCEDURE_ACCESS: Record = { "domainOverview.traefikLogs": "read", "env.applyChanges": "write", "env.keys": "read", + "env.layout": "read", "environment.byProjectId": "read", "environment.create": "write", "environment.duplicate": "write", diff --git a/packages/server/src/db/schema/env-variables.ts b/packages/server/src/db/schema/env-variables.ts index 41230ba25dd..6e6ead6bcfd 100644 --- a/packages/server/src/db/schema/env-variables.ts +++ b/packages/server/src/db/schema/env-variables.ts @@ -36,6 +36,39 @@ const envVariableName = z "Environment variable names must start with a letter or underscore and contain only letters, numbers, and underscores", ); +// dotenv also reads names with dots and dashes, so a layout has to be able to +// place them even though env-apply cannot create them. +const envLayoutName = z.string().regex(/^[\w.-]+$/); +const envLayoutCommentRef = z.string().regex(/^c[1-9][0-9]*$/); +const envLayoutHiddenRef = z.string().regex(/^h[1-9][0-9]*$/); +const envLayoutCommentText = z + .string() + .max(500) + .regex(/^[^\n\r\u2028\u2029]*$/, "A comment has to fit on one line"); + +export const envLayoutItem = z.discriminatedUnion("type", [ + z.object({ type: z.literal("variable"), name: z.string() }), + z.object({ type: z.literal("comment"), ref: z.string(), text: z.string() }), + z.object({ + type: z.literal("hidden"), + ref: z.string(), + kind: z.enum(["comment", "overridden", "unreadable"]), + variable: z.string().optional(), + }), + z.object({ type: z.literal("blank") }), +]); + +export const envLayoutInputItem = z.discriminatedUnion("type", [ + z.object({ type: z.literal("variable"), name: envLayoutName }), + z.object({ + type: z.literal("comment"), + ref: envLayoutCommentRef.optional(), + text: envLayoutCommentText.optional(), + }), + z.object({ type: z.literal("hidden"), ref: envLayoutHiddenRef }), + z.object({ type: z.literal("blank") }), +]); + export const isEnvFieldSupported = (scope: EnvScope, field: EnvField) => field === "env" || scope === "application"; @@ -72,6 +105,10 @@ export const apiEnvKeysResponse = z.object({ ), }); +export const apiEnvLayoutResponse = apiEnvKeysResponse.extend({ + layout: z.array(envLayoutItem), +}); + export const apiEnvApply = z .object({ scope: envScope, @@ -99,6 +136,7 @@ export const apiEnvApply = z }), ) .optional(), + layout: z.array(envLayoutInputItem).optional(), dryRun: z.boolean().optional(), expectedRevision: z.string().optional(), }) @@ -134,11 +172,42 @@ export const apiEnvApply = z } } - if (listsByName.size === 0) { + if (input.layout) { + // Refs and names of a layout describe one exact text, so it is applied + // alone and only to the revision it was read from. + if (listsByName.size > 0) { + ctx.addIssue({ + code: "custom", + path: ["layout"], + message: + "layout cannot be combined with set, remove, generate or copy", + }); + } + if (!input.expectedRevision) { + ctx.addIssue({ + code: "custom", + path: ["expectedRevision"], + message: "layout needs the expectedRevision it was read with", + }); + } + input.layout.forEach((item, index) => { + if ( + item.type === "comment" && + item.ref === undefined && + item.text === undefined + ) { + ctx.addIssue({ + code: "custom", + path: ["layout", index], + message: "A comment needs a ref, a text or both", + }); + } + }); + } else if (listsByName.size === 0) { ctx.addIssue({ code: "custom", message: - "At least one of set, remove, generate or copy must name a variable", + "At least one of set, remove, generate, copy or layout must be given", }); } @@ -168,4 +237,5 @@ export const apiEnvApplyResponse = z.object({ secret: z.boolean(), }), ), + layout: z.array(envLayoutItem).optional(), }); diff --git a/packages/server/src/services/env-variables.ts b/packages/server/src/services/env-variables.ts index 34a22f63e1b..5ffdf9e2275 100644 --- a/packages/server/src/services/env-variables.ts +++ b/packages/server/src/services/env-variables.ts @@ -22,6 +22,7 @@ import { TRPCError } from "@trpc/server"; import { eq } from "drizzle-orm"; import type { AnyPgColumn, PgTable } from "drizzle-orm/pg-core"; import type { z } from "zod"; +import { applyEnvLayout, readEnvLayout } from "../utils/env-layout"; import { applyEnvChanges, assertNoRedactedSecretPlaceholders, @@ -264,6 +265,18 @@ const readEnvText = async ( return text; }; +const writeEnvText = async ( + writer: Pick, + target: EnvTarget, + env: string, +) => { + const { table, idColumn } = ENV_TABLES[target.scope]; + await writer + .update(table) + .set({ [target.field]: env }) + .where(eq(idColumn, target.id)); +}; + const generateEnvValue = (length: number, format: EnvGenerateFormat) => { if (format === "alphanumeric") { return Array.from( @@ -287,6 +300,49 @@ export const listEnvVariableKeys = async ( }; }; +export const listEnvVariableLayout = async ( + input: z.infer, +) => { + const target = { scope: input.scope, id: input.id, field: input.field }; + const env = await readEnvText(db, target, false); + + let layout: ReturnType; + try { + layout = readEnvLayout(env); + } catch (error) { + throw new TRPCError({ + code: "PRECONDITION_FAILED", + message: error instanceof Error ? error.message : "Invalid env layout", + }); + } + + return { + ...target, + revision: getEnvTargetRevision(target, env), + variables: listEnvVariables(env), + layout, + }; +}; + +const applyLayoutChange = ( + currentEnv: string, + layout: NonNullable["layout"]>, +) => { + const texts = Object.fromEntries( + layout.flatMap((item, index) => + item.type === "comment" && item.text !== undefined + ? [[`layout[${index}]`, item.text]] + : [], + ), + ); + try { + assertNoRedactedSecretPlaceholders(texts); + return applyEnvLayout(currentEnv, layout); + } catch (error) { + throw badRequest(error); + } +}; + export const applyEnvVariableChanges = async ( input: z.infer, ) => { @@ -312,6 +368,22 @@ export const applyEnvVariableChanges = async ( }); } + if (input.layout) { + const result = applyLayoutChange(currentEnv, input.layout); + const summary = { + ...target, + changed: result.changed, + dryRun, + variables: [], + layout: result.layout, + }; + if (dryRun || !result.changed) { + return { ...summary, revision: currentRevision }; + } + await writeEnvText(tx, target, result.env); + return { ...summary, revision: getEnvTargetRevision(target, result.env) }; + } + const sourceKey = (source: EnvTarget) => `${source.scope}:${source.field}:${source.id}`; const sourceEnvs = new Map([[sourceKey(target), currentEnv]]); @@ -369,11 +441,7 @@ export const applyEnvVariableChanges = async ( return { ...summary, revision: currentRevision }; } - const { table, idColumn } = ENV_TABLES[target.scope]; - await tx - .update(table) - .set({ [target.field]: result.env }) - .where(eq(idColumn, target.id)); + await writeEnvText(tx, target, result.env); return { ...summary, revision: getEnvTargetRevision(target, result.env) }; }); diff --git a/packages/server/src/utils/env-layout.ts b/packages/server/src/utils/env-layout.ts new file mode 100644 index 00000000000..a09dab9dfd7 --- /dev/null +++ b/packages/server/src/utils/env-layout.ts @@ -0,0 +1,401 @@ +import { + type EnvEntry, + isEncodedDataFragment, + isSecretEnvName, + normalizeEnvText, + parseEnvValues, + scanEnvEntries, +} from "./env-upsert"; + +export type EnvLayoutHiddenKind = "comment" | "overridden" | "unreadable"; + +export type EnvLayoutItem = + | { type: "variable"; name: string } + | { type: "comment"; ref: string; text: string } + | { + type: "hidden"; + ref: string; + kind: EnvLayoutHiddenKind; + variable?: string; + } + | { type: "blank" }; + +export type EnvLayoutInputItem = + | { type: "variable"; name: string } + | { type: "comment"; ref?: string; text?: string } + | { type: "hidden"; ref: string } + | { type: "blank" }; + +export type EnvLayoutResult = { + env: string; + changed: boolean; + layout: EnvLayoutItem[]; +}; + +type EnvLayout = { + items: EnvLayoutItem[]; + variables: Map; + comments: Map; + hidden: Map; + earlierAssignments: Map; +}; + +type LineOwner = { entry: EnvEntry; first: number; last: number }; + +const COMMENT_LINE_REGEX = /^\s*#[^\S\n]?(.*)$/; +const BLANK_LINE_REGEX = /^\s*$/; +const COMMENTED_ASSIGNMENT_REGEX = /^\s*(?:export\s+)?([\w.-]+)\s*=/; +const COMMENTED_COLON_ASSIGNMENT_REGEX = + /^\s*(?:export\s+)?([A-Za-z_][\w.-]*)\s*:\s+\S/; +const URL_CREDENTIALS_REGEX = /[a-z][\w+.-]*:\/\/[^\s/@]+@/i; +const PEM_MARKER_REGEX = /-----(?:BEGIN|END)\b/; +const BEARER_REGEX = /\bbearer\s+\S/i; +const SECRET_WORD_VALUE_REGEX = + /(?:pass(?:word|wd)?|pwd|secret|token|key|credential|пароль|токен|ключ|секрет)\S*\s*[:=]\s*\S/iu; +const WORD_EDGE_PUNCTUATION_REGEX = /^[("'`<[{]+|[)"'`>\]},.;:!?]+$/g; + +const unsafeLayout = () => + new Error( + "The env text has a line structure that cannot be shown or rearranged safely", + ); + +const formatComment = (text: string) => (text === "" ? "#" : `# ${text}`); + +const looksLikeSecretWord = (word: string) => { + const bare = word.replace(WORD_EDGE_PUNCTUATION_REGEX, ""); + return ( + bare.length >= 40 || + (bare.length >= 16 && /[A-Za-z]/.test(bare) && /\d/.test(bare)) + ); +}; + +// Comments are shown so that sections and notes can be kept or rearranged, +// but people also park old values in them. Text that could hold a value is +// withheld: a commented-out assignment, credentials in a URL, a key block, +// a token-like word or a secret word followed by a value. +const describeComment = ( + text: string, +): { hidden: false } | { hidden: true; variable?: string } => { + const assignment = COMMENTED_ASSIGNMENT_REGEX.exec(text); + if (assignment) { + return { hidden: true, variable: assignment[1] }; + } + const colonAssignment = COMMENTED_COLON_ASSIGNMENT_REGEX.exec(text); + if ( + colonAssignment?.[1] && + (colonAssignment[1].includes("_") || isSecretEnvName(colonAssignment[1])) + ) { + return { hidden: true, variable: colonAssignment[1] }; + } + if ( + URL_CREDENTIALS_REGEX.test(text) || + PEM_MARKER_REGEX.test(text) || + BEARER_REGEX.test(text) || + SECRET_WORD_VALUE_REGEX.test(text) || + text.split(/\s+/).some(looksLikeSecretWord) + ) { + return { hidden: true }; + } + return { hidden: false }; +}; + +const splitEnvBody = (currentEnv: string | null | undefined) => { + const original = normalizeEnvText(currentEnv); + const trailingNewline = original.endsWith("\n"); + return { + original, + body: trailingNewline ? original.slice(0, -1) : original, + trailingNewline, + }; +}; + +const mapEntriesToLines = (body: string, lines: string[]) => { + const lineByOffset = new Map(); + let offset = 0; + lines.forEach((line, index) => { + lineByOffset.set(offset, index); + offset += line.length + 1; + }); + + const owners: (LineOwner | undefined)[] = new Array(lines.length); + for (const entry of scanEnvEntries(body)) { + const first = lineByOffset.get(entry.lineStart); + const endsText = entry.lineEnd >= body.length; + const next = endsText ? lines.length : lineByOffset.get(entry.lineEnd + 1); + // dotenv also ends a line at U+2028 and U+2029. An assignment that starts + // or ends at one of them shares a physical line with other text, and + // moving that line would move the other text with it. + if ( + first === undefined || + next === undefined || + (!endsText && body[entry.lineEnd] !== "\n") + ) { + throw unsafeLayout(); + } + const owner = { entry, first, last: next - 1 }; + for (let index = first; index < next; index++) { + if (owners[index]) { + throw unsafeLayout(); + } + owners[index] = owner; + } + } + return owners; +}; + +const scanEnvLayout = (body: string): EnvLayout => { + const lines = body.length > 0 ? body.split("\n") : []; + const owners = mapEntriesToLines(body, lines); + const values = parseEnvValues(body); + const isListed = (name: string) => + values.has(name) && !isEncodedDataFragment(name); + const isUnreadable = (index: number) => { + const owner = owners[index]; + if (owner) { + return !isListed(owner.entry.name); + } + const line = lines[index] ?? ""; + return !BLANK_LINE_REGEX.test(line) && !COMMENT_LINE_REGEX.test(line); + }; + const takeUnreadable = (from: number) => { + let index = from; + while (index < lines.length && isUnreadable(index)) { + index++; + } + return index; + }; + + type Segment = + | { kind: "variable"; name: string; text: string } + | { kind: "comment"; line: string; text: string } + | { + kind: "hidden"; + hiddenKind: EnvLayoutHiddenKind; + text: string; + variable?: string; + } + | { kind: "blank" }; + const segments: Segment[] = []; + let index = 0; + while (index < lines.length) { + const owner = owners[index]; + const line = lines[index] ?? ""; + if (owner && isListed(owner.entry.name)) { + // Lines dotenv cannot read right after an assignment are most often the + // rest of an unquoted value, such as a pasted key block, so they move + // with that variable. + const end = takeUnreadable(owner.last + 1); + segments.push({ + kind: "variable", + name: owner.entry.name, + text: lines.slice(index, end).join("\n"), + }); + index = end; + continue; + } + if (isUnreadable(index)) { + const end = takeUnreadable(index); + segments.push({ + kind: "hidden", + hiddenKind: "unreadable", + text: lines.slice(index, end).join("\n"), + }); + index = end; + continue; + } + const comment = COMMENT_LINE_REGEX.exec(line); + if (comment) { + const text = (comment[1] ?? "").trimEnd(); + const description = describeComment(text); + segments.push( + description.hidden + ? { + kind: "hidden", + hiddenKind: "comment", + text: line, + variable: description.variable, + } + : { kind: "comment", line, text }, + ); + } else { + segments.push({ kind: "blank" }); + } + index++; + } + + const lastSegmentByName = new Map(); + for (const segment of segments) { + if (segment.kind === "variable") { + lastSegmentByName.set(segment.name, segment); + } + } + + const layout: EnvLayout = { + items: [], + variables: new Map(), + comments: new Map(), + hidden: new Map(), + earlierAssignments: new Map(), + }; + for (const segment of segments) { + if (segment.kind === "variable") { + if (lastSegmentByName.get(segment.name) === segment) { + layout.items.push({ type: "variable", name: segment.name }); + layout.variables.set(segment.name, segment.text); + continue; + } + // An earlier assignment of a name that is assigned again has no effect + // but still holds a value. It is shown as a hidden line, so a layout + // sent back unchanged leaves the text as it is. + const ref = `h${layout.hidden.size + 1}`; + layout.items.push({ + type: "hidden", + ref, + kind: "overridden", + variable: segment.name, + }); + layout.hidden.set(ref, segment.text); + layout.earlierAssignments.set(ref, segment.name); + } else if (segment.kind === "comment") { + const ref = `c${layout.comments.size + 1}`; + layout.items.push({ type: "comment", ref, text: segment.text }); + layout.comments.set(ref, { line: segment.line, text: segment.text }); + } else if (segment.kind === "hidden") { + const ref = `h${layout.hidden.size + 1}`; + layout.items.push( + segment.variable === undefined + ? { type: "hidden", ref, kind: segment.hiddenKind } + : { + type: "hidden", + ref, + kind: segment.hiddenKind, + variable: segment.variable, + }, + ); + layout.hidden.set(ref, segment.text); + } else { + layout.items.push({ type: "blank" }); + } + } + return layout; +}; + +export const readEnvLayout = ( + currentEnv: string | null | undefined, +): EnvLayoutItem[] => scanEnvLayout(splitEnvBody(currentEnv).body).items; + +const listNames = (names: Iterable) => [...names].join(", "); + +export const applyEnvLayout = ( + currentEnv: string | null | undefined, + layout: EnvLayoutInputItem[], +): EnvLayoutResult => { + const { original, body, trailingNewline } = splitEnvBody(currentEnv); + const current = scanEnvLayout(body); + const unknownNames = new Set(); + const repeatedNames = new Set(); + const unknownRefs = new Set(); + const repeatedRefs = new Set(); + const lines: string[] = []; + const refPositions = new Map(); + const namePositions = new Map(); + + const placeRef = (ref: string, text: string | undefined) => { + if (text === undefined) { + unknownRefs.add(ref); + } else if (refPositions.has(ref)) { + repeatedRefs.add(ref); + } else { + refPositions.set(ref, lines.length); + lines.push(text); + } + }; + + for (const item of layout) { + if (item.type === "variable") { + const text = current.variables.get(item.name); + if (text === undefined) { + unknownNames.add(item.name); + } else if (namePositions.has(item.name)) { + repeatedNames.add(item.name); + } else { + namePositions.set(item.name, lines.length); + lines.push(text); + } + } else if (item.type === "comment") { + if (item.ref === undefined) { + lines.push(formatComment(item.text ?? "")); + continue; + } + const comment = current.comments.get(item.ref); + placeRef( + item.ref, + comment && + (item.text === undefined || item.text === comment.text + ? comment.line + : formatComment(item.text)), + ); + } else if (item.type === "hidden") { + placeRef(item.ref, current.hidden.get(item.ref)); + } else { + lines.push(""); + } + } + + const missingNames = [...current.variables.keys()].filter( + (name) => !namePositions.has(name), + ); + const missingHidden = [...current.hidden.keys()].filter( + (ref) => !refPositions.has(ref), + ); + const misplacedAssignments = [...current.earlierAssignments] + .filter( + ([ref, name]) => + (refPositions.get(ref) ?? -1) > + (namePositions.get(name) ?? Number.POSITIVE_INFINITY), + ) + .map(([ref, name]) => `${ref} (${name})`); + const problems = [ + unknownNames.size > 0 && + `The layout names variables that do not exist: ${listNames(unknownNames)}.`, + repeatedNames.size > 0 && + `The layout places variables more than once: ${listNames(repeatedNames)}.`, + missingNames.length > 0 && + `The layout leaves out variables: ${listNames(missingNames)}. Every variable has to be placed; delete one with remove instead.`, + unknownRefs.size > 0 && + `The layout refers to lines that do not exist: ${listNames(unknownRefs)}.`, + repeatedRefs.size > 0 && + `The layout places lines more than once: ${listNames(repeatedRefs)}.`, + missingHidden.length > 0 && + `The layout leaves out hidden lines: ${listNames(missingHidden)}. Their text is not shown, so they can be moved but not dropped.`, + misplacedAssignments.length > 0 && + `These hidden lines are earlier assignments of a variable and have to stay above it: ${listNames(misplacedAssignments)}.`, + ].filter((problem): problem is string => typeof problem === "string"); + if (problems.length > 0) { + throw new Error(problems.join(" ")); + } + + const nextBody = lines.join("\n"); + const nextEnv = + trailingNewline && nextBody.length > 0 ? `${nextBody}\n` : nextBody; + + // Moving whole lines keeps every value as long as dotenv splits the text the + // same way; a quote that pairs up with a different line would not. + const before = parseEnvValues(body); + const after = parseEnvValues(nextEnv); + const brokenNames = [...new Set([...before.keys(), ...after.keys()])].filter( + (name) => before.get(name) !== after.get(name), + ); + if (brokenNames.length > 0) { + throw new Error( + `Environment variables cannot be rearranged safely: ${listNames(brokenNames)}`, + ); + } + + const changed = nextEnv !== original; + return { + env: changed ? nextEnv : (currentEnv ?? ""), + changed, + layout: changed ? readEnvLayout(nextEnv) : current.items, + }; +}; diff --git a/packages/server/src/utils/env-upsert.ts b/packages/server/src/utils/env-upsert.ts index 9d482d18c05..ef38a2fbc03 100644 --- a/packages/server/src/utils/env-upsert.ts +++ b/packages/server/src/utils/env-upsert.ts @@ -220,7 +220,7 @@ export type EnvChangeResult = { const ENV_ENTRY_REGEX = /^(\s*)((?:export\s+)?)([\w.-]+)(\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*`(?:\\`|[^`])*`|[^#\r\n]+)?\s*(?:#.*)?$/gm; -type EnvEntry = { +export type EnvEntry = { name: string; lineStart: number; valueStart: number; @@ -228,12 +228,13 @@ type EnvEntry = { lineEnd: number; }; -const normalizeEnvText = (env: string | null | undefined) => +export const normalizeEnvText = (env: string | null | undefined) => (env ?? "").replace(/\r\n?/g, "\n"); -const parseEnvValues = (text: string) => new Map(Object.entries(parse(text))); +export const parseEnvValues = (text: string) => + new Map(Object.entries(parse(text))); -const scanEnvEntries = (text: string): EnvEntry[] => { +export const scanEnvEntries = (text: string): EnvEntry[] => { const lineBreak = /[\n\u2028\u2029]/g; return Array.from(text.matchAll(ENV_ENTRY_REGEX), (match) => { @@ -292,7 +293,7 @@ const serializeExactEnvValue = (name: string, value: string) => { // A key block pasted without quotes makes dotenv read its base64 lines that // end in `=` as assignments. Such a "name" is a piece of the secret, so a name // that looks like encoded data is left out of the listing. -const isEncodedDataFragment = (name: string) => +export const isEncodedDataFragment = (name: string) => name.length >= 20 && !name.includes("_") && /[a-z]/.test(name) &&