From 9b575dc2ac8015301ac8e24564722cc4f29d91c0 Mon Sep 17 00:00:00 2001 From: merge-check <140916359+agentHits@users.noreply.github.com> Date: Sun, 25 Oct 2026 06:31:16 -0400 Subject: [PATCH] feat(env): add comments and sections to env without reading values MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Что: - Добавлен `GET /env.layout` (tRPC `env.layout`): к ответу `env.keys` добавляется `layout`, текст сверху вниз. Переменные даны по имени, комментарии с текстом и ссылкой `c`, пустые строки, а строки, где может быть значение, скрыты под ссылкой `h` с видом `comment`, `overridden` или `unreadable`. - Скрываются закомментированные присваивания (с именем переменной), URL с паролем, PEM-маркеры, `Bearer`, секретное слово со значением после него, слова, похожие на токен, ранние перекрытые присваивания и строки, которые dotenv не читает. - `POST /env.apply` принимает `layout`: весь новый порядок строк. Можно переставлять переменные, добавлять комментарии и пустые строки, менять текст комментария по ссылке и удалять видимые комментарии. Строка каждой переменной переносится байт в байт вместе со строками неэкранированного значения сразу за ней. - `layout` отправляется только отдельно от set, remove, generate и copy и только с `expectedRevision`. Каждая переменная и каждая скрытая строка должны стоять в списке ровно один раз, а раннее присваивание — выше действующего. После перестановки dotenv должен прочитать те же значения, иначе запись отклоняется. Ошибки называют только имена и ссылки. - Раскладка, отправленная без изменений, текст не меняет. Ответ `env.apply` с `layout` содержит новую раскладку, `dryRun` её показывает без записи. - Помощники разбора env в `env-upsert.ts` экспортированы для нового модуля `env-layout.ts`; запись в таблицу вынесена в `writeEnvText`. - `env.layout` классифицирована для супер-сессии как `read`, как и `env.keys`. Зачем: - Агент через Dokploy MCP мог менять отдельные переменные, но не мог добавить комментарий или разложить переменные по секциям: для полной переписи env нужен был скрипт владельца. Риски: - Текст видимых комментариев теперь уходит клиенту API. Отбор строк со значениями идёт по шаблонам: фраза вроде «пароль hunter2» без `:` или `=` будет показана. MCP дополнительно прогоняет этот текст через свой скраббер. - Скрытые строки нельзя удалить через API, только в панели. - Текст, где присваивание делит строку с другим текстом через U+2028 или U+2029, раскладка не показывает и не меняет (412 и 400). Проверки: - Команды и результаты: `vitest run __test__/env/ __test__/api/openapi-document.test.ts` — 266 тестов проходят, 16 файлов (новые `layout.test.ts` и тесты `env layout` и REST в `blind-env-router.test.ts`); `bun run typecheck` в `apps/dokploy` и `packages/server` — без ошибок; `biome check` по изменённым файлам — чисто; полный `vitest run` — 4109 тестов проходят, 5 пропущены (первый прогон поймал неклассифицированную `env.layout` в `super-session-procedure-access.test.ts`, исправлено). - Ограничения: живая панель не вызывалась. What: - Added `GET /env.layout` (tRPC `env.layout`): the `env.keys` answer plus `layout`, the text from top to bottom. Variables are given by name, comments with their text and a `c` ref, blank lines, and lines that may hold a value are hidden behind an `h` ref with the kind `comment`, `overridden` or `unreadable`. - Hidden are commented-out assignments (with the variable name), URLs with a password, PEM markers, `Bearer`, a secret word followed by a value, token-like words, earlier overridden assignments and lines dotenv cannot read. - `POST /env.apply` takes `layout`: the whole new order of lines. Variables can be moved, comments and blank lines added, a comment's text changed by its ref and shown comments dropped. Each variable's lines move byte for byte, together with the lines of an unquoted value right after it. - `layout` is sent only apart from set, remove, generate and copy, and only with `expectedRevision`. Every variable and every hidden line has to be in the list exactly once, and an earlier assignment above the effective one. dotenv has to read the same values after the move, otherwise the write is refused. Errors name only names and refs. - A layout sent back unchanged leaves the text as it is. The `env.apply` answer for a layout holds the new layout, and `dryRun` shows it without writing. - The env parsing helpers in `env-upsert.ts` are exported for the new `env-layout.ts` module; the table write moved into `writeEnvText`. - `env.layout` is classified for the super session as `read`, like `env.keys`. Why: - Through Dokploy MCP an agent could change single variables but could not add a comment or arrange variables into sections: a full env rewrite needed the owner's script. Risks: - The text of shown comments now reaches the API client. Lines that may hold a value are picked by patterns: a phrase like "password hunter2" without `:` or `=` is shown. MCP also runs this text through its scrubber. - Hidden lines cannot be deleted through the API, only in the panel. - A text where an assignment shares a line with other text through U+2028 or U+2029 is neither shown nor changed as a layout (412 and 400). Checks: - Commands and results: `vitest run __test__/env/ __test__/api/openapi-document.test.ts` — 266 tests pass in 16 files (new `layout.test.ts` and the `env layout` and REST tests in `blind-env-router.test.ts`); `bun run typecheck` in `apps/dokploy` and `packages/server` — no errors; `biome check` on the changed files — clean; full `vitest run` — 4109 tests pass, 5 skipped (the first run caught the unclassified `env.layout` in `super-session-procedure-access.test.ts`, fixed). - Limitations: no live panel was called. Co-Authored-By: Claude Opus 5.5 --- .../__test__/api/openapi-document.test.ts | 53 ++- .../__test__/env/blind-env-router.test.ts | 313 ++++++++++++++ apps/dokploy/__test__/env/layout.test.ts | 313 ++++++++++++++ apps/dokploy/server/api/routers/env.ts | 9 + .../server/api/utils/procedure-access.ts | 1 + .../server/src/db/schema/env-variables.ts | 74 +++- packages/server/src/services/env-variables.ts | 78 +++- packages/server/src/utils/env-layout.ts | 401 ++++++++++++++++++ packages/server/src/utils/env-upsert.ts | 11 +- 9 files changed, 1238 insertions(+), 15 deletions(-) create mode 100644 apps/dokploy/__test__/env/layout.test.ts create mode 100644 packages/server/src/utils/env-layout.ts diff --git a/apps/dokploy/__test__/api/openapi-document.test.ts b/apps/dokploy/__test__/api/openapi-document.test.ts index 22bfbf3f5b5..473ea487676 100644 --- a/apps/dokploy/__test__/api/openapi-document.test.ts +++ b/apps/dokploy/__test__/api/openapi-document.test.ts @@ -24,19 +24,23 @@ describe("OpenAPI document", () => { ); }); - it("serves blind env management at GET /env.keys and POST /env.apply", () => { + it("serves blind env management at GET /env.keys, GET /env.layout and POST /env.apply", () => { const paths = generate().paths ?? {}; const keys = paths["/env.keys"]?.get as | { operationId?: string; parameters?: { name: string; in: string }[] } | undefined; + const layout = paths["/env.layout"]?.get as + | { operationId?: string; parameters?: { name: string; in: string }[] } + | undefined; const apply = paths["/env.apply"]?.post as | { operationId?: string; requestBody?: unknown } | undefined; expect( Object.keys(paths).filter((path) => path.startsWith("/env.")), - ).toEqual(["/env.keys", "/env.apply"]); + ).toEqual(["/env.keys", "/env.layout", "/env.apply"]); expect(Object.keys(paths["/env.keys"] ?? {})).toEqual(["get"]); + expect(Object.keys(paths["/env.layout"] ?? {})).toEqual(["get"]); expect(Object.keys(paths["/env.apply"] ?? {})).toEqual(["post"]); expect(keys?.operationId).toBe("env-keys"); expect(keys?.parameters).toEqual([ @@ -44,6 +48,12 @@ describe("OpenAPI document", () => { expect.objectContaining({ name: "id", in: "query", required: true }), expect.objectContaining({ name: "field", in: "query" }), ]); + expect(layout?.operationId).toBe("env-layout"); + expect(layout?.parameters?.map(({ name }) => name)).toEqual([ + "scope", + "id", + "field", + ]); expect(apply?.operationId).toBe("env-apply"); expect( Object.keys( @@ -63,6 +73,7 @@ describe("OpenAPI document", () => { "remove", "generate", "copy", + "layout", "dryRun", "expectedRevision", ]); @@ -95,7 +106,23 @@ describe("OpenAPI document", () => { } ).responses[200].content["application/json"].schema; const keys = responseSchema(paths["/env.keys"]?.get); + const layout = responseSchema(paths["/env.layout"]?.get); const apply = responseSchema(paths["/env.apply"]?.post); + const layoutItems = (schema: unknown) => + ( + schema as { + properties: { + layout: { + items: { + oneOf: { + properties: object; + additionalProperties: boolean; + }[]; + }; + }; + }; + } + ).properties.layout.items.oneOf; expect(Object.keys(keys.properties)).toEqual([ "scope", @@ -117,14 +144,34 @@ describe("OpenAPI document", () => { "revision", "dryRun", "variables", + "layout", ]); + expect(Object.keys(layout.properties)).toEqual([ + "scope", + "id", + "field", + "revision", + "variables", + "layout", + ]); + for (const items of [layoutItems(layout), layoutItems(apply)]) { + expect(items.map(({ properties }) => Object.keys(properties))).toEqual([ + ["type", "name"], + ["type", "ref", "text"], + ["type", "ref", "kind", "variable"], + ["type"], + ]); + for (const item of items) { + expect(item.additionalProperties).toBe(false); + } + } expect(Object.keys(apply.properties.variables.items.properties)).toEqual([ "name", "action", "source", "secret", ]); - for (const schema of [keys, apply]) { + for (const schema of [keys, layout, apply]) { expect(schema.additionalProperties).toBe(false); expect(schema.properties.variables.items.additionalProperties).toBe( false, diff --git a/apps/dokploy/__test__/env/blind-env-router.test.ts b/apps/dokploy/__test__/env/blind-env-router.test.ts index bdadfe48029..5f869ec34b3 100644 --- a/apps/dokploy/__test__/env/blind-env-router.test.ts +++ b/apps/dokploy/__test__/env/blind-env-router.test.ts @@ -1179,6 +1179,293 @@ describe("env.applyChanges", () => { }); }); +describe("env layout", () => { + const appLayout = [ + { type: "comment", ref: "c1", text: "App settings" }, + { type: "hidden", ref: "h1", kind: "overridden", variable: "SERVICE_URL" }, + { type: "variable", name: "SERVICE_TOKEN" }, + { type: "variable", name: "EMPTY_FLAG" }, + { type: "variable", name: "SERVICE_URL" }, + ]; + const sectioned = [ + { type: "comment", text: "=== Web ===" }, + { type: "hidden", ref: "h1" }, + { type: "variable", name: "SERVICE_URL" }, + { type: "variable", name: "EMPTY_FLAG" }, + { type: "blank" }, + { type: "comment", text: "=== Auth ===" }, + { type: "comment", ref: "c1", text: "Signs sessions" }, + { type: "variable", name: "SERVICE_TOKEN" }, + ] as const; + const sectionedEnv = `# === Web ===\nSERVICE_URL=https://app.example.test\nSERVICE_URL=https://override.example.test\nEMPTY_FLAG=\n\n# === Auth ===\n# Signs sessions\nSERVICE_TOKEN=${SENTINEL}\n`; + + it("reads the layout with names, comments and a revision but never a value", async () => { + const result = await createCaller().layout({ + scope: "application", + id: "app-1", + }); + + expect(result).toEqual({ + scope: "application", + id: "app-1", + field: "env", + revision: getApplicationEnvRevision("app-1", storedEnv(applications)), + variables: [ + { name: "SERVICE_URL", secret: false, empty: false }, + { name: "SERVICE_TOKEN", secret: true, empty: false }, + { name: "EMPTY_FLAG", secret: false, empty: true }, + ], + layout: appLayout, + }); + expect(JSON.stringify(result)).not.toContain(SENTINEL); + expect(JSON.stringify(result)).not.toContain("example.test"); + expect( + mocks.checkServicePermissionAndAccess, + ).toHaveBeenCalledExactlyOnceWith(ctxMatcher, "app-1", { + envVars: ["read"], + }); + expect(fake.locks).toEqual([]); + }); + + it("checks the project variable permission before showing a layout", async () => { + mocks.checkPermission.mockRejectedValueOnce( + new TRPCError({ code: "UNAUTHORIZED", message: "denied" }), + ); + + await expectTrpcError( + createCaller().layout({ scope: "project", id: "project-1" }), + "UNAUTHORIZED", + ); + expect(mocks.checkPermission).toHaveBeenCalledExactlyOnceWith(ctxMatcher, { + projectEnvVars: ["read"], + }); + }); + + it("rearranges variables into sections under the row lock and audits once", async () => { + const before = storedEnv(applications); + const result = await createCaller().applyChanges({ + scope: "application", + id: "app-1", + layout: [...sectioned], + expectedRevision: getApplicationEnvRevision("app-1", before), + }); + + const after = storedEnv(applications); + expect(after).toBe(sectionedEnv); + expect(parse(after ?? "")).toEqual(parse(before ?? "")); + expect(result).toEqual({ + scope: "application", + id: "app-1", + field: "env", + changed: true, + revision: getApplicationEnvRevision("app-1", after), + dryRun: false, + variables: [], + layout: [ + { type: "comment", ref: "c1", text: "=== Web ===" }, + { + type: "hidden", + ref: "h1", + kind: "overridden", + variable: "SERVICE_URL", + }, + { type: "variable", name: "SERVICE_URL" }, + { type: "variable", name: "EMPTY_FLAG" }, + { type: "blank" }, + { type: "comment", ref: "c2", text: "=== Auth ===" }, + { type: "comment", ref: "c3", text: "Signs sessions" }, + { type: "variable", name: "SERVICE_TOKEN" }, + ], + }); + expect(JSON.stringify(result)).not.toContain(SENTINEL); + expect(fake.locks).toEqual([{ table: "application", strength: "update" }]); + expect(mocks.audit).toHaveBeenCalledExactlyOnceWith(ctxMatcher, { + action: "update", + resourceType: "application", + resourceId: "app-1", + resourceName: "app-one", + }); + expect( + mocks.checkServicePermissionAndAccess, + ).toHaveBeenCalledExactlyOnceWith(ctxMatcher, "app-1", { + envVars: ["write"], + }); + }); + + it("previews a layout on a dry run without writing or auditing", async () => { + const before = storedEnv(applications); + const result = await createCaller().applyChanges({ + scope: "application", + id: "app-1", + layout: [...sectioned], + dryRun: true, + expectedRevision: getApplicationEnvRevision("app-1", before), + }); + + expect(result.changed).toBe(true); + expect(result.dryRun).toBe(true); + expect(result.revision).toBe(getApplicationEnvRevision("app-1", before)); + expect(result.layout?.map(({ type }) => type)).toEqual([ + "comment", + "hidden", + "variable", + "variable", + "blank", + "comment", + "comment", + "variable", + ]); + expect(storedEnv(applications)).toBe(before); + expect(mocks.audit).not.toHaveBeenCalled(); + }); + + it("does not write or audit a layout that is sent back unchanged", async () => { + const before = storedEnv(applications); + const result = await createCaller().applyChanges({ + scope: "application", + id: "app-1", + layout: appLayout as never, + expectedRevision: getApplicationEnvRevision("app-1", before), + }); + + expect(result).toMatchObject({ changed: false, layout: appLayout }); + expect(storedEnv(applications)).toBe(before); + expect(mocks.audit).not.toHaveBeenCalled(); + }); + + it("rearranges build args of an application and variables of a project", async () => { + await createCaller().applyChanges({ + scope: "application", + id: "app-1", + field: "buildArgs", + layout: [ + { type: "comment", text: "Build" }, + { type: "variable", name: "BUILD_MODE" }, + ], + expectedRevision: getEnvFieldRevision( + "application", + "app-1", + "buildArgs", + "BUILD_MODE=release\n", + ), + }); + await createCaller().applyChanges({ + scope: "project", + id: "project-1", + layout: [ + { type: "variable", name: "SHARED_TOKEN" }, + { type: "variable", name: "SHARED_URL" }, + ], + expectedRevision: getEnvFieldRevision( + "project", + "project-1", + "env", + storedEnv(projects), + ), + }); + + expect(storedEnv(applications, "buildArgs")).toBe( + "# Build\nBUILD_MODE=release\n", + ); + expect(storedEnv(projects)).toBe( + `SHARED_TOKEN=${SENTINEL}\nSHARED_URL=https://shared.example.test\n`, + ); + }); + + it("rejects a layout combined with value changes or without a revision", async () => { + const before = storedEnv(applications); + + const combined = await expectTrpcError( + createCaller().applyChanges({ + scope: "application", + id: "app-1", + set: { NEW_NAME: "1" }, + layout: [...sectioned], + expectedRevision: getApplicationEnvRevision("app-1", before), + }), + "BAD_REQUEST", + ); + const unpinned = await expectTrpcError( + createCaller().applyChanges({ + scope: "application", + id: "app-1", + layout: [...sectioned], + }), + "BAD_REQUEST", + ); + const empty = await expectTrpcError( + createCaller().applyChanges({ + scope: "application", + id: "app-1", + layout: [{ type: "comment" }, ...sectioned], + expectedRevision: getApplicationEnvRevision("app-1", before), + }), + "BAD_REQUEST", + ); + + expect(combined.message).toContain("layout cannot be combined"); + expect(unpinned.message).toContain("expectedRevision"); + expect(empty.message).toContain("A comment needs a ref, a text or both"); + expect(storedEnv(applications)).toBe(before); + expect(mocks.checkServicePermissionAndAccess).not.toHaveBeenCalled(); + }); + + it("rejects a stale revision without writing", async () => { + const before = storedEnv(applications); + + await expectTrpcError( + createCaller().applyChanges({ + scope: "application", + id: "app-1", + layout: [...sectioned], + expectedRevision: "stale", + }), + "CONFLICT", + ); + expect(storedEnv(applications)).toBe(before); + }); + + it("rejects a layout that drops a variable, naming only names", async () => { + const before = storedEnv(applications); + + const error = await expectTrpcError( + createCaller().applyChanges({ + scope: "application", + id: "app-1", + layout: sectioned.filter( + (item) => item.type !== "variable" || item.name !== "SERVICE_TOKEN", + ), + expectedRevision: getApplicationEnvRevision("app-1", before), + }), + "BAD_REQUEST", + ); + + expect(error.message).toContain( + "The layout leaves out variables: SERVICE_TOKEN.", + ); + expect(error.message).not.toContain(SENTINEL); + expect(storedEnv(applications)).toBe(before); + expect(mocks.audit).not.toHaveBeenCalled(); + }); + + it("rejects comment text that carries a redacted placeholder", async () => { + const before = storedEnv(applications); + + const error = await expectTrpcError( + createCaller().applyChanges({ + scope: "application", + id: "app-1", + layout: [{ type: "comment", text: "old [REDACTED]" }, ...sectioned], + expectedRevision: getApplicationEnvRevision("app-1", before), + }), + "BAD_REQUEST", + ); + + expect(error.message).toContain("layout[0]"); + expect(storedEnv(applications)).toBe(before); + }); +}); + describe("REST routes", () => { it("serves GET /env.keys from query parameters", async () => { const response = await callRest("GET", "env.keys", { @@ -1268,6 +1555,32 @@ describe("REST routes", () => { expect(JSON.stringify(response)).not.toContain(values.NEW_TOKEN); }); + it("serves GET /env.layout and a layout through POST /env.apply", async () => { + const read = await callRest("GET", "env.layout", { + scope: "postgres", + id: "postgres-1", + }); + const layout = (read.body as { layout: unknown[]; revision: string }) + .layout; + const applied = await callRest("POST", "env.apply", { + scope: "postgres", + id: "postgres-1", + layout: [{ type: "comment", text: "Replication" }, ...layout.reverse()], + expectedRevision: (read.body as { revision: string }).revision, + }); + + expect(read.status).toBe(200); + expect(layout).toEqual([ + { type: "variable", name: "REPLICATION_PASSWORD" }, + { type: "variable", name: "POSTGRES_INITDB_ARGS" }, + ]); + expect(applied.status).toBe(200); + expect(storedEnv(postgres)).toBe( + `# Replication\nREPLICATION_PASSWORD=${OTHER_SENTINEL}\nPOSTGRES_INITDB_ARGS=--data-checksums\n`, + ); + expect(JSON.stringify([read, applied])).not.toContain(OTHER_SENTINEL); + }); + it("maps errors to HTTP statuses without echoing values", async () => { const conflict = await callRest("POST", "env.apply", { scope: "application", diff --git a/apps/dokploy/__test__/env/layout.test.ts b/apps/dokploy/__test__/env/layout.test.ts new file mode 100644 index 00000000000..d2e9d1a33ce --- /dev/null +++ b/apps/dokploy/__test__/env/layout.test.ts @@ -0,0 +1,313 @@ +import { + applyEnvLayout, + type EnvLayoutInputItem, + readEnvLayout, +} from "@dokploy/server/utils/env-layout"; +import { parse } from "dotenv"; +import { describe, expect, it } from "vitest"; + +const SENTINEL = "sentinel-value-0001"; +const KEY_LINE = + "MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQC7VJTUt9Us8cKj="; + +const ENV = `# Production settings +SERVICE_URL=https://app.example.test # public address +# OLD_TOKEN=${SENTINEL} +# Signs the session cookies +AUTH_SECRET=${SENTINEL} + +SERVICE_KEY=-----BEGIN PRIVATE KEY----- +${KEY_LINE} +-----END PRIVATE KEY----- +NAME=first +MULTI_LINE="first line +second line" +NAME=second +`; + +describe("readEnvLayout", () => { + it("lists variables, comments and blank lines in file order without values", () => { + const layout = readEnvLayout(ENV); + + expect(layout).toEqual([ + { type: "comment", ref: "c1", text: "Production settings" }, + { type: "variable", name: "SERVICE_URL" }, + { type: "hidden", ref: "h1", kind: "comment", variable: "OLD_TOKEN" }, + { type: "comment", ref: "c2", text: "Signs the session cookies" }, + { type: "variable", name: "AUTH_SECRET" }, + { type: "blank" }, + { type: "variable", name: "SERVICE_KEY" }, + { type: "hidden", ref: "h2", kind: "overridden", variable: "NAME" }, + { type: "variable", name: "MULTI_LINE" }, + { type: "variable", name: "NAME" }, + ]); + const text = JSON.stringify(layout); + expect(text).not.toContain(SENTINEL); + expect(text).not.toContain(KEY_LINE); + expect(text).not.toContain("public address"); + }); + + it("withholds comments that could hold a value", () => { + const hidden = [ + `# export API_TOKEN = ${SENTINEL}`, + `# DATABASE_PASSWORD: ${SENTINEL}`, + "# admin password: hunter2", + "# пароль=hunter2", + "# mirror at https://deploy:hunter2@git.example.test/repo", + "# -----BEGIN OPENSSH PRIVATE KEY-----", + "# Authorization: Bearer abc", + `# old one was ${SENTINEL}`, + "# 0123456789abcdef0123", + ]; + const shown = [ + "# === Database ===", + "# TODO: rotate before release", + "# Secret used to sign sessions", + "#no space after the hash", + "# Docs: https://docs.example.test/env", + ]; + + const layout = readEnvLayout([...hidden, ...shown, "LAST=1"].join("\n")); + + expect(layout.slice(0, hidden.length)).toEqual( + hidden.map((_, index) => + expect.objectContaining({ ref: `h${index + 1}` }), + ), + ); + expect(layout[0]).toEqual({ + type: "hidden", + ref: "h1", + kind: "comment", + variable: "API_TOKEN", + }); + expect(layout[1]).toEqual({ + type: "hidden", + ref: "h2", + kind: "comment", + variable: "DATABASE_PASSWORD", + }); + expect(layout.slice(hidden.length, -1)).toEqual([ + { type: "comment", ref: "c1", text: "=== Database ===" }, + { type: "comment", ref: "c2", text: "TODO: rotate before release" }, + { type: "comment", ref: "c3", text: "Secret used to sign sessions" }, + { type: "comment", ref: "c4", text: "no space after the hash" }, + { + type: "comment", + ref: "c5", + text: "Docs: https://docs.example.test/env", + }, + ]); + const text = JSON.stringify(layout); + for (const secret of [SENTINEL, "hunter2", "0123456789abcdef0123"]) { + expect(text).not.toContain(secret); + } + }); + + it("hides lines dotenv cannot read when no variable comes right before them", () => { + expect( + readEnvLayout(`${KEY_LINE} +-----END PRIVATE KEY----- + +FIRST=1 +# note +stray text`), + ).toEqual([ + { type: "hidden", ref: "h1", kind: "unreadable" }, + { type: "blank" }, + { type: "variable", name: "FIRST" }, + { type: "comment", ref: "c1", text: "note" }, + { type: "hidden", ref: "h2", kind: "unreadable" }, + ]); + }); + + it("treats a missing field as an empty layout", () => { + expect(readEnvLayout(null)).toEqual([]); + expect(readEnvLayout(undefined)).toEqual([]); + expect(readEnvLayout("")).toEqual([]); + }); + + it("refuses a text where an assignment shares a line with other text", () => { + expect(() => readEnvLayout("# note
NAME=1\n")).toThrow( + "cannot be shown or rearranged safely", + ); + }); +}); + +describe("applyEnvLayout", () => { + const sections: EnvLayoutInputItem[] = [ + { type: "comment", ref: "c1" }, + { type: "blank" }, + { type: "comment", text: "=== Web ===" }, + { type: "variable", name: "SERVICE_URL" }, + { type: "hidden", ref: "h2" }, + { type: "variable", name: "NAME" }, + { type: "blank" }, + { type: "comment", text: "=== Auth ===" }, + { type: "comment", ref: "c2", text: "Signs session cookies" }, + { type: "variable", name: "AUTH_SECRET" }, + { type: "hidden", ref: "h1" }, + { type: "variable", name: "SERVICE_KEY" }, + { type: "comment", text: "" }, + { type: "variable", name: "MULTI_LINE" }, + ]; + + it("arranges variables into sections and keeps every value byte for byte", () => { + const result = applyEnvLayout(ENV, sections); + + expect(result.changed).toBe(true); + expect(result.env).toBe(`# Production settings + +# === Web === +SERVICE_URL=https://app.example.test # public address +NAME=first +NAME=second + +# === Auth === +# Signs session cookies +AUTH_SECRET=${SENTINEL} +# OLD_TOKEN=${SENTINEL} +SERVICE_KEY=-----BEGIN PRIVATE KEY----- +${KEY_LINE} +-----END PRIVATE KEY----- +# +MULTI_LINE="first line +second line" +`); + expect(parse(result.env)).toEqual(parse(ENV)); + expect(result.layout).toEqual([ + { type: "comment", ref: "c1", text: "Production settings" }, + { type: "blank" }, + { type: "comment", ref: "c2", text: "=== Web ===" }, + { type: "variable", name: "SERVICE_URL" }, + { type: "hidden", ref: "h1", kind: "overridden", variable: "NAME" }, + { type: "variable", name: "NAME" }, + { type: "blank" }, + { type: "comment", ref: "c3", text: "=== Auth ===" }, + { type: "comment", ref: "c4", text: "Signs session cookies" }, + { type: "variable", name: "AUTH_SECRET" }, + { type: "hidden", ref: "h2", kind: "comment", variable: "OLD_TOKEN" }, + { type: "variable", name: "SERVICE_KEY" }, + { type: "comment", ref: "c5", text: "" }, + { type: "variable", name: "MULTI_LINE" }, + ]); + expect(JSON.stringify(result.layout)).not.toContain(SENTINEL); + }); + + it("keeps the text untouched when the layout is sent back unchanged", () => { + const env = "# Header\r\n#Tight comment\r\nFIRST=1\r\n\r\nSECOND=2\r\n"; + const layout = readEnvLayout(env); + + expect(applyEnvLayout(env, layout)).toEqual({ + env, + changed: false, + layout, + }); + }); + + it("rewrites a comment only when its text changes", () => { + const env = "# Indented note\nFIRST=1\n"; + + expect(readEnvLayout(env)[0]).toEqual({ + type: "comment", + ref: "c1", + text: " Indented note", + }); + expect( + applyEnvLayout(env, [ + { type: "comment", ref: "c1", text: " Indented note" }, + { type: "variable", name: "FIRST" }, + ]).changed, + ).toBe(false); + expect( + applyEnvLayout(env, [ + { type: "comment", ref: "c1", text: "New note" }, + { type: "variable", name: "FIRST" }, + ]).env, + ).toBe("# New note\nFIRST=1\n"); + }); + + it("drops a shown comment that the layout leaves out", () => { + expect( + applyEnvLayout("# old header\nFIRST=1", [ + { type: "variable", name: "FIRST" }, + ]).env, + ).toBe("FIRST=1"); + }); + + it("starts an empty text with comments only", () => { + expect( + applyEnvLayout("", [{ type: "comment", text: "=== Settings ===" }]), + ).toEqual({ + env: "# === Settings ===", + changed: true, + layout: [{ type: "comment", ref: "c1", text: "=== Settings ===" }], + }); + }); + + it("names every problem with names and refs only", () => { + const attempt = () => + applyEnvLayout(ENV, [ + { type: "variable", name: "SERVICE_URL" }, + { type: "variable", name: "SERVICE_URL" }, + { type: "variable", name: "UNKNOWN_NAME" }, + { type: "comment", ref: "c1" }, + { type: "comment", ref: "c1" }, + { type: "comment", ref: "c9" }, + { type: "hidden", ref: "h7" }, + ]); + + expect(attempt).toThrow( + "The layout names variables that do not exist: UNKNOWN_NAME. " + + "The layout places variables more than once: SERVICE_URL. " + + "The layout leaves out variables: AUTH_SECRET, SERVICE_KEY, MULTI_LINE, NAME. Every variable has to be placed; delete one with remove instead. " + + "The layout refers to lines that do not exist: c9, h7. " + + "The layout places lines more than once: c1. " + + "The layout leaves out hidden lines: h1, h2. Their text is not shown, so they can be moved but not dropped.", + ); + const message = (() => { + try { + attempt(); + } catch (error) { + return (error as Error).message; + } + })(); + expect(message).not.toContain(SENTINEL); + }); + + it("keeps an earlier assignment of a name above the one that counts", () => { + const env = "NAME=first\nOTHER=1\nNAME=second\n"; + + expect(readEnvLayout(env)).toEqual([ + { type: "hidden", ref: "h1", kind: "overridden", variable: "NAME" }, + { type: "variable", name: "OTHER" }, + { type: "variable", name: "NAME" }, + ]); + expect( + applyEnvLayout(env, [ + { type: "variable", name: "OTHER" }, + { type: "hidden", ref: "h1" }, + { type: "variable", name: "NAME" }, + ]).env, + ).toBe("OTHER=1\nNAME=first\nNAME=second\n"); + expect(() => + applyEnvLayout(env, [ + { type: "variable", name: "NAME" }, + { type: "hidden", ref: "h1" }, + { type: "variable", name: "OTHER" }, + ]), + ).toThrow( + "These hidden lines are earlier assignments of a variable and have to stay above it: h1 (NAME).", + ); + }); + + it("refuses a layout after which dotenv would read another value", () => { + const env = 'LATER=1 "\nOPEN="value\n'; + + expect(() => + applyEnvLayout(env, [ + { type: "variable", name: "OPEN" }, + { type: "variable", name: "LATER" }, + ]), + ).toThrow("Environment variables cannot be rearranged safely: LATER, OPEN"); + }); +}); diff --git a/apps/dokploy/server/api/routers/env.ts b/apps/dokploy/server/api/routers/env.ts index 687ffbb2ed8..f2366d3f181 100644 --- a/apps/dokploy/server/api/routers/env.ts +++ b/apps/dokploy/server/api/routers/env.ts @@ -4,11 +4,13 @@ import { apiEnvApplyResponse, apiEnvKeys, apiEnvKeysResponse, + apiEnvLayoutResponse, } from "@dokploy/server/db/schema"; import { applyEnvVariableChanges, findEnvVariablesOwner, listEnvVariableKeys, + listEnvVariableLayout, } from "@dokploy/server/services/env-variables"; import { checkEnvironmentAccess, @@ -88,6 +90,13 @@ export const envRouter = createTRPCRouter({ await assertEnvAccess(ctx, input.scope, input.id, "read"); return await listEnvVariableKeys(input); }), + layout: protectedProcedure + .input(apiEnvKeys) + .output(apiEnvLayoutResponse) + .query(async ({ input, ctx }) => { + await assertEnvAccess(ctx, input.scope, input.id, "read"); + return await listEnvVariableLayout(input); + }), applyChanges: protectedProcedure // tRPC reserves `apply` as a procedure name, so the REST route and its // operation id are pinned here to keep the HTTP API at POST /env.apply. diff --git a/apps/dokploy/server/api/utils/procedure-access.ts b/apps/dokploy/server/api/utils/procedure-access.ts index a6f31dcdd5c..2dfca49a448 100644 --- a/apps/dokploy/server/api/utils/procedure-access.ts +++ b/apps/dokploy/server/api/utils/procedure-access.ts @@ -278,6 +278,7 @@ export const PROCEDURE_ACCESS: Record = { "domainOverview.traefikLogs": "read", "env.applyChanges": "write", "env.keys": "read", + "env.layout": "read", "environment.byProjectId": "read", "environment.create": "write", "environment.duplicate": "write", diff --git a/packages/server/src/db/schema/env-variables.ts b/packages/server/src/db/schema/env-variables.ts index 41230ba25dd..6e6ead6bcfd 100644 --- a/packages/server/src/db/schema/env-variables.ts +++ b/packages/server/src/db/schema/env-variables.ts @@ -36,6 +36,39 @@ const envVariableName = z "Environment variable names must start with a letter or underscore and contain only letters, numbers, and underscores", ); +// dotenv also reads names with dots and dashes, so a layout has to be able to +// place them even though env-apply cannot create them. +const envLayoutName = z.string().regex(/^[\w.-]+$/); +const envLayoutCommentRef = z.string().regex(/^c[1-9][0-9]*$/); +const envLayoutHiddenRef = z.string().regex(/^h[1-9][0-9]*$/); +const envLayoutCommentText = z + .string() + .max(500) + .regex(/^[^\n\r\u2028\u2029]*$/, "A comment has to fit on one line"); + +export const envLayoutItem = z.discriminatedUnion("type", [ + z.object({ type: z.literal("variable"), name: z.string() }), + z.object({ type: z.literal("comment"), ref: z.string(), text: z.string() }), + z.object({ + type: z.literal("hidden"), + ref: z.string(), + kind: z.enum(["comment", "overridden", "unreadable"]), + variable: z.string().optional(), + }), + z.object({ type: z.literal("blank") }), +]); + +export const envLayoutInputItem = z.discriminatedUnion("type", [ + z.object({ type: z.literal("variable"), name: envLayoutName }), + z.object({ + type: z.literal("comment"), + ref: envLayoutCommentRef.optional(), + text: envLayoutCommentText.optional(), + }), + z.object({ type: z.literal("hidden"), ref: envLayoutHiddenRef }), + z.object({ type: z.literal("blank") }), +]); + export const isEnvFieldSupported = (scope: EnvScope, field: EnvField) => field === "env" || scope === "application"; @@ -72,6 +105,10 @@ export const apiEnvKeysResponse = z.object({ ), }); +export const apiEnvLayoutResponse = apiEnvKeysResponse.extend({ + layout: z.array(envLayoutItem), +}); + export const apiEnvApply = z .object({ scope: envScope, @@ -99,6 +136,7 @@ export const apiEnvApply = z }), ) .optional(), + layout: z.array(envLayoutInputItem).optional(), dryRun: z.boolean().optional(), expectedRevision: z.string().optional(), }) @@ -134,11 +172,42 @@ export const apiEnvApply = z } } - if (listsByName.size === 0) { + if (input.layout) { + // Refs and names of a layout describe one exact text, so it is applied + // alone and only to the revision it was read from. + if (listsByName.size > 0) { + ctx.addIssue({ + code: "custom", + path: ["layout"], + message: + "layout cannot be combined with set, remove, generate or copy", + }); + } + if (!input.expectedRevision) { + ctx.addIssue({ + code: "custom", + path: ["expectedRevision"], + message: "layout needs the expectedRevision it was read with", + }); + } + input.layout.forEach((item, index) => { + if ( + item.type === "comment" && + item.ref === undefined && + item.text === undefined + ) { + ctx.addIssue({ + code: "custom", + path: ["layout", index], + message: "A comment needs a ref, a text or both", + }); + } + }); + } else if (listsByName.size === 0) { ctx.addIssue({ code: "custom", message: - "At least one of set, remove, generate or copy must name a variable", + "At least one of set, remove, generate, copy or layout must be given", }); } @@ -168,4 +237,5 @@ export const apiEnvApplyResponse = z.object({ secret: z.boolean(), }), ), + layout: z.array(envLayoutItem).optional(), }); diff --git a/packages/server/src/services/env-variables.ts b/packages/server/src/services/env-variables.ts index 34a22f63e1b..5ffdf9e2275 100644 --- a/packages/server/src/services/env-variables.ts +++ b/packages/server/src/services/env-variables.ts @@ -22,6 +22,7 @@ import { TRPCError } from "@trpc/server"; import { eq } from "drizzle-orm"; import type { AnyPgColumn, PgTable } from "drizzle-orm/pg-core"; import type { z } from "zod"; +import { applyEnvLayout, readEnvLayout } from "../utils/env-layout"; import { applyEnvChanges, assertNoRedactedSecretPlaceholders, @@ -264,6 +265,18 @@ const readEnvText = async ( return text; }; +const writeEnvText = async ( + writer: Pick, + target: EnvTarget, + env: string, +) => { + const { table, idColumn } = ENV_TABLES[target.scope]; + await writer + .update(table) + .set({ [target.field]: env }) + .where(eq(idColumn, target.id)); +}; + const generateEnvValue = (length: number, format: EnvGenerateFormat) => { if (format === "alphanumeric") { return Array.from( @@ -287,6 +300,49 @@ export const listEnvVariableKeys = async ( }; }; +export const listEnvVariableLayout = async ( + input: z.infer, +) => { + const target = { scope: input.scope, id: input.id, field: input.field }; + const env = await readEnvText(db, target, false); + + let layout: ReturnType; + try { + layout = readEnvLayout(env); + } catch (error) { + throw new TRPCError({ + code: "PRECONDITION_FAILED", + message: error instanceof Error ? error.message : "Invalid env layout", + }); + } + + return { + ...target, + revision: getEnvTargetRevision(target, env), + variables: listEnvVariables(env), + layout, + }; +}; + +const applyLayoutChange = ( + currentEnv: string, + layout: NonNullable["layout"]>, +) => { + const texts = Object.fromEntries( + layout.flatMap((item, index) => + item.type === "comment" && item.text !== undefined + ? [[`layout[${index}]`, item.text]] + : [], + ), + ); + try { + assertNoRedactedSecretPlaceholders(texts); + return applyEnvLayout(currentEnv, layout); + } catch (error) { + throw badRequest(error); + } +}; + export const applyEnvVariableChanges = async ( input: z.infer, ) => { @@ -312,6 +368,22 @@ export const applyEnvVariableChanges = async ( }); } + if (input.layout) { + const result = applyLayoutChange(currentEnv, input.layout); + const summary = { + ...target, + changed: result.changed, + dryRun, + variables: [], + layout: result.layout, + }; + if (dryRun || !result.changed) { + return { ...summary, revision: currentRevision }; + } + await writeEnvText(tx, target, result.env); + return { ...summary, revision: getEnvTargetRevision(target, result.env) }; + } + const sourceKey = (source: EnvTarget) => `${source.scope}:${source.field}:${source.id}`; const sourceEnvs = new Map([[sourceKey(target), currentEnv]]); @@ -369,11 +441,7 @@ export const applyEnvVariableChanges = async ( return { ...summary, revision: currentRevision }; } - const { table, idColumn } = ENV_TABLES[target.scope]; - await tx - .update(table) - .set({ [target.field]: result.env }) - .where(eq(idColumn, target.id)); + await writeEnvText(tx, target, result.env); return { ...summary, revision: getEnvTargetRevision(target, result.env) }; }); diff --git a/packages/server/src/utils/env-layout.ts b/packages/server/src/utils/env-layout.ts new file mode 100644 index 00000000000..a09dab9dfd7 --- /dev/null +++ b/packages/server/src/utils/env-layout.ts @@ -0,0 +1,401 @@ +import { + type EnvEntry, + isEncodedDataFragment, + isSecretEnvName, + normalizeEnvText, + parseEnvValues, + scanEnvEntries, +} from "./env-upsert"; + +export type EnvLayoutHiddenKind = "comment" | "overridden" | "unreadable"; + +export type EnvLayoutItem = + | { type: "variable"; name: string } + | { type: "comment"; ref: string; text: string } + | { + type: "hidden"; + ref: string; + kind: EnvLayoutHiddenKind; + variable?: string; + } + | { type: "blank" }; + +export type EnvLayoutInputItem = + | { type: "variable"; name: string } + | { type: "comment"; ref?: string; text?: string } + | { type: "hidden"; ref: string } + | { type: "blank" }; + +export type EnvLayoutResult = { + env: string; + changed: boolean; + layout: EnvLayoutItem[]; +}; + +type EnvLayout = { + items: EnvLayoutItem[]; + variables: Map; + comments: Map; + hidden: Map; + earlierAssignments: Map; +}; + +type LineOwner = { entry: EnvEntry; first: number; last: number }; + +const COMMENT_LINE_REGEX = /^\s*#[^\S\n]?(.*)$/; +const BLANK_LINE_REGEX = /^\s*$/; +const COMMENTED_ASSIGNMENT_REGEX = /^\s*(?:export\s+)?([\w.-]+)\s*=/; +const COMMENTED_COLON_ASSIGNMENT_REGEX = + /^\s*(?:export\s+)?([A-Za-z_][\w.-]*)\s*:\s+\S/; +const URL_CREDENTIALS_REGEX = /[a-z][\w+.-]*:\/\/[^\s/@]+@/i; +const PEM_MARKER_REGEX = /-----(?:BEGIN|END)\b/; +const BEARER_REGEX = /\bbearer\s+\S/i; +const SECRET_WORD_VALUE_REGEX = + /(?:pass(?:word|wd)?|pwd|secret|token|key|credential|пароль|токен|ключ|секрет)\S*\s*[:=]\s*\S/iu; +const WORD_EDGE_PUNCTUATION_REGEX = /^[("'`<[{]+|[)"'`>\]},.;:!?]+$/g; + +const unsafeLayout = () => + new Error( + "The env text has a line structure that cannot be shown or rearranged safely", + ); + +const formatComment = (text: string) => (text === "" ? "#" : `# ${text}`); + +const looksLikeSecretWord = (word: string) => { + const bare = word.replace(WORD_EDGE_PUNCTUATION_REGEX, ""); + return ( + bare.length >= 40 || + (bare.length >= 16 && /[A-Za-z]/.test(bare) && /\d/.test(bare)) + ); +}; + +// Comments are shown so that sections and notes can be kept or rearranged, +// but people also park old values in them. Text that could hold a value is +// withheld: a commented-out assignment, credentials in a URL, a key block, +// a token-like word or a secret word followed by a value. +const describeComment = ( + text: string, +): { hidden: false } | { hidden: true; variable?: string } => { + const assignment = COMMENTED_ASSIGNMENT_REGEX.exec(text); + if (assignment) { + return { hidden: true, variable: assignment[1] }; + } + const colonAssignment = COMMENTED_COLON_ASSIGNMENT_REGEX.exec(text); + if ( + colonAssignment?.[1] && + (colonAssignment[1].includes("_") || isSecretEnvName(colonAssignment[1])) + ) { + return { hidden: true, variable: colonAssignment[1] }; + } + if ( + URL_CREDENTIALS_REGEX.test(text) || + PEM_MARKER_REGEX.test(text) || + BEARER_REGEX.test(text) || + SECRET_WORD_VALUE_REGEX.test(text) || + text.split(/\s+/).some(looksLikeSecretWord) + ) { + return { hidden: true }; + } + return { hidden: false }; +}; + +const splitEnvBody = (currentEnv: string | null | undefined) => { + const original = normalizeEnvText(currentEnv); + const trailingNewline = original.endsWith("\n"); + return { + original, + body: trailingNewline ? original.slice(0, -1) : original, + trailingNewline, + }; +}; + +const mapEntriesToLines = (body: string, lines: string[]) => { + const lineByOffset = new Map(); + let offset = 0; + lines.forEach((line, index) => { + lineByOffset.set(offset, index); + offset += line.length + 1; + }); + + const owners: (LineOwner | undefined)[] = new Array(lines.length); + for (const entry of scanEnvEntries(body)) { + const first = lineByOffset.get(entry.lineStart); + const endsText = entry.lineEnd >= body.length; + const next = endsText ? lines.length : lineByOffset.get(entry.lineEnd + 1); + // dotenv also ends a line at U+2028 and U+2029. An assignment that starts + // or ends at one of them shares a physical line with other text, and + // moving that line would move the other text with it. + if ( + first === undefined || + next === undefined || + (!endsText && body[entry.lineEnd] !== "\n") + ) { + throw unsafeLayout(); + } + const owner = { entry, first, last: next - 1 }; + for (let index = first; index < next; index++) { + if (owners[index]) { + throw unsafeLayout(); + } + owners[index] = owner; + } + } + return owners; +}; + +const scanEnvLayout = (body: string): EnvLayout => { + const lines = body.length > 0 ? body.split("\n") : []; + const owners = mapEntriesToLines(body, lines); + const values = parseEnvValues(body); + const isListed = (name: string) => + values.has(name) && !isEncodedDataFragment(name); + const isUnreadable = (index: number) => { + const owner = owners[index]; + if (owner) { + return !isListed(owner.entry.name); + } + const line = lines[index] ?? ""; + return !BLANK_LINE_REGEX.test(line) && !COMMENT_LINE_REGEX.test(line); + }; + const takeUnreadable = (from: number) => { + let index = from; + while (index < lines.length && isUnreadable(index)) { + index++; + } + return index; + }; + + type Segment = + | { kind: "variable"; name: string; text: string } + | { kind: "comment"; line: string; text: string } + | { + kind: "hidden"; + hiddenKind: EnvLayoutHiddenKind; + text: string; + variable?: string; + } + | { kind: "blank" }; + const segments: Segment[] = []; + let index = 0; + while (index < lines.length) { + const owner = owners[index]; + const line = lines[index] ?? ""; + if (owner && isListed(owner.entry.name)) { + // Lines dotenv cannot read right after an assignment are most often the + // rest of an unquoted value, such as a pasted key block, so they move + // with that variable. + const end = takeUnreadable(owner.last + 1); + segments.push({ + kind: "variable", + name: owner.entry.name, + text: lines.slice(index, end).join("\n"), + }); + index = end; + continue; + } + if (isUnreadable(index)) { + const end = takeUnreadable(index); + segments.push({ + kind: "hidden", + hiddenKind: "unreadable", + text: lines.slice(index, end).join("\n"), + }); + index = end; + continue; + } + const comment = COMMENT_LINE_REGEX.exec(line); + if (comment) { + const text = (comment[1] ?? "").trimEnd(); + const description = describeComment(text); + segments.push( + description.hidden + ? { + kind: "hidden", + hiddenKind: "comment", + text: line, + variable: description.variable, + } + : { kind: "comment", line, text }, + ); + } else { + segments.push({ kind: "blank" }); + } + index++; + } + + const lastSegmentByName = new Map(); + for (const segment of segments) { + if (segment.kind === "variable") { + lastSegmentByName.set(segment.name, segment); + } + } + + const layout: EnvLayout = { + items: [], + variables: new Map(), + comments: new Map(), + hidden: new Map(), + earlierAssignments: new Map(), + }; + for (const segment of segments) { + if (segment.kind === "variable") { + if (lastSegmentByName.get(segment.name) === segment) { + layout.items.push({ type: "variable", name: segment.name }); + layout.variables.set(segment.name, segment.text); + continue; + } + // An earlier assignment of a name that is assigned again has no effect + // but still holds a value. It is shown as a hidden line, so a layout + // sent back unchanged leaves the text as it is. + const ref = `h${layout.hidden.size + 1}`; + layout.items.push({ + type: "hidden", + ref, + kind: "overridden", + variable: segment.name, + }); + layout.hidden.set(ref, segment.text); + layout.earlierAssignments.set(ref, segment.name); + } else if (segment.kind === "comment") { + const ref = `c${layout.comments.size + 1}`; + layout.items.push({ type: "comment", ref, text: segment.text }); + layout.comments.set(ref, { line: segment.line, text: segment.text }); + } else if (segment.kind === "hidden") { + const ref = `h${layout.hidden.size + 1}`; + layout.items.push( + segment.variable === undefined + ? { type: "hidden", ref, kind: segment.hiddenKind } + : { + type: "hidden", + ref, + kind: segment.hiddenKind, + variable: segment.variable, + }, + ); + layout.hidden.set(ref, segment.text); + } else { + layout.items.push({ type: "blank" }); + } + } + return layout; +}; + +export const readEnvLayout = ( + currentEnv: string | null | undefined, +): EnvLayoutItem[] => scanEnvLayout(splitEnvBody(currentEnv).body).items; + +const listNames = (names: Iterable) => [...names].join(", "); + +export const applyEnvLayout = ( + currentEnv: string | null | undefined, + layout: EnvLayoutInputItem[], +): EnvLayoutResult => { + const { original, body, trailingNewline } = splitEnvBody(currentEnv); + const current = scanEnvLayout(body); + const unknownNames = new Set(); + const repeatedNames = new Set(); + const unknownRefs = new Set(); + const repeatedRefs = new Set(); + const lines: string[] = []; + const refPositions = new Map(); + const namePositions = new Map(); + + const placeRef = (ref: string, text: string | undefined) => { + if (text === undefined) { + unknownRefs.add(ref); + } else if (refPositions.has(ref)) { + repeatedRefs.add(ref); + } else { + refPositions.set(ref, lines.length); + lines.push(text); + } + }; + + for (const item of layout) { + if (item.type === "variable") { + const text = current.variables.get(item.name); + if (text === undefined) { + unknownNames.add(item.name); + } else if (namePositions.has(item.name)) { + repeatedNames.add(item.name); + } else { + namePositions.set(item.name, lines.length); + lines.push(text); + } + } else if (item.type === "comment") { + if (item.ref === undefined) { + lines.push(formatComment(item.text ?? "")); + continue; + } + const comment = current.comments.get(item.ref); + placeRef( + item.ref, + comment && + (item.text === undefined || item.text === comment.text + ? comment.line + : formatComment(item.text)), + ); + } else if (item.type === "hidden") { + placeRef(item.ref, current.hidden.get(item.ref)); + } else { + lines.push(""); + } + } + + const missingNames = [...current.variables.keys()].filter( + (name) => !namePositions.has(name), + ); + const missingHidden = [...current.hidden.keys()].filter( + (ref) => !refPositions.has(ref), + ); + const misplacedAssignments = [...current.earlierAssignments] + .filter( + ([ref, name]) => + (refPositions.get(ref) ?? -1) > + (namePositions.get(name) ?? Number.POSITIVE_INFINITY), + ) + .map(([ref, name]) => `${ref} (${name})`); + const problems = [ + unknownNames.size > 0 && + `The layout names variables that do not exist: ${listNames(unknownNames)}.`, + repeatedNames.size > 0 && + `The layout places variables more than once: ${listNames(repeatedNames)}.`, + missingNames.length > 0 && + `The layout leaves out variables: ${listNames(missingNames)}. Every variable has to be placed; delete one with remove instead.`, + unknownRefs.size > 0 && + `The layout refers to lines that do not exist: ${listNames(unknownRefs)}.`, + repeatedRefs.size > 0 && + `The layout places lines more than once: ${listNames(repeatedRefs)}.`, + missingHidden.length > 0 && + `The layout leaves out hidden lines: ${listNames(missingHidden)}. Their text is not shown, so they can be moved but not dropped.`, + misplacedAssignments.length > 0 && + `These hidden lines are earlier assignments of a variable and have to stay above it: ${listNames(misplacedAssignments)}.`, + ].filter((problem): problem is string => typeof problem === "string"); + if (problems.length > 0) { + throw new Error(problems.join(" ")); + } + + const nextBody = lines.join("\n"); + const nextEnv = + trailingNewline && nextBody.length > 0 ? `${nextBody}\n` : nextBody; + + // Moving whole lines keeps every value as long as dotenv splits the text the + // same way; a quote that pairs up with a different line would not. + const before = parseEnvValues(body); + const after = parseEnvValues(nextEnv); + const brokenNames = [...new Set([...before.keys(), ...after.keys()])].filter( + (name) => before.get(name) !== after.get(name), + ); + if (brokenNames.length > 0) { + throw new Error( + `Environment variables cannot be rearranged safely: ${listNames(brokenNames)}`, + ); + } + + const changed = nextEnv !== original; + return { + env: changed ? nextEnv : (currentEnv ?? ""), + changed, + layout: changed ? readEnvLayout(nextEnv) : current.items, + }; +}; diff --git a/packages/server/src/utils/env-upsert.ts b/packages/server/src/utils/env-upsert.ts index 9d482d18c05..ef38a2fbc03 100644 --- a/packages/server/src/utils/env-upsert.ts +++ b/packages/server/src/utils/env-upsert.ts @@ -220,7 +220,7 @@ export type EnvChangeResult = { const ENV_ENTRY_REGEX = /^(\s*)((?:export\s+)?)([\w.-]+)(\s*=\s*?|:\s+?)(\s*'(?:\\'|[^'])*'|\s*"(?:\\"|[^"])*"|\s*`(?:\\`|[^`])*`|[^#\r\n]+)?\s*(?:#.*)?$/gm; -type EnvEntry = { +export type EnvEntry = { name: string; lineStart: number; valueStart: number; @@ -228,12 +228,13 @@ type EnvEntry = { lineEnd: number; }; -const normalizeEnvText = (env: string | null | undefined) => +export const normalizeEnvText = (env: string | null | undefined) => (env ?? "").replace(/\r\n?/g, "\n"); -const parseEnvValues = (text: string) => new Map(Object.entries(parse(text))); +export const parseEnvValues = (text: string) => + new Map(Object.entries(parse(text))); -const scanEnvEntries = (text: string): EnvEntry[] => { +export const scanEnvEntries = (text: string): EnvEntry[] => { const lineBreak = /[\n\u2028\u2029]/g; return Array.from(text.matchAll(ENV_ENTRY_REGEX), (match) => { @@ -292,7 +293,7 @@ const serializeExactEnvValue = (name: string, value: string) => { // A key block pasted without quotes makes dotenv read its base64 lines that // end in `=` as assignments. Such a "name" is a piece of the secret, so a name // that looks like encoded data is left out of the listing. -const isEncodedDataFragment = (name: string) => +export const isEncodedDataFragment = (name: string) => name.length >= 20 && !name.includes("_") && /[a-z]/.test(name) &&