From 30a04a445f3a321d7c3400240a01d45860e9176b Mon Sep 17 00:00:00 2001 From: Rishabh Date: Wed, 14 Oct 2026 14:47:22 -0700 Subject: [PATCH 1/2] fix(smtp): hot-reload renewed TLS certificates without restart The relay read its certificate once at startup, so renewals by Traefik/certbot were ignored and ports 465/587 kept serving the old certificate until it expired. Watch the mounted certificate files and swap renewed certificates into both servers via SMTPServer#updateSecureContext(). Co-Authored-By: Claude Opus 5.5 --- apps/smtp/src/server.ts | 54 +++++++++++++++++-- .../wiki/content/docs/self-hosting/docker.mdx | 2 + 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/apps/smtp/src/server.ts b/apps/smtp/src/server.ts index cbd0b7633..9a62f6f77 100644 --- a/apps/smtp/src/server.ts +++ b/apps/smtp/src/server.ts @@ -12,6 +12,7 @@ import { } from 'smtp-server'; import fs from 'fs'; import path from 'path'; +import tls from 'tls'; import {prisma} from './database/prisma.js'; @@ -24,6 +25,7 @@ const PORT_SECURE = parseInt(process.env.PORT_SECURE ?? '465', 10); const PORT_SUBMISSION = parseInt(process.env.PORT_SUBMISSION ?? '587', 10); const CERT_PATH = process.env.CERT_PATH ?? '/certs'; const ACME_JSON_PATH = process.env.ACME_JSON_PATH ?? path.join(CERT_PATH, 'acme.json'); +const CERT_WATCH_INTERVAL_MS = 60 * 1000; // Extended session interface to store authenticated project interface ExtendedSession extends SMTPServerSession { @@ -143,6 +145,41 @@ async function getCertificates(): Promise<{key: Buffer; cert: Buffer} | null> { } } +/** + * Watch the mounted certificate files and hot-swap renewed certificates into the running servers, + * so renewals by Traefik/certbot take effect without restarting the relay + */ +function watchCertificates(servers: SMTPServer[], initialCert: Buffer): void { + let currentCert = initialCert; + + const reload = () => { + const certs = loadFromTraefikAcme() ?? loadFromPemFiles(); + if (!certs || certs.cert.equals(currentCert)) { + return; + } + + try { + // updateSecureContext() drops the live context before building the new one, so an invalid pair + // (e.g. key rewritten but cert not yet) would leave the servers without TLS - validate it first + tls.createSecureContext({key: certs.key, cert: certs.cert}); + } catch (error) { + signale.warn('Skipping certificate reload, new key/certificate pair is invalid:', error); + return; + } + + for (const server of servers) { + server.updateSecureContext({key: certs.key, cert: certs.cert}); + } + + currentCert = certs.cert; + signale.success('Reloaded renewed TLS certificate'); + }; + + for (const file of [ACME_JSON_PATH, path.join(CERT_PATH, 'privkey.pem'), path.join(CERT_PATH, 'fullchain.pem')]) { + fs.watchFile(file, {interval: CERT_WATCH_INTERVAL_MS, persistent: false}, reload); + } +} + /** * Authentication handler * Validates API key against the database @@ -440,7 +477,7 @@ function handleData( /** * Create and start an SMTP server */ -function createSMTPServer(options: {secure: boolean; port: number; key?: Buffer; cert?: Buffer}): void { +function createSMTPServer(options: {secure: boolean; port: number; key?: Buffer; cert?: Buffer}): SMTPServer | null { const {secure, port, key, cert} = options; const serverOptions: SMTPServerOptions = { @@ -460,7 +497,7 @@ function createSMTPServer(options: {secure: boolean; port: number; key?: Buffer; serverOptions.secure = secure; } else if (secure) { signale.warn(`Cannot start secure server on port ${port} without TLS certificates`); - return; + return null; } const server = new SMTPServer(serverOptions); @@ -472,6 +509,8 @@ function createSMTPServer(options: {secure: boolean; port: number; key?: Buffer; server.listen(port, '0.0.0.0', () => { signale.success(`SMTP server listening on port ${port} (secure=${secure})`); }); + + return server; } /** @@ -530,8 +569,9 @@ async function main(): Promise { const certs = await getCertificates(); // Start secure SMTP (port 465) - only if TLS is available + let secureServer: SMTPServer | null = null; if (certs) { - createSMTPServer({ + secureServer = createSMTPServer({ secure: true, port: PORT_SECURE, key: certs.key, @@ -540,13 +580,19 @@ async function main(): Promise { } // Start submission SMTP (port 587) - STARTTLS if certs available, plaintext otherwise - createSMTPServer({ + const submissionServer = createSMTPServer({ secure: false, port: PORT_SUBMISSION, key: certs?.key, cert: certs?.cert, }); + // Pick up certificate renewals without a restart + if (certs) { + const servers = [secureServer, submissionServer].filter((server): server is SMTPServer => server !== null); + watchCertificates(servers, certs.cert); + } + signale.success('SMTP relay started successfully'); if (!certs) { diff --git a/apps/wiki/content/docs/self-hosting/docker.mdx b/apps/wiki/content/docs/self-hosting/docker.mdx index 547c4f276..1b474ad2a 100644 --- a/apps/wiki/content/docs/self-hosting/docker.mdx +++ b/apps/wiki/content/docs/self-hosting/docker.mdx @@ -106,6 +106,8 @@ volumes: If no certificates are mounted, SMTP runs without TLS. +Renewed certificates are picked up automatically within a minute of the mounted files changing, without restarting the container. Mount files that your ACME client updates in place, or mount the containing directory: a single-file bind mount of a file that gets replaced (such as certbot's `live/` symlinks) keeps pointing at the old certificate. + ## Building from Source ```bash From 873849cd5542061be840cd4b6073224a0e9fa49b Mon Sep 17 00:00:00 2001 From: Rishabh Date: Fri, 16 Oct 2026 22:42:34 -0700 Subject: [PATCH 2/2] fix(smtp): pick up certificates issued after startup and harden reloads Review follow-up for the certificate hot-reload: - Watch for certificates even when none were found at startup, and start the port 465 server when the first valid pair appears. - Reload only from the source that was active at startup (no fallback from a half-written acme.json to PEM files), watch only that source's files, and load quietly on reload. - Use a read only if no watched file changed from the change event until the read finished, at least 5s later, so a half-written file is never served. Coalesces the per-file change events into one reload. - If updateSecureContext() throws, put the previous pair back on every server instead of pre-validating the pair, and never let an error escape the watcher. - Docs: mount the PEM directory instead of single files; for certbot, copy the files from a deploy hook, since certbot's live/ and archive/ are root-only and the container runs as UID 1001. Co-Authored-By: Claude Opus 5.5 --- apps/smtp/README.md | 7 +- apps/smtp/src/server.ts | 239 +++++++++++++----- .../wiki/content/docs/self-hosting/docker.mdx | 20 +- docker-compose.yml | 6 +- 4 files changed, 197 insertions(+), 75 deletions(-) diff --git a/apps/smtp/README.md b/apps/smtp/README.md index 0de95b4b3..2f102654b 100644 --- a/apps/smtp/README.md +++ b/apps/smtp/README.md @@ -142,11 +142,14 @@ docker run -d \ -e API_URI=https://api.yourdomain.com \ -e DATABASE_URL=postgresql://... \ -e REDIS_URL=redis://... \ - -v /etc/letsencrypt/live/smtp.yourdomain.com/privkey.pem:/certs/privkey.pem:ro \ - -v /etc/letsencrypt/live/smtp.yourdomain.com/fullchain.pem:/certs/fullchain.pem:ro \ + -v /path/to/certs:/certs:ro \ plunk:latest ``` +Mount the directory holding `privkey.pem` and `fullchain.pem`, not the files: a single-file mount keeps the old file +when a renewal replaces it. For certbot, copy the files there from a deploy hook (see the self-hosting docs), since +certbot's `live/` files are symlinks into directories only root can read. + **Option 3: Without TLS (Development Only)** ```bash diff --git a/apps/smtp/src/server.ts b/apps/smtp/src/server.ts index 9a62f6f77..14c0936c6 100644 --- a/apps/smtp/src/server.ts +++ b/apps/smtp/src/server.ts @@ -12,7 +12,6 @@ import { } from 'smtp-server'; import fs from 'fs'; import path from 'path'; -import tls from 'tls'; import {prisma} from './database/prisma.js'; @@ -25,7 +24,11 @@ const PORT_SECURE = parseInt(process.env.PORT_SECURE ?? '465', 10); const PORT_SUBMISSION = parseInt(process.env.PORT_SUBMISSION ?? '587', 10); const CERT_PATH = process.env.CERT_PATH ?? '/certs'; const ACME_JSON_PATH = process.env.ACME_JSON_PATH ?? path.join(CERT_PATH, 'acme.json'); +const PEM_KEY_PATH = path.join(CERT_PATH, 'privkey.pem'); +const PEM_CERT_PATH = path.join(CERT_PATH, 'fullchain.pem'); const CERT_WATCH_INTERVAL_MS = 60 * 1000; +// How long certificate files must stay unchanged before a reload reads them +const CERT_SETTLE_MS = 5 * 1000; // Extended session interface to store authenticated project interface ExtendedSession extends SMTPServerSession { @@ -37,22 +40,44 @@ interface ExtendedSession extends SMTPServerSession { }; } +interface Certificates { + key: Buffer; + cert: Buffer; +} + +// Progress logging of the certificate loaders; reloads pass a silent one +type CertificateLog = Pick; + +interface CertificateSource { + name: string; + // Files the source reads, watched for renewals + files: string[]; + load: (log: CertificateLog) => Certificates | null; +} + +interface ResolvedCertificates { + source: CertificateSource; + certs: Certificates; +} + +const SILENT_LOG: CertificateLog = {info: () => {}, success: () => {}, warn: () => {}}; + /** * Load certificates from Traefik's acme.json file * Requires SMTP_DOMAIN to be set to select the correct certificate */ -function loadFromTraefikAcme(): {key: Buffer; cert: Buffer} | null { +function loadFromTraefikAcme(log: CertificateLog): Certificates | null { try { if (!fs.existsSync(ACME_JSON_PATH)) { return null; } if (!SMTP_DOMAIN) { - signale.warn('SMTP_DOMAIN not set, cannot load certificate from acme.json'); + log.warn('SMTP_DOMAIN not set, cannot load certificate from acme.json'); return null; } - signale.info(`Loading certificates from Traefik acme.json: ${ACME_JSON_PATH}`); + log.info(`Loading certificates from Traefik acme.json: ${ACME_JSON_PATH}`); const acmeData = JSON.parse(fs.readFileSync(ACME_JSON_PATH, 'utf8')); // Traefik acme.json structure: {letsencrypt: {Certificates: [{domain: {main: ...}, certificate: ..., key: ...}]}} @@ -70,7 +95,7 @@ function loadFromTraefikAcme(): {key: Buffer; cert: Buffer} | null { ); if (!certData) { - signale.warn(`Certificate for domain ${SMTP_DOMAIN} not found in acme.json`); + log.warn(`Certificate for domain ${SMTP_DOMAIN} not found in acme.json`); return null; } @@ -78,7 +103,7 @@ function loadFromTraefikAcme(): {key: Buffer; cert: Buffer} | null { const cert = Buffer.from(certData.certificate, 'base64'); const key = Buffer.from(certData.key, 'base64'); - signale.success(`Loaded certificate for ${SMTP_DOMAIN} from Traefik acme.json`); + log.success(`Loaded certificate for ${SMTP_DOMAIN} from Traefik acme.json`); return {key, cert}; } catch (error) { signale.error('Failed to load Traefik acme.json:', error); @@ -90,20 +115,17 @@ function loadFromTraefikAcme(): {key: Buffer; cert: Buffer} | null { * Load certificates from PEM files * Looks for standard filenames: privkey.pem and fullchain.pem */ -function loadFromPemFiles(): {key: Buffer; cert: Buffer} | null { +function loadFromPemFiles(log: CertificateLog): Certificates | null { try { - const keyPath = path.join(CERT_PATH, 'privkey.pem'); - const certPath = path.join(CERT_PATH, 'fullchain.pem'); - - if (!fs.existsSync(keyPath) || !fs.existsSync(certPath)) { + if (!fs.existsSync(PEM_KEY_PATH) || !fs.existsSync(PEM_CERT_PATH)) { return null; } - signale.info(`Loading certificates from ${keyPath} and ${certPath}`); - const key = fs.readFileSync(keyPath); - const cert = fs.readFileSync(certPath); + log.info(`Loading certificates from ${PEM_KEY_PATH} and ${PEM_CERT_PATH}`); + const key = fs.readFileSync(PEM_KEY_PATH); + const cert = fs.readFileSync(PEM_CERT_PATH); - signale.success('Loaded certificates from PEM files'); + log.success('Loaded certificates from PEM files'); return {key, cert}; } catch (error) { signale.error('Failed to load PEM files:', error); @@ -111,6 +133,26 @@ function loadFromPemFiles(): {key: Buffer; cert: Buffer} | null { } } +// Certificate sources in priority order +const CERTIFICATE_SOURCES: CertificateSource[] = [ + {name: 'Traefik acme.json', files: [ACME_JSON_PATH], load: loadFromTraefikAcme}, + {name: 'PEM files', files: [PEM_KEY_PATH, PEM_CERT_PATH], load: loadFromPemFiles}, +]; + +/** + * Load certificates from the first of the given sources that has them + */ +function resolveCertificates(sources: CertificateSource[], log: CertificateLog): ResolvedCertificates | null { + for (const source of sources) { + const certs = source.load(log); + if (certs) { + return {source, certs}; + } + } + + return null; +} + /** * Load TLS certificates * Priority: @@ -118,66 +160,109 @@ function loadFromPemFiles(): {key: Buffer; cert: Buffer} | null { * 2. Mounted PEM files * 3. No TLS */ -async function getCertificates(): Promise<{key: Buffer; cert: Buffer} | null> { - try { - signale.info('Checking for mounted certificates...'); +function getCertificates(): ResolvedCertificates | null { + signale.info('Checking for mounted certificates...'); - // Try loading from Traefik acme.json - let certs = loadFromTraefikAcme(); - if (certs) { - return certs; - } + const resolved = resolveCertificates(CERTIFICATE_SOURCES, signale); + if (resolved) { + return resolved; + } - // Try loading from PEM files - certs = loadFromPemFiles(); - if (certs) { - return certs; - } + signale.warn('No certificates found. SMTP server will run without TLS until certificates appear.'); + signale.info('To use TLS, mount certificates to one of:'); + signale.info(` 1. Traefik acme.json: ${ACME_JSON_PATH}`); + signale.info(` 2. PEM files: ${PEM_KEY_PATH} and ${PEM_CERT_PATH}`); + return null; +} - signale.warn('No certificates found. SMTP server will run without TLS.'); - signale.info('To use TLS, mount certificates to one of:'); - signale.info(` 1. Traefik acme.json: ${ACME_JSON_PATH}`); - signale.info(` 2. PEM files: ${CERT_PATH}/privkey.pem and ${CERT_PATH}/fullchain.pem`); - return null; - } catch (error) { - signale.error('Failed to load certificates:', error); - return null; - } +/** + * Fingerprint the files' identity, size and mtime, to tell whether any of them changed in between + */ +function snapshotFiles(files: string[]): string { + return files + .map(file => { + try { + const stats = fs.statSync(file); + return `${stats.ino}:${stats.size}:${stats.mtimeMs}`; + } catch { + return 'missing'; + } + }) + .join('|'); } /** - * Watch the mounted certificate files and hot-swap renewed certificates into the running servers, - * so renewals by Traefik/certbot take effect without restarting the relay + * Watch the certificate files and hand new certificates to `apply`, so certificates issued or renewed by + * Traefik/certbot after startup take effect without restarting the relay. Only the active source is watched + * and reloaded; until a certificate is found, every source is, and the first one to produce a certificate + * becomes the active source */ -function watchCertificates(servers: SMTPServer[], initialCert: Buffer): void { - let currentCert = initialCert; +function watchCertificates( + initial: ResolvedCertificates | null, + apply: (next: Certificates, previous: Certificates | null) => boolean, +): void { + let activeSource = initial?.source ?? null; + let appliedCerts = initial?.certs ?? null; + let watchedFiles: string[] = []; + let settleTimer: NodeJS.Timeout | undefined; + let settleSnapshot = ''; + + // A poll can land while a file is being written (Traefik rewrites acme.json in place), and a half-written + // fullchain.pem still parses and matches the key. So a read is used only if no file changed between the + // change event and the end of the read, at least CERT_SETTLE_MS later + const scheduleReload = (snapshot: string) => { + clearTimeout(settleTimer); + settleSnapshot = snapshot; + settleTimer = setTimeout(reload, CERT_SETTLE_MS); + }; const reload = () => { - const certs = loadFromTraefikAcme() ?? loadFromPemFiles(); - if (!certs || certs.cert.equals(currentCert)) { - return; - } - try { - // updateSecureContext() drops the live context before building the new one, so an invalid pair - // (e.g. key rewritten but cert not yet) would leave the servers without TLS - validate it first - tls.createSecureContext({key: certs.key, cert: certs.cert}); + const resolved = resolveCertificates(activeSource ? [activeSource] : CERTIFICATE_SOURCES, SILENT_LOG); + + const snapshot = snapshotFiles(watchedFiles); + if (snapshot !== settleSnapshot) { + return scheduleReload(snapshot); + } + + if (!resolved || appliedCerts?.cert.equals(resolved.certs.cert)) { + return; + } + + if (!apply(resolved.certs, appliedCerts)) { + return; + } + + appliedCerts = resolved.certs; + + if (activeSource) { + signale.success('Reloaded renewed TLS certificate'); + return; + } + + activeSource = resolved.source; + watch(activeSource.files); + signale.success(`Loaded TLS certificate from ${activeSource.name}, TLS is now enabled`); } catch (error) { - signale.warn('Skipping certificate reload, new key/certificate pair is invalid:', error); - return; + signale.error('Failed to reload TLS certificate:', error); + } + }; + + const onChange = () => scheduleReload(snapshotFiles(watchedFiles)); + + const watch = (files: string[]) => { + for (const file of watchedFiles) { + fs.unwatchFile(file, onChange); } - for (const server of servers) { - server.updateSecureContext({key: certs.key, cert: certs.cert}); + for (const file of files) { + fs.watchFile(file, {interval: CERT_WATCH_INTERVAL_MS, persistent: false}, onChange); } - currentCert = certs.cert; - signale.success('Reloaded renewed TLS certificate'); + watchedFiles = files; }; - for (const file of [ACME_JSON_PATH, path.join(CERT_PATH, 'privkey.pem'), path.join(CERT_PATH, 'fullchain.pem')]) { - fs.watchFile(file, {interval: CERT_WATCH_INTERVAL_MS, persistent: false}, reload); - } + watch(activeSource ? activeSource.files : CERTIFICATE_SOURCES.flatMap(source => source.files)); } /** @@ -566,9 +651,10 @@ async function main(): Promise { signale.info(`SMTP domain: ${SMTP_DOMAIN}`); } - const certs = await getCertificates(); + const initial = getCertificates(); + const certs = initial?.certs; - // Start secure SMTP (port 465) - only if TLS is available + // Start secure SMTP (port 465) - only if TLS is available, otherwise once a certificate appears let secureServer: SMTPServer | null = null; if (certs) { secureServer = createSMTPServer({ @@ -587,11 +673,34 @@ async function main(): Promise { cert: certs?.cert, }); - // Pick up certificate renewals without a restart - if (certs) { - const servers = [secureServer, submissionServer].filter((server): server is SMTPServer => server !== null); - watchCertificates(servers, certs.cert); - } + // Pick up certificates issued or renewed after startup without a restart + watchCertificates(initial, (next, previous) => { + const runningServers = [secureServer, submissionServer].filter((server): server is SMTPServer => server !== null); + + try { + // On the first certificate, creating the 465 server builds a context from the pair, so an invalid pair + // throws here, before any running server is touched + secureServer ??= createSMTPServer({secure: true, port: PORT_SECURE, key: next.key, cert: next.cert}); + + for (const server of runningServers) { + server.updateSecureContext({key: next.key, cert: next.cert}); + } + + return true; + } catch (error) { + signale.warn('Could not apply the new TLS certificate, keeping the previous one:', error); + + // updateSecureContext() drops a server's live context before building the new one, so put the previous + // pair back on every server rather than leaving some without TLS or on a different certificate + if (previous) { + for (const server of runningServers) { + server.updateSecureContext({key: previous.key, cert: previous.cert}); + } + } + + return false; + } + }); signale.success('SMTP relay started successfully'); diff --git a/apps/wiki/content/docs/self-hosting/docker.mdx b/apps/wiki/content/docs/self-hosting/docker.mdx index 1b474ad2a..97335384a 100644 --- a/apps/wiki/content/docs/self-hosting/docker.mdx +++ b/apps/wiki/content/docs/self-hosting/docker.mdx @@ -92,21 +92,31 @@ volumes: - /path/to/acme.json:/certs/acme.json:ro ``` -Plunk automatically extracts the certificate for `SMTP_DOMAIN` from acme.json. +Plunk automatically extracts the certificate for `SMTP_DOMAIN` from acme.json. Traefik rewrites `acme.json` in place, so mounting the single file picks up renewals. ### PEM Files -Mount certificate files directly: +Mount the directory that holds `privkey.pem` and `fullchain.pem`, not the files themselves: a single-file bind mount keeps pointing at the old file once a renewal replaces it. ```yaml volumes: - - /path/to/privkey.pem:/certs/privkey.pem:ro - - /path/to/fullchain.pem:/certs/fullchain.pem:ro + - /path/to/certs:/certs:ro ``` +### Certbot + +Certbot's `live//` files are relative symlinks into `archive/`, and certbot makes both directories readable by root only, while the Plunk container runs as UID 1001. Have certbot copy the files into a certificates directory after every issuance and renewal with a deploy hook, and mount that directory as in [PEM Files](#pem-files): + +```bash +certbot certonly -d smtp.yourdomain.com --deploy-hook \ + 'install -m 644 "$RENEWED_LINEAGE/fullchain.pem" /path/to/certs/ && install -m 600 -o 1001 "$RENEWED_LINEAGE/privkey.pem" /path/to/certs/' +``` + +For a certificate certbot already manages, add the hook with `certbot reconfigure --cert-name smtp.yourdomain.com --deploy-hook '...'`. + If no certificates are mounted, SMTP runs without TLS. -Renewed certificates are picked up automatically within a minute of the mounted files changing, without restarting the container. Mount files that your ACME client updates in place, or mount the containing directory: a single-file bind mount of a file that gets replaced (such as certbot's `live/` symlinks) keeps pointing at the old certificate. +Certificates issued or renewed while Plunk is running are picked up within about a minute, without restarting the container. ## Building from Source diff --git a/docker-compose.yml b/docker-compose.yml index 8fb80a0d6..3fbf2a9b4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -207,9 +207,9 @@ services: # Optional: Mount certificates for SMTP TLS # For Traefik acme.json: # - /path/to/acme.json:/certs/acme.json:ro - # For PEM files: - # - /path/to/privkey.pem:/certs/privkey.pem:ro - # - /path/to/fullchain.pem:/certs/fullchain.pem:ro + # For PEM files, mount the directory holding privkey.pem and fullchain.pem + # (a single-file mount keeps the old file when a renewal replaces it): + # - /path/to/certs:/certs:ro ports: # SMTP ports (for email relay)