fix(plugin-email): wire sys_email persistence + the template loader in init, so a boot-time verification send stops throwing TEMPLATE_NOT_FOUND

.changeset/23946-email-template-loader-boot-order.md

@@ -0,0 +1,9 @@
+---
+"@objectstack/plugin-email": patch
+---
+
+Clause-②: no
+
+fix(plugin-email): a template send issued before `EmailServicePlugin`'s own `kernel:ready` renders, persists its `sys_email` row and delivers, instead of throwing `TEMPLATE_NOT_FOUND: no templateLoader configured on EmailService`
+
+`EmailServicePlugin` now wires `sys_email` persistence and the registry template loader in `init`, before it registers the `email` service. Before this, both were wired in the plugin's own `kernel:ready` hook, and a plugin that starts earlier runs its `kernel:ready` hooks first. The dev admin seed (`objectstack dev --seed-admin`, the default) signs its account up from one of those hooks, so on a boot with email verification required the seed's verification mail failed. Each such boot logged `ERROR [Better Auth]: Failed to run background task: Error: TEMPLATE_NOT_FOUND …` and left no `sys_email` row. That send now goes through the transport and default sender the boot was configured with. The `mail` settings namespace is still applied at `kernel:ready`. Sends made after boot behave as before.

packages/plugins/plugin-email/src/email-plugin.outbox-sweep.test.ts

@@ -15,6 +15,8 @@
 import { describe, it, expect, vi, beforeEach } from 'vitest';
 import { assertEngineDeleteDispatch } from '@objectstack/objectql';
 import { DbQueueAdapter } from '@objectstack/service-queue';
+import { LiteKernel } from '@objectstack/core';
+import type { Plugin, PluginContext } from '@objectstack/core';
 import { EmailServicePlugin } from './email-plugin.js';
 import { EmailService, EMAIL_SEND_QUEUE } from './email-service.js';
 import { encodeAttachmentsForRow, encodeHeadersForRow } from './sys-email-payload.js';
@@ -493,3 +495,113 @@ describe('a datasource that re-keys a sys_email insert', () => {
     expect(h.sysEmail()[0]).toMatchObject({ id: 'db-pk-7', status: 'sent' });
   });
 });
+
+// ── a send issued before this plugin's own kernel:ready (#23946) ────────────
+
+/**
+ * Registers what the email plugin resolves at `init`, in ObjectQL's slot (its
+ * hard dependency): this file's engine as `objectql`, the manifest door, and a
+ * `protocol` whose layered `email_template` list serves exactly what that door
+ * was handed — the email plugin's own code-declared built-ins.
+ */
+class EngineFixturePlugin implements Plugin {
+  name = 'com.objectstack.engine.objectql';
+  type = 'standard' as const;
+  version = '1.0.0';
+  providesServices = ['objectql', 'manifest', 'protocol'];
+  readonly engine = fakeEngine();
+  private readonly manifests: Array<Record<string, any>> = [];
+  init(ctx: PluginContext): void {
+    ctx.registerService('objectql', this.engine);
+    ctx.registerService('manifest', { register: (m: Record<string, any>) => { this.manifests.push(m); } });
+    ctx.registerService('protocol', {
+      getMetaItemsForExecution: async ({ type }: { type: string }) => ({
+        items: type === 'email_template' ? this.manifests.flatMap((m) => m.emailTemplates ?? []) : [],
+      }),
+    });
+  }
+}
+
+/**
+ * Stands in for plugin-auth's dev admin seed: a plugin that depends on ObjectQL
+ * only (as `AuthPlugin` does), is registered ahead of the email plugin (as the
+ * served composition orders them), and from its own `kernel:ready` hook makes
+ * the verification send plugin-auth hands better-auth. A throw is caught and
+ * kept — better-auth's `runInBackgroundOrAwait` logs it and swallows it.
+ */
+class EarlySenderPlugin implements Plugin {
+  name = 'test.early-verification-sender';
+  type = 'standard' as const;
+  version = '1.0.0';
+  dependencies = ['com.objectstack.engine.objectql'];
+  readonly seen: { drainHooksAtSend?: number; result?: { id: string; status: string }; error?: unknown } = {};
+  constructor(private readonly engine: ReturnType<typeof fakeEngine>) {}
+  init(): void { /* nothing to register */ }
+  start(ctx: PluginContext): void {
+    ctx.hook('kernel:ready', async () => {
+      // The email plugin installs its outbox drain from ITS kernel:ready, so
+      // "none yet" is the measured proof that this send runs ahead of it.
+      this.seen.drainHooksAtSend = this.engine.afterInsertHooks().length;
+      try {
+        const email = ctx.getService<EmailService>('email');
+        this.seen.result = await email.sendTemplate({
+          template: 'auth.verify_email',
+          to: { address: 'admin@example.test', name: 'Dev Admin' },
+          data: {
+            user: { name: 'Dev Admin', email: 'admin@example.test', id: 'usr_seed' },
+            verificationUrl: 'https://example.test/verify?token=t1',
+            token: 't1',
+            expiresInMinutes: 60,
+            appName: 'Acme',
+          },
+          relatedObject: 'sys_user',
+          relatedId: 'usr_seed',
+        });
+      } catch (err) {
+        this.seen.error = err;
+      }
+    });
+  }
+}
+
+describe('a template send from an earlier-starting plugin\'s kernel:ready (#23946)', () => {
+  it('renders, persists its sys_email row and delivers once — no TEMPLATE_NOT_FOUND', async () => {
+    const transport = { send: vi.fn(async (_m: Record<string, any>) => ({ messageId: '<early@x>' })) };
+    const kernel = new LiteKernel({ logger: { level: 'error' } });
+    const fixture = new EngineFixturePlugin();
+    const sender = new EarlySenderPlugin(fixture.engine);
+    const plugin = new EmailServicePlugin({ transport, defaultFrom: { address: 'no-reply@example.test' } });
+    kernel.use(fixture);
+    kernel.use(sender);
+    kernel.use(plugin);
+    await kernel.bootstrap();
+    await plugin.outboxSweepSettled;
+    await flush();
+
+    // POSITIVE CONTROL — the send really ran before the email plugin's own
+    // kernel:ready wiring, and that wiring did run afterwards.
+    expect(sender.seen.drainHooksAtSend).toBe(0);
+    expect(fixture.engine.afterInsertHooks()).toHaveLength(1);
+
+    // THE PIN. Before the fix this was `TEMPLATE_NOT_FOUND: no templateLoader
+    // configured on EmailService`, and no row.
+    expect(sender.seen.error).toBeUndefined();
+    expect(sender.seen.result).toMatchObject({ status: 'sent' });
+    const rows = fixture.engine.rows('sys_email');
+    expect(rows).toHaveLength(1);
+    expect(rows[0]).toMatchObject({
+      id: sender.seen.result?.id,
+      status: 'sent',
+      message_id: '<early@x>',
+      subject: 'Verify your Acme email address',
+      to_addresses: 'Dev Admin <admin@example.test>',
+      related_object: 'sys_user',
+      related_id: 'usr_seed',
+    });
+    // Delivered once: the boot sweep that runs after it finds nothing stranded.
+    expect(transport.send).toHaveBeenCalledTimes(1);
+    expect(transport.send.mock.calls[0][0]).toMatchObject({ subject: 'Verify your Acme email address' });
+
+    await kernel.shutdown();
+  });
+});

packages/plugins/plugin-email/src/email-plugin.ts

@@ -249,17 +249,30 @@ function assertSendableDefaultFrom(from: EmailAddress | undefined): void {
   );
 }
 
+/**
+ * The data engine this plugin persists through and reads templates beside:
+ * `objectql`, else `data`, else `null`. Read, never recorded — a `null` here
+ * is not a verdict (see {@link EmailServicePlugin.init}).
+ */
+function resolveDataEngine(ctx: PluginContext): IDataEngine | null {
+  try { return ctx.getService<IDataEngine>('objectql') ?? null; }
+  catch {
+    try { return ctx.getService<IDataEngine>('data') ?? null; } catch { return null; }
+  }
+}
+
 /**
  * EmailServicePlugin — registers the `email` service.
  *
  * Lifecycle:
  *   - `init`: register sys_email + sys_email_template and the built-in auth
  *     templates (code-declared `email_template` items) via manifest;
  *     build transport (config → provider+apiKey → LogTransport fallback);
- *     register a transport-only EmailService so dependents can resolve it.
- *   - `start` (kernel:ready): wire ObjectQL-backed sys_email persistence
- *     + the registry TemplateLoader, and close the `sys_email_template`
- *     organization door.
+ *     wire ObjectQL-backed sys_email persistence + the registry
+ *     TemplateLoader; register the EmailService so dependents can resolve it.
+ *   - `start` (kernel:ready): bind the `mail` settings namespace, the
+ *     delivery mode, the outbox drain, the queue subscribers and the boot
+ *     sweep, and close the `sys_email_template` organization door.
  */
 export class EmailServicePlugin implements Plugin {
   name = 'com.objectstack.service.email';
@@ -301,6 +314,8 @@ export class EmailServicePlugin implements Plugin {
 
   private readonly options: EmailServicePluginOptions;
   private service?: EmailService;
+  /** The engine {@link wireEngine} installed persistence + the template loader over, and what it built. */
+  private engineWiring?: { engine: IDataEngine; persistence: EmailPersistence | undefined };
   /** Engine carrying the closed `sys_email_template` door — unbound in destroy(). */
   private boundEngine?: IDataEngine;
   /** SMTP transport currently in use, if any — closed in dispose(). */
@@ -383,25 +398,105 @@ export class EmailServicePlugin implements Plugin {
       );
     }
 
-    // Persistence + templateLoader are wired in `start` once the
-    // ObjectQL engine is available; here we register the service
-    // synchronously so dependents can resolve it.
     this.service = new EmailService({
       transport,
       defaultFrom: this.options.defaultFrom,
       retries: this.options.retries,
       defaultTemplateContext: this.options.defaultTemplateContext,
       logger: ctx.logger,
     });
+
+    // [#23946] Persistence + the template loader are wired HERE, before the
+    // service is handed out, and not at this plugin's `kernel:ready`. A
+    // plugin that STARTS before this one runs its own `kernel:ready` hooks
+    // first, and those can send mail: plugin-auth's dev admin seed signs its
+    // account up from one, and with verification armed better-auth awaits
+    // `sendVerificationEmail` → `sendTemplate` inside that hook. Wired at
+    // `kernel:ready`, that send met a service with no loader and threw
+    // `TEMPLATE_NOT_FOUND` into better-auth's background-task log, with no
+    // `sys_email` row. Holding the send until `kernel:ready` cannot work: the
+    // kernel runs `kernel:ready` handlers one after another, so this plugin's
+    // handler waits behind the very hook that is waiting on the send.
+    //
+    // Everything the two read is already there: ObjectQL is this plugin's
+    // hard dependency and registers `objectql` in its own `init`, the
+    // engine's registry is fixed at construction, and the loader resolves
+    // `protocol` at every load. An engine registered only later is wired at
+    // `kernel:ready` instead, so absence here records nothing.
+    const engine = resolveDataEngine(ctx);
+    if (engine) this.wireEngine(ctx, engine);
+
     ctx.registerService('email', this.service);
     ctx.logger.info('EmailServicePlugin: email service registered');
   }
 
+  /**
+   * Install ObjectQL-backed `sys_email` persistence and the registry
+   * {@link TemplateLoader} on the service, once per engine. Returns the
+   * persistence it installed (`undefined` under `persist: false`).
+   */
+  private wireEngine(ctx: PluginContext, engine: IDataEngine): EmailPersistence | undefined {
+    if (!this.service) return undefined;
+    if (this.engineWiring?.engine === engine) return this.engineWiring.persistence;
+
+    const persistence: EmailPersistence | undefined = this.options.persist === false
+      ? undefined
+      : {
+        async insert(row) {
+          const created = await (engine as any).insert('sys_email', row, {
+            context: SYSTEM_CTX,
+          });
+          // The engine's OWN answer is forwarded, not laundered into
+          // `row.id`. ObjectQL honours the id it is handed, so this confirms
+          // `row.id` — and on the day some driver re-keys the row instead,
+          // `EmailPersistence`'s insert contract (#5523) says so loudly here
+          // rather than letting the outbox drain hook double-send the
+          // message. `created` without a usable id means the engine reported
+          // no identity at all, which is not a disagreement: confirm the row
+          // we asked for.
+          return created?.id ? { id: String(created.id) } : { id: String(row.id) };
+        },
+        async update(id, patch) {
+          await (engine as any).update('sys_email', { id, ...patch }, {
+            context: SYSTEM_CTX,
+          });
+        },
+        // [#8149] The privileged readback for the `internal: true`
+        // `headers_json` column. Taken off the RAW engine on purpose: the
+        // dereference is a separately-named privileged verb (#8118)
+        // precisely so it cannot be reached from a query string, and the
+        // delivery paths above re-read rows through `find`, which omits the
+        // column for every caller including `SYSTEM_CTX`. Inert — and never
+        // calls the accessor — when the column is not flagged.
+        async readHeadersJson(rowIds) {
+          return readInternalHeadersJson(engine as any, rowIds);
+        },
+      };
+
+    // [ADR-0131 D6/D10] Templates resolve from the registry: the protocol's
+    // layered `email_template` list (environment over code) when a metadata
+    // door is mounted, the ObjectQL registry otherwise. Both are looked up
+    // at each load — resolve where it is used — so a Studio save is live on
+    // the next send, with no row, no projector and no restart.
+    const registry = (engine as { registry?: unknown }).registry;
+    const templateLoader: TemplateLoader = createRegistryEmailTemplateLoader(() => {
+      let protocol: unknown;
+      try { protocol = ctx.getService('protocol'); } catch { /* no metadata door on this host */ }
+      return { protocol, registry } as never;
+    });
+
+    // Mutate the existing service instance so consumers that already
+    // captured a reference (e.g. AuthManager) see the upgrade.
+    if (persistence) this.service.setPersistence(persistence);
+    this.service.setTemplateLoader(templateLoader);
+    this.engineWiring = { engine, persistence };
+    ctx.logger.info('EmailServicePlugin: sys_email persistence + template loader enabled');
+    return persistence;
+  }
+
   async start(ctx: PluginContext): Promise<void> {
     ctx.hook('kernel:ready', async () => {
-      let engine: IDataEngine | null = null;
-      try { engine = ctx.getService<IDataEngine>('objectql'); }
-      catch { try { engine = ctx.getService<IDataEngine>('data'); } catch { /* ignore */ } }
+      const engine = resolveDataEngine(ctx);
       if (!engine || !this.service) return;
 
       // ── Bind to the `mail` settings namespace (Phase 1) ──────────────
@@ -586,57 +681,11 @@ export class EmailServicePlugin implements Plugin {
         // settings service not registered — env/constructor opts remain authoritative.
       }
 
-      const persistence: EmailPersistence | undefined = this.options.persist === false
-        ? undefined
-        : {
-          async insert(row) {
-            const created = await (engine as any).insert('sys_email', row, {
-              context: SYSTEM_CTX,
-            });
-            // The engine's OWN answer is forwarded, not laundered into
-            // `row.id`. ObjectQL honours the id it is handed, so this confirms
-            // `row.id` — and on the day some driver re-keys the row instead,
-            // `EmailPersistence`'s insert contract (#5523) says so loudly here
-            // rather than letting the outbox drain hook double-send the
-            // message. `created` without a usable id means the engine reported
-            // no identity at all, which is not a disagreement: confirm the row
-            // we asked for.
-            return created?.id ? { id: String(created.id) } : { id: String(row.id) };
-          },
-          async update(id, patch) {
-            await (engine as any).update('sys_email', { id, ...patch }, {
-              context: SYSTEM_CTX,
-            });
-          },
-          // [#8149] The privileged readback for the `internal: true`
-          // `headers_json` column. Taken off the RAW engine on purpose: the
-          // dereference is a separately-named privileged verb (#8118)
-          // precisely so it cannot be reached from a query string, and the
-          // delivery paths above re-read rows through `find`, which omits the
-          // column for every caller including `SYSTEM_CTX`. Inert — and never
-          // calls the accessor — when the column is not flagged.
-          async readHeadersJson(rowIds) {
-            return readInternalHeadersJson(engine as any, rowIds);
-          },
-        };
-
-      // [ADR-0131 D6/D10] Templates resolve from the registry: the protocol's
-      // layered `email_template` list (environment over code) when a metadata
-      // door is mounted, the ObjectQL registry otherwise. Both are looked up
-      // at each load — resolve where it is used — so a Studio save is live on
-      // the next send, with no row, no projector and no restart.
-      const registry = (engine as { registry?: unknown }).registry;
-      const templateLoader: TemplateLoader = createRegistryEmailTemplateLoader(() => {
-        let protocol: unknown;
-        try { protocol = ctx.getService('protocol'); } catch { /* no metadata door on this host */ }
-        return { protocol, registry } as never;
-      });
-
-      // Mutate the existing service instance so consumers that already
-      // captured a reference (e.g. AuthManager) see the upgrade.
-      if (persistence) this.service.setPersistence(persistence);
-      this.service.setTemplateLoader(templateLoader);
-      ctx.logger.info('EmailServicePlugin: sys_email persistence + template loader enabled');
+      // Persistence + the template loader were wired in `init` (see there) on
+      // every boot whose engine was registered by then; this wires them only
+      // for a host that registered its engine later, and otherwise answers
+      // what `init` installed.
+      const persistence = this.wireEngine(ctx, engine);
 
       // Out-of-row attachment content (#5172). A thunk for the same reason the
       // queue is one: `storage` is a SwappableStorageService whose