0.4.0: speaking-style instructions for Gemini and OpenAI, carried by avatar.json

README.md

@@ -123,8 +123,8 @@ does not change when the provider or model does:
   | provider | key | timing | voice emotion | options |
   |---|---|---|---|---|
   | `elevenlabs` | `ELEVENLABS_API_KEY` | ElevenLabs', corrected against the audio | v3 audio tags | `emotionTags`, `seed` |
-  | `openai` | `OPENAI_API_KEY` | forced alignment | `instructions` (gpt-4o-mini-tts) | `emotionInstructions`, `speed` |
-  | `gemini` | `GEMINI_API_KEY` | forced alignment | none: only the text is sent | — |
+  | `openai` | `OPENAI_API_KEY` | forced alignment | `instructions` (gpt-4o-mini-tts) | `instructions`, `emotionInstructions`, `speed` |
+  | `gemini` | `GEMINI_API_KEY` | forced alignment | director's notes | `instructions`, `emotionInstructions` |
   | `mock` | — | exact, made up | — | — |
 
 - OpenAI and Gemini return audio only. Since the text is known, its timing is found by **forced
@@ -135,12 +135,29 @@ does not change when the provider or model does:
   error 19 ms (one model frame is 20 ms); 79% (Japanese) and 91% (English) of characters within 50 ms.
 - The model (635 MB) is downloaded once from Hugging Face into `~/.cache/avatarscript/models`
   (or `AVATARSCRIPT_MODEL_DIR`). Audio and text never leave the machine for alignment.
-- Speech that does not say the text — a provider reading extra words — is detected from the
-  alignment, requested once more, and otherwise reported as an error. (Gemini models read style
-  instructions aloud when they are written into the prompt, which is why only the text is sent.)
+- **Speaking style** (openai, gemini): `instructions` describes the character for every line
+  ("You are an energetic 7-year-old boy. Use a high-pitched young child's voice."), and is followed
+  by a style for the line's emotion — a default, or your own from `emotionInstructions`
+  (`{ "happy": "…" }`). Gemini's prompt is always `### DIRECTOR'S NOTES` (the style) followed by
+  `#### TRANSCRIPT` (the text), a form Gemini follows without reading it aloud; text sent alone
+  fails on `gemini-2.5-flash-preview-tts` when it is a question.
+- Speech that does not say the text — a provider reading extra words, such as its instructions —
+  is detected from the alignment, requested once more, and otherwise reported as an error.
+- Gemini requests answered with 503 or 429 (an overloaded model) are retried three times, after
+  1, 2 and 4 seconds.
+
+An avatar can name its default voice, model and provider options per provider in `avatar.json`,
+so a character's speaking style travels with it:
+
+```json
+"voice": {
+  "gemini": { "voice": "Puck", "options": { "instructions": "You are an energetic 7-year-old boy…" } },
+  "openai": { "voice": "verse", "options": { "instructions": "…", "emotionInstructions": { "happy": "…" } } },
+  "elevenlabs": { "voice": "<voice id>", "model": "eleven_v3" }
+}
+```
 
-An avatar can name its default voice per provider in `avatar.json`:
-`"voice": { "elevenlabs": { "voice": "<voice id>", "model": "eleven_v3" } }`.
+The CLI's `--voice` and `--model` override the avatar's; its options are used as they are.
 
 ## Scripts
 

package.json

@@ -1,6 +1,6 @@
 {
   "name": "avatarscript",
-  "version": "0.3.2",
+  "version": "0.4.0",
   "description": "Turn an avatar and text into a lip-synced, acted video.",
   "keywords": [
     "avatar",

src/avatar.ts

@@ -7,6 +7,7 @@ import { homedir } from "node:os";
 import { basename, dirname, join, relative, resolve, sep } from "node:path";
 import { z } from "zod";
 import { parseJson } from "./json.ts";
+import { VoiceSettingsShape } from "./tts/types.ts";
 
 export const AvatarManifestSchema = z.object({
   format: z.literal("avatarscript-avatar/1"),
@@ -15,8 +16,8 @@ export const AvatarManifestSchema = z.object({
   /** folder the asset paths are relative to, relative to the manifest */
   root: z.string().optional(),
   assets: z.object({ rig: z.string(), layers: z.string(), sprites: z.string().optional() }),
-  /** default voice and model per text-to-speech provider: { "elevenlabs": { "voice": "…", "model": "…" } } */
-  voice: z.record(z.string(), z.object({ voice: z.string().optional(), model: z.string().optional() })).optional(),
+  /** default voice, model and options per text-to-speech provider: { "gemini": { "voice": "…", "options": { "instructions": "…" } } } */
+  voice: z.record(z.string(), z.object(VoiceSettingsShape)).optional(),
   license: z.object({ attribution: z.string().optional(), terms: z.string().optional() }).optional(),
 });
 export type AvatarManifest = z.infer<typeof AvatarManifestSchema>;

src/cli.ts

@@ -23,6 +23,7 @@ Speech (make, compile):
                            GEMINI_API_KEY), or mock (offline buzz, no key)
   --voice <id>             the provider's voice (default: avatar.json, then a stock voice)
   --model <id>             the provider's model (default: avatar.json, then the provider's default)
+                           (provider options, such as a speaking style, come from avatar.json)
   --lang <code>            language of the text (default: front matter, then detected)
 
 Video (make, render):
@@ -90,9 +91,9 @@ const num = (name: "fps" | "seed", fallback: number) => {
 function textToSpeech(avatar: Avatar, cacheDir: string): TextToSpeech {
   const provider = need("tts");
   if (!isTtsProvider(provider)) throw new Error(`unknown --tts "${provider}" (use ${TTS_PROVIDERS.join(" or ")})`);
-  // avatar.json can name a default voice and model per provider
+  // avatar.json can name a default voice, model and options (speaking style) per provider
   const preset = avatar.manifest.voice?.[provider] ?? {};
-  return createTts({ provider, model: args.model ?? preset.model, voice: args.voice ?? preset.voice, cacheDir });
+  return createTts({ provider, model: args.model ?? preset.model, voice: args.voice ?? preset.voice, options: preset.options, cacheDir });
 }
 
 async function loadScript(path: string): Promise<Script> {

src/timing.ts

@@ -43,13 +43,23 @@ export function voicedFrames(samples: Float32Array, sampleRate: number): boolean
 
 /**
  * Ends each spoken character that comes before a space, punctuation or the end of the text where
- * the voice stops: an aligner tends to stretch the last sound of a phrase into the silence after it.
+ * the voice stops. An aligner tends to stretch the last sound of a phrase into the silence after
+ * it, and to cut a held one short (a long, sad "の？" sounds 0.5 s past its aligned end): the
+ * character is trimmed to the voice, or extended over voice that runs on from it without a pause,
+ * up to the next spoken character.
  */
 export function trimToVoice(chars: string[], timing: CharTime[], samples: Float32Array, sampleRate: number): CharTime[] {
   const loud = voicedFrames(samples, sampleRate);
   return timing.map((t, k) => {
     if (!isSpoken(chars[k]) || isSpoken(chars[k + 1] ?? " ")) return t;
-    let last = Math.min(loud.length, Math.ceil(t.end / FRAME)) - 1;
+    const endFrame = Math.min(loud.length, Math.ceil(t.end / FRAME));
+    if (endFrame > 0 && loud[endFrame - 1]) {
+      const next = timing.find((n, j) => j > k && isSpoken(chars[j]))?.start ?? Infinity;
+      let held = endFrame;
+      while (held < loud.length && loud[held] && (held + 1) * FRAME <= next) held++;
+      return { start: t.start, end: Math.max(t.end, held * FRAME) };
+    }
+    let last = endFrame - 1;
     while (last >= 0 && !loud[last] && (last + 1) * FRAME > t.start + MIN_VOICE) last--;
     return { start: t.start, end: Math.max(t.start + MIN_VOICE, Math.min(t.end, (last + 1) * FRAME)) };
   });

src/tts/gemini.ts

@@ -1,14 +1,32 @@
 // Gemini text-to-speech. It returns audio only; timing comes from forced alignment.
 // https://ai.google.dev/gemini-api/docs/speech-generation
 //
-// Only the text is sent. Gemini TTS models read a style instruction aloud when it is written into
-// the prompt ("Say cheerfully: …", or a "TRANSCRIPT:" preamble — both measured on
-// gemini-3.8-flash-tts) and reject systemInstruction, so emotion is shown on the face only.
+// The text is always sent after director's notes, MulmoCast's "### DIRECTOR'S NOTES" /
+// "#### TRANSCRIPT" form: the character from `instructions`, if any, and the style of the line's
+// emotion. It was never read aloud in 84 measured requests on gemini-2.5-flash-preview-tts and
+// gemini-3.8-flash-tts (2026-10-07). Text sent alone fails on gemini-2.5-flash-preview-tts when it
+// is a question ("Model tried to generate text": the model answers it). Other forms were read
+// aloud on gemini-3.8-flash-tts ("Say cheerfully: …", a bare "TRANSCRIPT:" preamble), and
+// systemInstruction is rejected. Speech that still includes the notes is caught by the mismatch
+// check in createTts().
 import { z } from "zod";
 import { fromWav, s16leToFloat } from "../audio.ts";
-import { parseJson } from "../json.ts";
+import { check, parseJson } from "../json.ts";
+import { StyleOptionsShape, styleInstruction } from "./style.ts";
 import type { NativeSpeech, ProviderDefinition, ProviderSettings, SpeechRequest } from "./types.ts";
 
+/** `options` for the gemini provider. */
+const OptionsSchema = z.strictObject(StyleOptionsShape);
+
+/** The prompt: the text after director's notes. */
+export function geminiPrompt(text: string, notes: string): string {
+  return `### DIRECTOR'S NOTES\n${notes}\n\n#### TRANSCRIPT\n${text}`;
+}
+
+/** Overload answers (gemini-3.8-flash-tts returns 503 now and then) are retried after these waits. */
+const RETRY_DELAYS_MS = [1000, 2000, 4000];
+const RETRY_STATUS = new Set([429, 503]);
+
 const ResponseSchema = z.object({
   candidates: z
     .array(z.object({ content: z.object({ parts: z.array(z.object({ inlineData: z.object({ mimeType: z.string(), data: z.string() }).optional() })) }) }))
@@ -24,16 +42,21 @@ export function decodeGeminiAudio(mimeType: string, data: Buffer): NativeSpeech
 }
 
 function synthesizer(settings: ProviderSettings): (req: SpeechRequest) => Promise<NativeSpeech> {
+  const options = check(OptionsSchema, settings.options, "gemini options");
+  const url = `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(settings.model)}:generateContent`;
   return async (req) => {
-    const url = `https://generativelanguage.googleapis.com/v1beta/models/${encodeURIComponent(settings.model)}:generateContent`;
-    const res = await fetch(url, {
-      method: "POST",
-      headers: { "x-goog-api-key": settings.apiKey, "content-type": "application/json" },
-      body: JSON.stringify({
-        contents: [{ parts: [{ text: req.text }] }],
-        generationConfig: { responseModalities: ["AUDIO"], speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: settings.voice } } } },
-      }),
+    const body = JSON.stringify({
+      contents: [{ parts: [{ text: geminiPrompt(req.text, styleInstruction(options, req.emotion)) }] }],
+      generationConfig: { responseModalities: ["AUDIO"], speechConfig: { voiceConfig: { prebuiltVoiceConfig: { voiceName: settings.voice } } } },
     });
+    const post = () => fetch(url, { method: "POST", headers: { "x-goog-api-key": settings.apiKey, "content-type": "application/json" }, body });
+    let res = await post();
+    for (const delay of RETRY_DELAYS_MS) {
+      if (!RETRY_STATUS.has(res.status)) break;
+      await res.body?.cancel();
+      await new Promise((done) => setTimeout(done, delay));
+      res = await post();
+    }
     if (!res.ok) throw new Error(`Gemini request failed (${res.status}): ${(await res.text()).slice(0, 500)}`);
     const json = parseJson(ResponseSchema, await res.text(), "Gemini response");
     const audio = json.candidates[0].content.parts.find((p) => p.inlineData)?.inlineData;

src/tts/index.ts

@@ -11,7 +11,15 @@ import { elevenLabsProvider } from "./elevenlabs.ts";
 import { geminiProvider } from "./gemini.ts";
 import { mockProvider } from "./mock.ts";
 import { openAiProvider } from "./openai.ts";
-import type { NativeSpeech, ProviderDefinition, Speech, SpeechRequest, TextToSpeech, TimingSource } from "./types.ts";
+import {
+  VoiceSettingsShape,
+  type NativeSpeech,
+  type ProviderDefinition,
+  type Speech,
+  type SpeechRequest,
+  type TextToSpeech,
+  type TimingSource,
+} from "./types.ts";
 
 export type { Speech, SpeechRequest, TextToSpeech } from "./types.ts";
 
@@ -36,14 +44,9 @@ const DEFINITIONS: Record<TtsProvider, ProviderDefinition> = {
 
 export const TtsConfigSchema = z.strictObject({
   provider: z.enum(TTS_PROVIDERS),
-  /** provider's model; default: the provider's recommended one */
-  model: z.string().min(1).optional(),
-  /** provider's voice id; default: one of the provider's stock voices */
-  voice: z.string().min(1).optional(),
+  ...VoiceSettingsShape,
   /** default: the provider's environment variable (ELEVENLABS_API_KEY, …) */
   apiKey: z.string().min(1).optional(),
-  /** provider-specific options, checked by the provider */
-  options: z.record(z.string(), z.unknown()).optional(),
   /** cache synthesized speech here, keyed by everything that changes it */
   cacheDir: z.string().min(1).optional(),
 });

src/tts/openai.ts

@@ -3,36 +3,26 @@
 import { z } from "zod";
 import { s16leToFloat } from "../audio.ts";
 import { check } from "../json.ts";
-import { EMOTIONS, type Emotion } from "../script.ts";
+import { StyleOptionsShape, styleInstruction } from "./style.ts";
 import type { NativeSpeech, ProviderDefinition, ProviderSettings, SpeechRequest } from "./types.ts";
 
 const SAMPLE_RATE = 24000; // response_format "pcm": 24 kHz 16-bit mono
 
 /** `options` for the openai provider. */
 const OptionsSchema = z.strictObject({
-  /** emotion → spoken-style instruction (models that take instructions: gpt-4o-mini-tts and later) */
-  emotionInstructions: z.partialRecord(z.enum(EMOTIONS), z.string()).optional(),
+  /** `instructions` and `emotionInstructions`: used by models that take instructions (gpt-4o-mini-tts and later) */
+  ...StyleOptionsShape,
   /** 0.25–4.0 */
   speed: z.number().min(0.25).max(4).optional(),
 });
 
-const DEFAULT_INSTRUCTIONS: Record<Emotion, string> = {
-  neutral: "Speak naturally and clearly.",
-  happy: "Speak warmly and cheerfully.",
-  sad: "Speak softly, with a sad tone.",
-  angry: "Speak with an irritated, angry tone.",
-  surprised: "Speak with surprise and excitement.",
-  relaxed: "Speak in a calm, relaxed way.",
-};
-
 function synthesizer(settings: ProviderSettings): (req: SpeechRequest) => Promise<NativeSpeech> {
   const options = check(OptionsSchema, settings.options, "openai options");
-  const instructions = { ...DEFAULT_INSTRUCTIONS, ...options.emotionInstructions };
   // tts-1 / tts-1-hd take no instructions
   const takesInstructions = !settings.model.startsWith("tts-1");
   return async (req) => {
     const body: Record<string, unknown> = { model: settings.model, voice: settings.voice, input: req.text, response_format: "pcm" };
-    if (takesInstructions) body.instructions = instructions[req.emotion];
+    if (takesInstructions) body.instructions = styleInstruction(options, req.emotion);
     if (options.speed !== undefined) body.speed = options.speed;
     const res = await fetch("https://api.openai.com/v1/audio/speech", {
       method: "POST",

src/tts/style.ts

@@ -0,0 +1,28 @@
+// Speaking-style instructions for providers that take them in words (OpenAI, Gemini): a character
+// instruction that stays the same for an avatar, combined with a style for the current emotion.
+import { z } from "zod";
+import { EMOTIONS, type Emotion } from "../script.ts";
+
+/** The `options` fields every instruction-taking provider accepts; each adds its own. */
+export const StyleOptionsShape = {
+  /** who is speaking and how, for every line: "You are an energetic 7-year-old boy…" */
+  instructions: z.string().min(1).optional(),
+  /** emotion → spoken-style instruction, replacing the default for that emotion */
+  emotionInstructions: z.partialRecord(z.enum(EMOTIONS), z.string()).optional(),
+};
+type StyleOptions = z.infer<z.ZodObject<typeof StyleOptionsShape>>;
+
+const DEFAULT_EMOTION_STYLES: Record<Emotion, string> = {
+  neutral: "Speak naturally and clearly.",
+  happy: "Speak warmly and cheerfully.",
+  sad: "Speak softly, with a sad tone.",
+  angry: "Speak with an irritated, angry tone.",
+  surprised: "Speak with surprise and excitement.",
+  relaxed: "Speak in a calm, relaxed way.",
+};
+
+/** The character instruction, if any, followed by the style for `emotion`. */
+export function styleInstruction(options: StyleOptions, emotion: Emotion): string {
+  const style = options.emotionInstructions?.[emotion] ?? DEFAULT_EMOTION_STYLES[emotion];
+  return options.instructions ? `${options.instructions}\n${style}` : style;
+}

src/tts/types.ts

@@ -1,6 +1,17 @@
+import { z } from "zod";
 import type { CharTime } from "../g2p/types.ts";
 import type { Emotion } from "../script.ts";
 
+/** How one provider speaks: in createTts()'s config, and per provider in avatar.json's `voice`. */
+export const VoiceSettingsShape = {
+  /** provider's model; default: the provider's recommended one */
+  model: z.string().min(1).optional(),
+  /** provider's voice id; default: one of the provider's stock voices */
+  voice: z.string().min(1).optional(),
+  /** provider-specific options, checked by the provider */
+  options: z.record(z.string(), z.unknown()).optional(),
+};
+
 /** One piece of text to speak. */
 export interface SpeechRequest {
   /** text to speak; the timing comes back with one entry per character of it */

tests/avatar.test.ts

@@ -4,7 +4,8 @@ import type { AddressInfo } from "node:net";
 import { tmpdir } from "node:os";
 import { join, resolve } from "node:path";
 import { afterAll, beforeAll, describe, expect, it } from "vitest";
-import { loadAvatar } from "../src/avatar.ts";
+import { AvatarManifestSchema, loadAvatar } from "../src/avatar.ts";
+import { check } from "../src/json.ts";
 
 // Serves avatars/ani over HTTP, with per-test overrides, and counts requests per path.
 const ANI = resolve(import.meta.dirname, "../avatars/ani");
@@ -144,3 +145,16 @@ describe("loadAvatar", () => {
     await expect(loadAvatar("ftp://example.com/ani/", { cacheDir })).rejects.toThrow("only http and https");
   });
 });
+
+describe("avatar.json voice", () => {
+  const manifest = (voice: unknown) => ({ format: "avatarscript-avatar/1", name: "x", runtime: "mesh-avatar/1", assets: { rig: "r", layers: "l" }, voice });
+
+  it("carries provider options, such as a speaking style, with the voice", () => {
+    const voice = { gemini: { voice: "Puck", model: "gemini-2.5-flash-preview-tts", options: { instructions: "You are a boy." } } };
+    expect(check(AvatarManifestSchema, manifest(voice), "avatar.json").voice).toEqual(voice);
+  });
+
+  it("rejects options that are not an object", () => {
+    expect(() => check(AvatarManifestSchema, manifest({ gemini: { options: "loud" } }), "avatar.json")).toThrow("avatar.json is not valid");
+  });
+});

tests/timing.test.ts

@@ -73,6 +73,27 @@ describe("refineTiming", () => {
     expect(out[4].end).toBeCloseTo(1.1, 2);
   });
 
+  it("extends a held sound over the voice that runs on from it, up to the next character", () => {
+    // the aligner cut "b" and "d" short, though the voice holds them to 0.50 and 1.10
+    const timing = [
+      { start: 0.1, end: 0.2 },
+      { start: 0.2, end: 0.3 },
+      { start: 0.3, end: 0.45 },
+      { start: 0.45, end: 0.8 },
+      { start: 0.8, end: 0.9 },
+    ];
+    const out = trimToVoice([..."ab、cd"], timing, audio, RATE);
+    expect(out[1].end).toBeCloseTo(0.45, 2); // stops at "c", which starts inside the voice
+    expect(out[4].end).toBeCloseTo(1.1, 2);
+    expect(out[0]).toEqual(timing[0]);
+  });
+
+  it("does not carry a held sound across a pause", () => {
+    // "a" ends inside the first span; the voice after the pause is not part of it
+    const out = trimToVoice(["a"], [{ start: 0.1, end: 0.3 }], audio, RATE);
+    expect(out[0].end).toBeCloseTo(0.5, 2);
+  });
+
   it("leaves timing alone for silent audio", () => {
     const timing = [{ start: 0, end: 0.1 }];
     expect(refineTiming(["a"], timing, new Float32Array(800), RATE)).toBe(timing);