diff --git a/README.md b/README.md index b01e1c0..f824246 100644 --- a/README.md +++ b/README.md @@ -123,8 +123,8 @@ does not change when the provider or model does: | provider | key | timing | voice emotion | options | |---|---|---|---|---| | `elevenlabs` | `ELEVENLABS_API_KEY` | ElevenLabs', corrected against the audio | v3 audio tags | `emotionTags`, `seed` | - | `openai` | `OPENAI_API_KEY` | forced alignment | `instructions` (gpt-4o-mini-tts) | `emotionInstructions`, `speed` | - | `gemini` | `GEMINI_API_KEY` | forced alignment | none: only the text is sent | — | + | `openai` | `OPENAI_API_KEY` | forced alignment | `instructions` (gpt-4o-mini-tts) | `instructions`, `emotionInstructions`, `speed` | + | `gemini` | `GEMINI_API_KEY` | forced alignment | director's notes | `instructions`, `emotionInstructions` | | `mock` | — | exact, made up | — | — | - OpenAI and Gemini return audio only. Since the text is known, its timing is found by **forced @@ -135,12 +135,29 @@ does not change when the provider or model does: error 19 ms (one model frame is 20 ms); 79% (Japanese) and 91% (English) of characters within 50 ms. - The model (635 MB) is downloaded once from Hugging Face into `~/.cache/avatarscript/models` (or `AVATARSCRIPT_MODEL_DIR`). Audio and text never leave the machine for alignment. -- Speech that does not say the text — a provider reading extra words — is detected from the - alignment, requested once more, and otherwise reported as an error. (Gemini models read style - instructions aloud when they are written into the prompt, which is why only the text is sent.) +- **Speaking style** (openai, gemini): `instructions` describes the character for every line + ("You are an energetic 7-year-old boy. Use a high-pitched young child's voice."), and is followed + by a style for the line's emotion — a default, or your own from `emotionInstructions` + (`{ "happy": "…" }`). Gemini's prompt is always `### DIRECTOR'S NOTES` (the style) followed by + `#### TRANSCRIPT` (the text), a form Gemini follows without reading it aloud; text sent alone + fails on `gemini-2.5-flash-preview-tts` when it is a question. +- Speech that does not say the text — a provider reading extra words, such as its instructions — + is detected from the alignment, requested once more, and otherwise reported as an error. +- Gemini requests answered with 503 or 429 (an overloaded model) are retried three times, after + 1, 2 and 4 seconds. + +An avatar can name its default voice, model and provider options per provider in `avatar.json`, +so a character's speaking style travels with it: + +```json +"voice": { + "gemini": { "voice": "Puck", "options": { "instructions": "You are an energetic 7-year-old boy…" } }, + "openai": { "voice": "verse", "options": { "instructions": "…", "emotionInstructions": { "happy": "…" } } }, + "elevenlabs": { "voice": "", "model": "eleven_v3" } +} +``` -An avatar can name its default voice per provider in `avatar.json`: -`"voice": { "elevenlabs": { "voice": "", "model": "eleven_v3" } }`. +The CLI's `--voice` and `--model` override the avatar's; its options are used as they are. ## Scripts diff --git a/package-lock.json b/package-lock.json index 460d638..2b554a1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "avatarscript", - "version": "0.3.2", + "version": "0.4.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "avatarscript", - "version": "0.3.2", + "version": "0.4.0", "license": "MIT", "dependencies": { "kuromoji": "^0.1.2", diff --git a/package.json b/package.json index af9bd4f..29449b5 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "avatarscript", - "version": "0.3.2", + "version": "0.4.0", "description": "Turn an avatar and text into a lip-synced, acted video.", "keywords": [ "avatar", diff --git a/src/avatar.ts b/src/avatar.ts index 941f6ed..ec89a59 100644 --- a/src/avatar.ts +++ b/src/avatar.ts @@ -7,6 +7,7 @@ import { homedir } from "node:os"; import { basename, dirname, join, relative, resolve, sep } from "node:path"; import { z } from "zod"; import { parseJson } from "./json.ts"; +import { VoiceSettingsShape } from "./tts/types.ts"; export const AvatarManifestSchema = z.object({ format: z.literal("avatarscript-avatar/1"), @@ -15,8 +16,8 @@ export const AvatarManifestSchema = z.object({ /** folder the asset paths are relative to, relative to the manifest */ root: z.string().optional(), assets: z.object({ rig: z.string(), layers: z.string(), sprites: z.string().optional() }), - /** default voice and model per text-to-speech provider: { "elevenlabs": { "voice": "…", "model": "…" } } */ - voice: z.record(z.string(), z.object({ voice: z.string().optional(), model: z.string().optional() })).optional(), + /** default voice, model and options per text-to-speech provider: { "gemini": { "voice": "…", "options": { "instructions": "…" } } } */ + voice: z.record(z.string(), z.object(VoiceSettingsShape)).optional(), license: z.object({ attribution: z.string().optional(), terms: z.string().optional() }).optional(), }); export type AvatarManifest = z.infer; diff --git a/src/cli.ts b/src/cli.ts index 455ac39..a744cff 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -23,6 +23,7 @@ Speech (make, compile): GEMINI_API_KEY), or mock (offline buzz, no key) --voice the provider's voice (default: avatar.json, then a stock voice) --model the provider's model (default: avatar.json, then the provider's default) + (provider options, such as a speaking style, come from avatar.json) --lang language of the text (default: front matter, then detected) Video (make, render): @@ -90,9 +91,9 @@ const num = (name: "fps" | "seed", fallback: number) => { function textToSpeech(avatar: Avatar, cacheDir: string): TextToSpeech { const provider = need("tts"); if (!isTtsProvider(provider)) throw new Error(`unknown --tts "${provider}" (use ${TTS_PROVIDERS.join(" or ")})`); - // avatar.json can name a default voice and model per provider + // avatar.json can name a default voice, model and options (speaking style) per provider const preset = avatar.manifest.voice?.[provider] ?? {}; - return createTts({ provider, model: args.model ?? preset.model, voice: args.voice ?? preset.voice, cacheDir }); + return createTts({ provider, model: args.model ?? preset.model, voice: args.voice ?? preset.voice, options: preset.options, cacheDir }); } async function loadScript(path: string): Promise