diff --git a/AGENTS.md b/AGENTS.md index 0eb8262..9785d7c 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -138,7 +138,7 @@ Use `.env.example` as the source of truth. Never commit `.env.local`, API keys, - Text and Vision use `TEXT_*` and `VISION_*`; default protocol is `openai_compatible`, with native `anthropic` and `google` available via `TEXT_PROVIDER` / `VISION_PROVIDER`. - Image uses `IMAGE_*`; supported protocols are `runware`, `openai_compatible`, native `openai`, and native `google`. When `IMAGE_PROVIDER` is unset, Runware is inferred from `*.runware.ai` URLs and otherwise falls back to OpenAI-compatible image generations. -- TTS uses Xiaomi MiMo protocol and is optional: blank config means silent mode. +- TTS supports Xiaomi MiMo (voicedesign + voiceclone) or StepFun (preset voices auto-selected by keyword scoring), inferred from `TTS_BASE_URL` (host containing `stepfun.com` → StepFun, otherwise → MiMo). `CharacterVoice` is a discriminated union on `provider`; synth dispatches on the voice's own tag so a session may carry both shapes through a provider switch. Blank config means silent mode. - `MOCK_IMAGE=true` skips image generation and returns a placeholder for cheap local iteration. - `NEXT_PUBLIC_IMAGE_PROXY_URL` and `NEXT_PUBLIC_IMAGE_PROXY_ALLOWED_HOSTS` opt into browser-side image proxying for allowed hosts. - Analytics uses optional Umami `NEXT_PUBLIC_UMAMI_*` values and must stay content-free/privacy-preserving. diff --git a/app/api/beat-audio/route.ts b/app/api/beat-audio/route.ts index 7815d9b..fe881f2 100644 --- a/app/api/beat-audio/route.ts +++ b/app/api/beat-audio/route.ts @@ -13,9 +13,12 @@ export async function POST(req: Request) { return NextResponse.json({ error: "Invalid JSON" }, { status: 400 }); } - if (!body.beat?.id || !body.beat?.line || !body.voice?.referenceAudioBase64) { + // Accept either provider's voice shape — xiaomi carries referenceAudioBase64, + // stepfun carries voiceId. We only check the discriminator + the line text; + // shape-specific validation lives in each provider's synth function. + if (!body.beat?.id || !body.beat?.line || !body.voice?.provider) { return NextResponse.json( - { error: "beat.id, beat.line and voice.referenceAudioBase64 are required" }, + { error: "beat.id, beat.line and voice.provider are required" }, { status: 400 }, ); } diff --git a/lib/tts-client/index.ts b/lib/tts-client/index.ts index eb83fd2..c4c6515 100644 --- a/lib/tts-client/index.ts +++ b/lib/tts-client/index.ts @@ -1 +1,38 @@ -export { xiaomiProvision as provisionVoice, xiaomiSynthesize as synthesize } from "./xiaomi"; +import type { CharacterVoice, TtsConfig } from "@infiplot/types"; +import { stepfunProvision, stepfunSynthesize } from "./stepfun"; +import { xiaomiProvision, xiaomiSynthesize } from "./xiaomi"; + +// Provider auto-detection by base URL — mirrors the image client convention +// of inferring Runware from *.runware.ai and falling back otherwise. Keeps +// the BYO client flow unchanged: TTS_PROVIDER env var stays unused, and +// browser-side keys (Xiaomi only today) keep working through the xiaomi path. +function isStepfun(cfg: TtsConfig): boolean { + return /(^|[./])stepfun\.com\b/i.test(cfg.baseUrl); +} + +export async function provisionVoice( + cfg: TtsConfig, + description: string, +): Promise { + return isStepfun(cfg) + ? stepfunProvision(cfg, description) + : xiaomiProvision(cfg, description); +} + +// Dispatch by the voice's own provider tag, not by the current config. A +// session can outlive a provider switch (e.g. .env.local flip mid-game), and +// each voice must be synthesized via the protocol that minted it. The cfg +// still needs to point at the matching provider's endpoint; mismatch surfaces +// as a transparent network error, which `synthesizeBeat` already swallows. +export async function synthesize( + cfg: TtsConfig, + voice: CharacterVoice, + text: string, + delivery?: string, + signal?: AbortSignal, +): Promise<{ audioBase64: string; mimeType: string }> { + if (voice.provider === "stepfun") { + return stepfunSynthesize(cfg, voice, text, delivery, signal); + } + return xiaomiSynthesize(cfg, voice, text, delivery, signal); +} diff --git a/lib/tts-client/stepfun.ts b/lib/tts-client/stepfun.ts new file mode 100644 index 0000000..00238ac --- /dev/null +++ b/lib/tts-client/stepfun.ts @@ -0,0 +1,183 @@ +import type { CharacterVoice, TtsConfig } from "@infiplot/types"; + +// StepFun TTS uses an OpenAI-compatible /v1/audio/speech endpoint with PRESET +// voice IDs only — there is no "design a new voice from text description" +// equivalent to Xiaomi MiMo's voicedesign. We therefore translate the LLM's +// Chinese voiceDescription into a preset voice ID by keyword matching +// (gender + age + tone), with a deterministic hash-based spread across the +// top-N candidates so multiple similar characters don't collapse onto the +// same voice. Provision is a pure function — no network call needed. + +const OUTPUT_FORMAT = "mp3"; +const OUTPUT_MIME = "audio/mpeg"; + +type PresetVoice = { + id: string; + gender: "male" | "female"; + age: "teen" | "young" | "middle"; + /** Keywords (中文 or English) that, when present in the LLM's voice + * description, boost this preset's score. Drawn from StepFun's published + * voice name + recommended scenario. */ + tones: string[]; +}; + +// Full catalog from StepFun's docs (32 presets across step-tts-mini / +// step-tts-2 / stepaudio-2.5-tts). Adding more later is safe — the scorer +// degrades gracefully when an unknown id is picked. +const PRESET_VOICES: PresetVoice[] = [ + { id: "cixingnansheng", gender: "male", age: "young", tones: ["磁性", "成熟", "narrative"] }, + { id: "wenrounansheng", gender: "male", age: "young", tones: ["温柔", "gentle", "supportive"] }, + { id: "wenrougongzi", gender: "male", age: "young", tones: ["温柔", "公子", "tender"] }, + { id: "yuanqinansheng", gender: "male", age: "teen", tones: ["元气", "energetic", "阳光"] }, + { id: "zhengpaiqingnian", gender: "male", age: "young", tones: ["正派", "正气", "earnest"] }, + { id: "shuangkuainansheng", gender: "male", age: "young", tones: ["爽快", "干脆", "brisk"] }, + { id: "boyinnansheng", gender: "male", age: "middle", tones: ["播音", "broadcast", "稳重"] }, + { id: "ruyananshi", gender: "male", age: "middle", tones: ["儒雅", "斯文", "refined"] }, + { id: "shenchennanyin", gender: "male", age: "middle", tones: ["深沉", "低沉", "deep"] }, + { id: "qingniandaxuesheng", gender: "male", age: "young", tones: ["大学生", "青年", "student"] }, + { id: "zixinnansheng", gender: "male", age: "young", tones: ["自信", "confident"] }, + { id: "elegantgentle-female", gender: "female", age: "young", tones: ["气质", "温婉", "professional"] }, + { id: "livelybreezy-female", gender: "female", age: "teen", tones: ["活力", "轻快", "upbeat"] }, + { id: "jingdiannvsheng", gender: "female", age: "middle", tones: ["经典", "classic", "成熟"] }, + { id: "wenroushunv", gender: "female", age: "middle", tones: ["温柔", "熟女", "mature"] }, + { id: "tianmeinvsheng", gender: "female", age: "young", tones: ["甜美", "sweet"] }, + { id: "qingchunshaonv", gender: "female", age: "teen", tones: ["清纯", "少女", "pure"] }, + { id: "yuanqishaonv", gender: "female", age: "teen", tones: ["元气", "少女", "活力", "energetic"] }, + { id: "linjiajiejie", gender: "female", age: "young", tones: ["邻家", "姐姐"] }, + { id: "jilingshaonv", gender: "female", age: "teen", tones: ["机灵", "灵动", "少女"] }, + { id: "ruanmengnvsheng", gender: "female", age: "teen", tones: ["软萌", "可爱", "稚嫩", "甜软"] }, + { id: "youyanvsheng", gender: "female", age: "young", tones: ["优雅", "elegant"] }, + { id: "lengyanyujie", gender: "female", age: "middle", tones: ["冷艳", "御姐", "高冷"] }, + { id: "shuangkuaijiejie", gender: "female", age: "young", tones: ["爽快", "姐姐", "干脆"] }, + { id: "wenjingxuejie", gender: "female", age: "young", tones: ["文静", "学姐", "安静"] }, + { id: "linjiameimei", gender: "female", age: "teen", tones: ["邻家", "妹妹"] }, + { id: "zhixingjiejie", gender: "female", age: "young", tones: ["知性", "姐姐", "聪慧"] }, + { id: "ganliannvsheng", gender: "female", age: "middle", tones: ["干练", "sharp", "professional"] }, + { id: "qinhenvsheng", gender: "female", age: "young", tones: ["亲和", "warm", "亲切"] }, + { id: "huolinvsheng", gender: "female", age: "young", tones: ["活力", "lively", "活泼"] }, + { id: "qinqienvsheng", gender: "female", age: "middle", tones: ["亲切", "温暖"] }, + { id: "wenrounvsheng", gender: "female", age: "young", tones: ["温柔", "tender", "柔和"] }, +]; + +// Cheap deterministic 32-bit hash — used only to spread similar descriptions +// across the top-N candidate voices so two "温柔女声" characters don't collide. +function hashStr(s: string): number { + let h = 5381; + for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) | 0; + return Math.abs(h); +} + +function detectGender(desc: string): "male" | "female" { + // Female signals (broader cast — galgame skews toward female NPCs). + if (/女性|女声|少女|姐姐|妹妹|熟女|御姐|阿姨|奶奶|女孩|姑娘|大妈|女子|女生|女士|她|小姐/.test(desc)) { + return "female"; + } + if (/男性|男声|少年|青年|大叔|哥哥|弟弟|男人|男孩|大爷|爷爷|男子|男生|先生|他|公子|师傅/.test(desc)) { + return "male"; + } + // No strong signal: default female (matches the catalog's center of mass). + return "female"; +} + +function detectAge(desc: string): "teen" | "young" | "middle" { + if (/中年|熟女|大叔|大妈|阿姨|奶奶|爷爷|老师|师傅|御姐|经理|总监|教授|博士|总裁|长辈|父亲|母亲|爸爸|妈妈/.test(desc)) { + return "middle"; + } + if (/少女|少年|学生|高中|初中|妹妹|弟弟|小学|童年|稚嫩|十几岁|十六|十七|十八|未成年/.test(desc)) { + return "teen"; + } + return "young"; +} + +/** Map an LLM-written 中文 voice description to a StepFun preset voice ID. + * Pure function — exported for tests and for the synthesis-time sanity log. + */ +export function pickStepfunVoiceId(description: string, salt = ""): string { + const desc = description.toLowerCase(); + const gender = detectGender(desc); + const age = detectAge(desc); + + const scored = PRESET_VOICES + .filter((v) => v.gender === gender) + .map((v) => { + let score = 0; + if (v.age === age) score += 4; + for (const tone of v.tones) { + if (desc.includes(tone.toLowerCase())) score += 2; + } + return { v, score }; + }) + .sort((a, b) => b.score - a.score); + + // Catalog can't be filtered to zero; this guards against a future edit + // that prunes the table too aggressively. + if (scored.length === 0) return PRESET_VOICES[0]!.id; + + // Pick from the top 3 (or fewer) deterministically by hashing the + // description + an optional salt (charName) so two characters that share + // archetype keywords don't collapse onto the identical preset. + const top = scored.slice(0, Math.min(3, scored.length)); + const idx = hashStr(description + "|" + salt) % top.length; + return top[idx]!.v.id; +} + +// Provision is synchronous / no network — StepFun has no voicedesign equivalent. +// We mirror xiaomiProvision's async signature so the router stays uniform. +export async function stepfunProvision( + cfg: TtsConfig, + description: string, +): Promise { + const voiceId = pickStepfunVoiceId(description); + return { + provider: "stepfun", + voiceId, + model: cfg.speechModel, + mimeType: OUTPUT_MIME, + }; +} + +export async function stepfunSynthesize( + cfg: TtsConfig, + voice: CharacterVoice, + text: string, + _delivery?: string, + signal?: AbortSignal, +): Promise<{ audioBase64: string; mimeType: string }> { + if (voice.provider !== "stepfun") { + throw new Error( + `stepfunSynthesize received non-stepfun voice (provider="${voice.provider}")`, + ); + } + + // Strip trailing slash so /v1 + /audio/speech doesn't double up. + const base = cfg.baseUrl.replace(/\/$/, ""); + const url = `${base}/audio/speech`; + + const body = { + model: voice.model || cfg.speechModel, + input: text, + voice: voice.voiceId, + response_format: OUTPUT_FORMAT, + }; + + const res = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + Authorization: `Bearer ${cfg.apiKey}`, + }, + body: JSON.stringify(body), + signal, + }); + + if (!res.ok) { + const txt = await res.text(); + throw new Error(`StepFun TTS ${res.status}: ${txt.slice(0, 300)}`); + } + + const ab = await res.arrayBuffer(); + // Buffer is fine here — TTS routes run on runtime="nodejs". Falls back to + // btoa+chunks if we ever target Edge. + const audioBase64 = Buffer.from(ab).toString("base64"); + return { audioBase64, mimeType: OUTPUT_MIME }; +} diff --git a/lib/tts-client/xiaomi.ts b/lib/tts-client/xiaomi.ts index 6faa116..4708807 100644 --- a/lib/tts-client/xiaomi.ts +++ b/lib/tts-client/xiaomi.ts @@ -79,6 +79,11 @@ export async function xiaomiSynthesize( delivery?: string, signal?: AbortSignal, ): Promise<{ audioBase64: string; mimeType: string }> { + if (voice.provider !== "xiaomi") { + throw new Error( + `xiaomiSynthesize received non-xiaomi voice (provider="${voice.provider}")`, + ); + } const url = joinUrl(cfg.baseUrl, "/chat/completions"); // The free-form delivery direction rides in the `user` (director) message, diff --git a/lib/types/index.ts b/lib/types/index.ts index 47e81ab..c0a1989 100644 --- a/lib/types/index.ts +++ b/lib/types/index.ts @@ -160,12 +160,24 @@ export type WriterPlan = { // Characters & voices (TTS) // ────────────────────────────────────────────────────────────────────── -export type CharacterVoice = { - provider: "xiaomi"; - /** Xiaomi MiMo design output stored as reference audio for later clones. */ - referenceAudioBase64: string; - mimeType: string; -}; +export type CharacterVoice = + | { + provider: "xiaomi"; + /** Xiaomi MiMo design output stored as reference audio for later clones. */ + referenceAudioBase64: string; + mimeType: string; + } + | { + provider: "stepfun"; + /** StepFun preset voice ID (e.g. "cixingnansheng"). Selected by keyword + * matching against the LLM-written voiceDescription — no network call + * on provision (StepFun has no voicedesign endpoint), so this carries + * only the picked preset, not a clip. */ + voiceId: string; + /** TTS model used at synth time (step-tts-mini / step-tts-2 / stepaudio-2.5-tts). */ + model: string; + mimeType: string; + }; export type Character = { name: string;