feat(tts): add StepFun preset-voice provider, route by URL + voice tag

Add StepFun step-tts-mini / step-tts-2 / stepaudio-2.5-tts as an alternate
TTS provider alongside Xiaomi MiMo. Auto-detected from TTS_BASE_URL host
(contains `stepfun.com` → StepFun; otherwise → MiMo), mirroring how the
image client infers Runware from `*.runware.ai`.

CharacterVoice becomes a discriminated union on `provider`:
- xiaomi: { referenceAudioBase64, mimeType } — unchanged
- stepfun: { voiceId, model, mimeType } — preset voice ID + chosen model

Provision dispatches on the current cfg's base URL; synthesis dispatches
on the voice's own `provider` tag so a session with mixed voices (e.g. a
provider switch mid-development) routes each beat through the correct
protocol. xiaomiSynthesize now guards against being called with a non-
xiaomi voice, surfacing the bug as a clear runtime error instead of a
TypeScript narrow violation at the access site.

StepFun has no voicedesign equivalent — only preset voices + voice
cloning from a reference audio upload. Cloning would require an extra
asset per character, so v1 maps the LLM's Chinese voiceDescription to one
of the 32 published preset IDs via gender + age + tone keyword scoring,
with a deterministic hash spread across the top-3 candidates so multiple
characters with similar descriptions don't collapse onto the identical
preset. lineDelivery is accepted but not yet propagated to StepFun's
voice_label.emotion / .style fields — left as a follow-up.

beat-audio route validation relaxed from `voice.referenceAudioBase64`
(xiaomi-shaped) to `voice.provider` (shape-agnostic), so stepfun voices
pass the gate; provider-specific shape errors still surface from the
synth function.

Observed latency on InfiPlot's dev loop: StepFun step-tts-mini median
~2.3s per beat with 0% timeouts across the test session, vs MiMo's
median ~8s with the long tail tripping the existing 15s synth budget
on roughly 2 of 3 beats. Pricing: step-tts-mini ¥0.9/万字符 (~¥0.14
per typical 50-beat session) vs MiMo TTS currently free under the
Token Plan creator incentive.

AGENTS.md provider matrix updated to describe both providers and the
discriminated-union dispatch.

Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
DESKTOP-I1T6TF3\Q
2026-06-08 17:15:02 +08:00
parent 75548ce005
commit 19bbee16fe
6 changed files with 250 additions and 10 deletions
+38 -1
View File
@@ -1 +1,38 @@
export { xiaomiProvision as provisionVoice, xiaomiSynthesize as synthesize } from "./xiaomi";
import type { CharacterVoice, TtsConfig } from "@infiplot/types";
import { stepfunProvision, stepfunSynthesize } from "./stepfun";
import { xiaomiProvision, xiaomiSynthesize } from "./xiaomi";
// Provider auto-detection by base URL — mirrors the image client convention
// of inferring Runware from *.runware.ai and falling back otherwise. Keeps
// the BYO client flow unchanged: TTS_PROVIDER env var stays unused, and
// browser-side keys (Xiaomi only today) keep working through the xiaomi path.
function isStepfun(cfg: TtsConfig): boolean {
return /(^|[./])stepfun\.com\b/i.test(cfg.baseUrl);
}
export async function provisionVoice(
cfg: TtsConfig,
description: string,
): Promise<CharacterVoice> {
return isStepfun(cfg)
? stepfunProvision(cfg, description)
: xiaomiProvision(cfg, description);
}
// Dispatch by the voice's own provider tag, not by the current config. A
// session can outlive a provider switch (e.g. .env.local flip mid-game), and
// each voice must be synthesized via the protocol that minted it. The cfg
// still needs to point at the matching provider's endpoint; mismatch surfaces
// as a transparent network error, which `synthesizeBeat` already swallows.
export async function synthesize(
cfg: TtsConfig,
voice: CharacterVoice,
text: string,
delivery?: string,
signal?: AbortSignal,
): Promise<{ audioBase64: string; mimeType: string }> {
if (voice.provider === "stepfun") {
return stepfunSynthesize(cfg, voice, text, delivery, signal);
}
return xiaomiSynthesize(cfg, voice, text, delivery, signal);
}
+183
View File
@@ -0,0 +1,183 @@
import type { CharacterVoice, TtsConfig } from "@infiplot/types";
// StepFun TTS uses an OpenAI-compatible /v1/audio/speech endpoint with PRESET
// voice IDs only — there is no "design a new voice from text description"
// equivalent to Xiaomi MiMo's voicedesign. We therefore translate the LLM's
// Chinese voiceDescription into a preset voice ID by keyword matching
// (gender + age + tone), with a deterministic hash-based spread across the
// top-N candidates so multiple similar characters don't collapse onto the
// same voice. Provision is a pure function — no network call needed.
const OUTPUT_FORMAT = "mp3";
const OUTPUT_MIME = "audio/mpeg";
type PresetVoice = {
id: string;
gender: "male" | "female";
age: "teen" | "young" | "middle";
/** Keywords (中文 or English) that, when present in the LLM's voice
* description, boost this preset's score. Drawn from StepFun's published
* voice name + recommended scenario. */
tones: string[];
};
// Full catalog from StepFun's docs (32 presets across step-tts-mini /
// step-tts-2 / stepaudio-2.5-tts). Adding more later is safe — the scorer
// degrades gracefully when an unknown id is picked.
const PRESET_VOICES: PresetVoice[] = [
{ id: "cixingnansheng", gender: "male", age: "young", tones: ["磁性", "成熟", "narrative"] },
{ id: "wenrounansheng", gender: "male", age: "young", tones: ["温柔", "gentle", "supportive"] },
{ id: "wenrougongzi", gender: "male", age: "young", tones: ["温柔", "公子", "tender"] },
{ id: "yuanqinansheng", gender: "male", age: "teen", tones: ["元气", "energetic", "阳光"] },
{ id: "zhengpaiqingnian", gender: "male", age: "young", tones: ["正派", "正气", "earnest"] },
{ id: "shuangkuainansheng", gender: "male", age: "young", tones: ["爽快", "干脆", "brisk"] },
{ id: "boyinnansheng", gender: "male", age: "middle", tones: ["播音", "broadcast", "稳重"] },
{ id: "ruyananshi", gender: "male", age: "middle", tones: ["儒雅", "斯文", "refined"] },
{ id: "shenchennanyin", gender: "male", age: "middle", tones: ["深沉", "低沉", "deep"] },
{ id: "qingniandaxuesheng", gender: "male", age: "young", tones: ["大学生", "青年", "student"] },
{ id: "zixinnansheng", gender: "male", age: "young", tones: ["自信", "confident"] },
{ id: "elegantgentle-female", gender: "female", age: "young", tones: ["气质", "温婉", "professional"] },
{ id: "livelybreezy-female", gender: "female", age: "teen", tones: ["活力", "轻快", "upbeat"] },
{ id: "jingdiannvsheng", gender: "female", age: "middle", tones: ["经典", "classic", "成熟"] },
{ id: "wenroushunv", gender: "female", age: "middle", tones: ["温柔", "熟女", "mature"] },
{ id: "tianmeinvsheng", gender: "female", age: "young", tones: ["甜美", "sweet"] },
{ id: "qingchunshaonv", gender: "female", age: "teen", tones: ["清纯", "少女", "pure"] },
{ id: "yuanqishaonv", gender: "female", age: "teen", tones: ["元气", "少女", "活力", "energetic"] },
{ id: "linjiajiejie", gender: "female", age: "young", tones: ["邻家", "姐姐"] },
{ id: "jilingshaonv", gender: "female", age: "teen", tones: ["机灵", "灵动", "少女"] },
{ id: "ruanmengnvsheng", gender: "female", age: "teen", tones: ["软萌", "可爱", "稚嫩", "甜软"] },
{ id: "youyanvsheng", gender: "female", age: "young", tones: ["优雅", "elegant"] },
{ id: "lengyanyujie", gender: "female", age: "middle", tones: ["冷艳", "御姐", "高冷"] },
{ id: "shuangkuaijiejie", gender: "female", age: "young", tones: ["爽快", "姐姐", "干脆"] },
{ id: "wenjingxuejie", gender: "female", age: "young", tones: ["文静", "学姐", "安静"] },
{ id: "linjiameimei", gender: "female", age: "teen", tones: ["邻家", "妹妹"] },
{ id: "zhixingjiejie", gender: "female", age: "young", tones: ["知性", "姐姐", "聪慧"] },
{ id: "ganliannvsheng", gender: "female", age: "middle", tones: ["干练", "sharp", "professional"] },
{ id: "qinhenvsheng", gender: "female", age: "young", tones: ["亲和", "warm", "亲切"] },
{ id: "huolinvsheng", gender: "female", age: "young", tones: ["活力", "lively", "活泼"] },
{ id: "qinqienvsheng", gender: "female", age: "middle", tones: ["亲切", "温暖"] },
{ id: "wenrounvsheng", gender: "female", age: "young", tones: ["温柔", "tender", "柔和"] },
];
// Cheap deterministic 32-bit hash — used only to spread similar descriptions
// across the top-N candidate voices so two "温柔女声" characters don't collide.
function hashStr(s: string): number {
let h = 5381;
for (let i = 0; i < s.length; i++) h = ((h << 5) + h + s.charCodeAt(i)) | 0;
return Math.abs(h);
}
function detectGender(desc: string): "male" | "female" {
// Female signals (broader cast — galgame skews toward female NPCs).
if (/女性|女声|少女|姐姐|妹妹|熟女|御姐|阿姨|奶奶|女孩|姑娘|大妈|女子|女生|女士|她|小姐/.test(desc)) {
return "female";
}
if (/男性|男声|少年|青年|大叔|哥哥|弟弟|男人|男孩|大爷|爷爷|男子|男生|先生|他|公子|师傅/.test(desc)) {
return "male";
}
// No strong signal: default female (matches the catalog's center of mass).
return "female";
}
function detectAge(desc: string): "teen" | "young" | "middle" {
if (/中年|熟女|大叔|大妈|阿姨|奶奶|爷爷|老师|师傅|御姐|经理|总监|教授|博士|总裁|长辈|父亲|母亲|爸爸|妈妈/.test(desc)) {
return "middle";
}
if (/少女|少年|学生|高中|初中|妹妹|弟弟|小学|童年|稚嫩|十几岁|十六|十七|十八|未成年/.test(desc)) {
return "teen";
}
return "young";
}
/** Map an LLM-written 中文 voice description to a StepFun preset voice ID.
* Pure function — exported for tests and for the synthesis-time sanity log.
*/
export function pickStepfunVoiceId(description: string, salt = ""): string {
const desc = description.toLowerCase();
const gender = detectGender(desc);
const age = detectAge(desc);
const scored = PRESET_VOICES
.filter((v) => v.gender === gender)
.map((v) => {
let score = 0;
if (v.age === age) score += 4;
for (const tone of v.tones) {
if (desc.includes(tone.toLowerCase())) score += 2;
}
return { v, score };
})
.sort((a, b) => b.score - a.score);
// Catalog can't be filtered to zero; this guards against a future edit
// that prunes the table too aggressively.
if (scored.length === 0) return PRESET_VOICES[0]!.id;
// Pick from the top 3 (or fewer) deterministically by hashing the
// description + an optional salt (charName) so two characters that share
// archetype keywords don't collapse onto the identical preset.
const top = scored.slice(0, Math.min(3, scored.length));
const idx = hashStr(description + "|" + salt) % top.length;
return top[idx]!.v.id;
}
// Provision is synchronous / no network — StepFun has no voicedesign equivalent.
// We mirror xiaomiProvision's async signature so the router stays uniform.
export async function stepfunProvision(
cfg: TtsConfig,
description: string,
): Promise<CharacterVoice> {
const voiceId = pickStepfunVoiceId(description);
return {
provider: "stepfun",
voiceId,
model: cfg.speechModel,
mimeType: OUTPUT_MIME,
};
}
export async function stepfunSynthesize(
cfg: TtsConfig,
voice: CharacterVoice,
text: string,
_delivery?: string,
signal?: AbortSignal,
): Promise<{ audioBase64: string; mimeType: string }> {
if (voice.provider !== "stepfun") {
throw new Error(
`stepfunSynthesize received non-stepfun voice (provider="${voice.provider}")`,
);
}
// Strip trailing slash so /v1 + /audio/speech doesn't double up.
const base = cfg.baseUrl.replace(/\/$/, "");
const url = `${base}/audio/speech`;
const body = {
model: voice.model || cfg.speechModel,
input: text,
voice: voice.voiceId,
response_format: OUTPUT_FORMAT,
};
const res = await fetch(url, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${cfg.apiKey}`,
},
body: JSON.stringify(body),
signal,
});
if (!res.ok) {
const txt = await res.text();
throw new Error(`StepFun TTS ${res.status}: ${txt.slice(0, 300)}`);
}
const ab = await res.arrayBuffer();
// Buffer is fine here — TTS routes run on runtime="nodejs". Falls back to
// btoa+chunks if we ever target Edge.
const audioBase64 = Buffer.from(ab).toString("base64");
return { audioBase64, mimeType: OUTPUT_MIME };
}
+5
View File
@@ -79,6 +79,11 @@ export async function xiaomiSynthesize(
delivery?: string,
signal?: AbortSignal,
): Promise<{ audioBase64: string; mimeType: string }> {
if (voice.provider !== "xiaomi") {
throw new Error(
`xiaomiSynthesize received non-xiaomi voice (provider="${voice.provider}")`,
);
}
const url = joinUrl(cfg.baseUrl, "/chat/completions");
// The free-form delivery direction rides in the `user` (director) message,
+18 -6
View File
@@ -160,12 +160,24 @@ export type WriterPlan = {
// Characters & voices (TTS)
// ──────────────────────────────────────────────────────────────────────
export type CharacterVoice = {
provider: "xiaomi";
/** Xiaomi MiMo design output stored as reference audio for later clones. */
referenceAudioBase64: string;
mimeType: string;
};
export type CharacterVoice =
| {
provider: "xiaomi";
/** Xiaomi MiMo design output stored as reference audio for later clones. */
referenceAudioBase64: string;
mimeType: string;
}
| {
provider: "stepfun";
/** StepFun preset voice ID (e.g. "cixingnansheng"). Selected by keyword
* matching against the LLM-written voiceDescription — no network call
* on provision (StepFun has no voicedesign endpoint), so this carries
* only the picked preset, not a clip. */
voiceId: string;
/** TTS model used at synth time (step-tts-mini / step-tts-2 / stepaudio-2.5-tts). */
model: string;
mimeType: string;
};
export type Character = {
name: string;