feat(tts): StepFun voice selection via CharacterDesigner + provider-aware beat-audio
Make homepage cards and live sessions produce sound when the server is configured for StepFun TTS, instead of silently failing (the prebaked Xiaomi voice was useless on a StepFun server, and wasted ~220KB/beat in Fast Origin Transfer). Three coordinated changes: 1. CharacterDesigner now picks a StepFun preset voice id directly from the 32-entry catalog in the SAME LLM call that designs the character — zero extra latency, LLM-grade match quality. The Xiaomi prompt path is byte-identical to history (verified programmatically) so cache hit rate and voice quality are preserved. pickStepfunVoiceId (keyword scorer) remains the fallback for orphan speakers / invalid LLM picks. 2. The 32-preset catalog moves to lib/tts-client/stepfun-voices.json as the single source of truth, shared by the scorer, the CharacterDesigner prompt, /api/tts-provider, and the offline enrich script. 3. A new GET /api/tts-provider endpoint lets the client probe the server's TTS provider at /play mount. fetchBeatAudio then shapes its request body: on a StepFun server it sends the lightweight stepfunVoiceId / voiceDescription and omits the ~220KB Xiaomi reference audio (FOT saving ~13MB per protagonist per session on prebaked cards). requestBeatAudio re-provisions on a provider mismatch before synth, so audio never goes silent on a cross-provider replay or mid-session provider flip. New type fields are all optional and backward-compatible: Character.stepfunVoiceId, BeatAudioRequest.voiceDescription/characterName/stepfunVoiceId, voice made optional. AGENTS.md updated for the new route, type fields, dependency map, and StepFun voice-selection flow.
This commit is contained in:
yuanzonghao
+27
-9
@@ -1,15 +1,32 @@
|
||||
import type { CharacterVoice, TtsConfig } from "@infiplot/types";
|
||||
import { stepfunProvision, stepfunSynthesize } from "./stepfun";
|
||||
import type { CharacterVoice, TtsConfig, TtsProvider } from "@infiplot/types";
|
||||
import {
|
||||
formatStepfunCatalogForPrompt,
|
||||
isStepfun,
|
||||
isValidStepfunVoiceId,
|
||||
stepfunProvision,
|
||||
type StepfunProvisionOptions,
|
||||
stepfunSynthesize,
|
||||
} from "./stepfun";
|
||||
import { xiaomiProvision, xiaomiSynthesize } from "./xiaomi";
|
||||
|
||||
// Provider auto-detection by base URL — mirrors the image client convention
|
||||
// of inferring Runware from *.runware.ai and falling back otherwise. Keeps
|
||||
// the BYO client flow unchanged: TTS_PROVIDER env var stays unused, and
|
||||
// browser-side keys (Xiaomi only today) keep working through the xiaomi path.
|
||||
function isStepfun(cfg: TtsConfig): boolean {
|
||||
return /(^|[./])stepfun\.com\b/i.test(cfg.baseUrl);
|
||||
// Re-export so /api/tts-provider, orchestrator, CharacterDesigner prompt, and
|
||||
// the client all share ONE provider-detection rule + ONE catalog rendering +
|
||||
// ONE validity check with the synth path.
|
||||
export { isStepfun, isValidStepfunVoiceId, formatStepfunCatalogForPrompt };
|
||||
|
||||
/** Map a configured TtsConfig to its provider tag. Single source of truth for
|
||||
* the inference rule (host contains stepfun.com → stepfun, else xiaomi) so
|
||||
* /api/tts-provider and resolveVoice can't drift when a third provider is
|
||||
* added. A PRESENT TtsConfig always maps to a concrete provider — `null`
|
||||
* (no TTS configured) is the caller's responsibility to handle separately. */
|
||||
export function inferTtsProvider(cfg: TtsConfig): Exclude<TtsProvider, null> {
|
||||
return isStepfun(cfg) ? "stepfun" : "xiaomi";
|
||||
}
|
||||
|
||||
// `opts.stepfunVoiceId` threads the CharacterDesigner's LLM-selected preset
|
||||
// down to stepfunProvision. Xiaomi ignores it. See StepfunProvisionOptions.
|
||||
export type ProvisionVoiceOptions = StepfunProvisionOptions;
|
||||
|
||||
export async function provisionVoice(
|
||||
cfg: TtsConfig,
|
||||
description: string,
|
||||
@@ -18,9 +35,10 @@ export async function provisionVoice(
|
||||
// clip per call regardless. Threading it through keeps the API uniform
|
||||
// and prevents archetype collisions on the StepFun path.
|
||||
salt?: string,
|
||||
opts?: ProvisionVoiceOptions,
|
||||
): Promise<CharacterVoice> {
|
||||
return isStepfun(cfg)
|
||||
? stepfunProvision(cfg, description, salt)
|
||||
? stepfunProvision(cfg, description, salt, opts)
|
||||
: xiaomiProvision(cfg, description);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
[
|
||||
{ "id": "cixingnansheng", "gender": "male", "age": "young", "tones": ["磁性", "成熟", "narrative"], "desc": "磁性成熟男声，沉稳有厚度，适合旁白/叙事/解说" },
|
||||
{ "id": "wenrounansheng", "gender": "male", "age": "young", "tones": ["温柔", "gentle", "supportive"], "desc": "温柔男声，暖系治愈，适合陪伴/安抚/暖男主" },
|
||||
{ "id": "wenrougongzi", "gender": "male", "age": "young", "tones": ["温柔", "公子", "tender"], "desc": "温柔公子型男声，清润书卷气，适合古风公子/儒雅青年" },
|
||||
{ "id": "yuanqinansheng", "gender": "male", "age": "teen", "tones": ["元气", "energetic", "阳光"], "desc": "元气阳光少年男声，明亮有活力，适合少年/阳光系男主" },
|
||||
{ "id": "zhengpaiqingnian", "gender": "male", "age": "young", "tones": ["正派", "正气", "earnest"], "desc": "正派正气青年男声，端庄坚定，适合正剧男主/英雄" },
|
||||
{ "id": "shuangkuainansheng", "gender": "male", "age": "young", "tones": ["爽快", "干脆", "brisk"], "desc": "爽快干脆男声，利落不拖沓，适合热血/爽文男主" },
|
||||
{ "id": "boyinnansheng", "gender": "male", "age": "middle", "tones": ["播音", "broadcast", "稳重"], "desc": "播音腔稳重男声，字正腔圆，适合新闻/旁白/中年男主" },
|
||||
{ "id": "ruyananshi", "gender": "male", "age": "middle", "tones": ["儒雅", "斯文", "refined"], "desc": "儒雅斯文中年男声，文气内敛，适合学者/师者/儒雅男性" },
|
||||
{ "id": "shenchennanyin", "gender": "male", "age": "middle", "tones": ["深沉", "低沉", "deep"], "desc": "深沉低沉男声，厚重磁性，适合成熟/权威/反派男主" },
|
||||
{ "id": "qingniandaxuesheng", "gender": "male", "age": "young", "tones": ["大学生", "青年", "student"], "desc": "大学生青年男声，自然清爽，适合校园男主/学生" },
|
||||
{ "id": "zixinnansheng", "gender": "male", "age": "young", "tones": ["自信", "confident"], "desc": "自信青年男声，有底气不张扬，适合精英/自信男主" },
|
||||
{ "id": "elegantgentle-female", "gender": "female", "age": "young", "tones": ["气质", "温婉", "professional"], "desc": "气质温婉女声，得体大方，适合职业女性/气质女主" },
|
||||
{ "id": "livelybreezy-female", "gender": "female", "age": "teen", "tones": ["活力", "轻快", "upbeat"], "desc": "活力轻快少女声，明快有节奏，适合元气少女" },
|
||||
{ "id": "jingdiannvsheng", "gender": "female", "age": "middle", "tones": ["经典", "classic", "成熟"], "desc": "经典成熟女声，圆润端庄，适合旁白/成熟女性" },
|
||||
{ "id": "wenroushunv", "gender": "female", "age": "middle", "tones": ["温柔", "熟女", "mature"], "desc": "温柔熟女声，成熟柔润，适合熟女/姐姐型角色" },
|
||||
{ "id": "tianmeinvsheng", "gender": "female", "age": "young", "tones": ["甜美", "sweet"], "desc": "甜美女声，甜润可爱，适合甜系女主/甜妹" },
|
||||
{ "id": "qingchunshaonv", "gender": "female", "age": "teen", "tones": ["清纯", "少女", "pure"], "desc": "清纯少女声，干净清澈，适合清纯少女/初恋感" },
|
||||
{ "id": "yuanqishaonv", "gender": "female", "age": "teen", "tones": ["元气", "少女", "活力", "energetic"], "desc": "元气活力少女声，明亮张扬，适合元气少女/活泼女主" },
|
||||
{ "id": "linjiajiejie", "gender": "female", "age": "young", "tones": ["邻家", "姐姐"], "desc": "邻家姐姐声，亲切自然，适合邻家姐姐/青梅竹马" },
|
||||
{ "id": "jilingshaonv", "gender": "female", "age": "teen", "tones": ["机灵", "灵动", "少女"], "desc": "机灵灵动少女声，俏皮跳脱，适合机灵少女/鬼马角色" },
|
||||
{ "id": "ruanmengnvsheng", "gender": "female", "age": "teen", "tones": ["软萌", "可爱", "稚嫩", "甜软"], "desc": "软萌可爱稚嫩女声，甜软奶气，适合萝莉/软萌角色" },
|
||||
{ "id": "youyanvsheng", "gender": "female", "age": "young", "tones": ["优雅", "elegant"], "desc": "优雅女声，从容矜持，适合优雅/淑女型角色" },
|
||||
{ "id": "lengyanyujie", "gender": "female", "age": "middle", "tones": ["冷艳", "御姐", "高冷"], "desc": "冷艳御姐声，高冷有气场，适合御姐/女王/高冷女主" },
|
||||
{ "id": "shuangkuaijiejie", "gender": "female", "age": "young", "tones": ["爽快", "姐姐", "干脆"], "desc": "爽快干脆姐姐声，利落飒爽，适合飒爽女主/大姐大" },
|
||||
{ "id": "wenjingxuejie", "gender": "female", "age": "young", "tones": ["文静", "学姐", "安静"], "desc": "文静学姐声，安静内敛，适合文静/学姐/内向女主" },
|
||||
{ "id": "linjiameimei", "gender": "female", "age": "teen", "tones": ["邻家", "妹妹"], "desc": "邻家妹妹声，稚气天真，适合妹妹型/天真少女" },
|
||||
{ "id": "zhixingjiejie", "gender": "female", "age": "young", "tones": ["知性", "姐姐", "聪慧"], "desc": "知性聪慧姐姐声，沉稳理性，适合知性女性/学姐" },
|
||||
{ "id": "ganliannvsheng", "gender": "female", "age": "middle", "tones": ["干练", "sharp", "professional"], "desc": "干练职业女声，利落专业，适合职场女性/女强人" },
|
||||
{ "id": "qinhenvsheng", "gender": "female", "age": "young", "tones": ["亲和", "warm", "亲切"], "desc": "亲和温暖女声，亲切易接近，适合亲和型/治愈系女主" },
|
||||
{ "id": "huolinvsheng", "gender": "female", "age": "young", "tones": ["活力", "lively", "活泼"], "desc": "活力活泼女声，热情外放，适合活泼女主/开朗角色" },
|
||||
{ "id": "qinqienvsheng", "gender": "female", "age": "middle", "tones": ["亲切", "温暖"], "desc": "亲切温暖中年女声，温厚母性，适合阿姨/母亲/温暖长辈" },
|
||||
{ "id": "wenrounvsheng", "gender": "female", "age": "young", "tones": ["温柔", "tender", "柔和"], "desc": "温柔柔和女声，轻柔不张扬，适合温柔女主/治愈系" }
|
||||
]
|
||||
+79
-47
@@ -1,4 +1,28 @@
|
||||
import type { CharacterVoice, TtsConfig } from "@infiplot/types";
|
||||
import catalogData from "./stepfun-voices.json";
|
||||
|
||||
// Preset voice record. The 32 presets live in stepfun-voices.json (the single
|
||||
// source of truth — shared with the CharacterDesigner prompt, /api/tts-provider
|
||||
// validity check, and the offline enrich script). gender/age are discriminant
|
||||
// unions so detectGender / detectAge scoring stays type-safe.
|
||||
export type PresetVoice = {
|
||||
id: string;
|
||||
gender: "male" | "female";
|
||||
age: "teen" | "young" | "middle";
|
||||
/** Keywords (中文 or English) that, when present in the LLM's voice
|
||||
* description, boost this preset's score. Drawn from StepFun's published
|
||||
* voice name + recommended scenario. */
|
||||
tones: string[];
|
||||
/** 中文人设短语，供 LLM（设定师 prompt / enrich 脚本）在选音色时理解每个
|
||||
* 预设适合的角色类型。打分函数（pickStepfunVoiceId）仍只用 tones。 */
|
||||
desc: string;
|
||||
};
|
||||
|
||||
// JSON literals widen gender/age to `string`; cast back to the discriminant
|
||||
// unions. The catalog is a build-time-checked asset (touched rarely), and
|
||||
// pickStepfunVoiceId / isValidStepfunVoiceId tolerate anything we ship, so a
|
||||
// wrong entry surfaces as a bad voice pick rather than a crash.
|
||||
const PRESET_VOICES = catalogData as unknown as PresetVoice[];
|
||||
|
||||
// StepFun TTS uses an OpenAI-compatible /v1/audio/speech endpoint with PRESET
|
||||
// voice IDs only — there is no "design a new voice from text description"
|
||||
@@ -8,6 +32,14 @@ import type { CharacterVoice, TtsConfig } from "@infiplot/types";
|
||||
// top-N candidates so multiple similar characters don't collapse onto the
|
||||
// same voice. Provision is a pure function — no network call needed.
|
||||
|
||||
/** Provider detection — shared by /api/tts-provider, orchestrator fallback,
|
||||
* and the client (via the route). StepFun is inferred from a *.stepfun.com
|
||||
* host in the base URL, matching lib/tts-client/index.ts. Exported so every
|
||||
* caller agrees on the same rule. */
|
||||
export function isStepfun(cfg: TtsConfig): boolean {
|
||||
return /(^|[./])stepfun\.com\b/i.test(cfg.baseUrl);
|
||||
}
|
||||
|
||||
function arrayBufferToBase64(buffer: ArrayBuffer): string {
|
||||
const bytes = new Uint8Array(buffer);
|
||||
let binary = "";
|
||||
@@ -21,53 +53,37 @@ function arrayBufferToBase64(buffer: ArrayBuffer): string {
|
||||
const OUTPUT_FORMAT = "mp3";
|
||||
const OUTPUT_MIME = "audio/mpeg";
|
||||
|
||||
type PresetVoice = {
|
||||
id: string;
|
||||
gender: "male" | "female";
|
||||
age: "teen" | "young" | "middle";
|
||||
/** Keywords (中文 or English) that, when present in the LLM's voice
|
||||
* description, boost this preset's score. Drawn from StepFun's published
|
||||
* voice name + recommended scenario. */
|
||||
tones: string[];
|
||||
};
|
||||
|
||||
// Full catalog from StepFun's docs (32 presets across step-tts-mini /
|
||||
// step-tts-2 / stepaudio-2.5-tts). Adding more later is safe — the scorer
|
||||
// degrades gracefully when an unknown id is picked.
|
||||
const PRESET_VOICES: PresetVoice[] = [
|
||||
{ id: "cixingnansheng", gender: "male", age: "young", tones: ["磁性", "成熟", "narrative"] },
|
||||
{ id: "wenrounansheng", gender: "male", age: "young", tones: ["温柔", "gentle", "supportive"] },
|
||||
{ id: "wenrougongzi", gender: "male", age: "young", tones: ["温柔", "公子", "tender"] },
|
||||
{ id: "yuanqinansheng", gender: "male", age: "teen", tones: ["元气", "energetic", "阳光"] },
|
||||
{ id: "zhengpaiqingnian", gender: "male", age: "young", tones: ["正派", "正气", "earnest"] },
|
||||
{ id: "shuangkuainansheng", gender: "male", age: "young", tones: ["爽快", "干脆", "brisk"] },
|
||||
{ id: "boyinnansheng", gender: "male", age: "middle", tones: ["播音", "broadcast", "稳重"] },
|
||||
{ id: "ruyananshi", gender: "male", age: "middle", tones: ["儒雅", "斯文", "refined"] },
|
||||
{ id: "shenchennanyin", gender: "male", age: "middle", tones: ["深沉", "低沉", "deep"] },
|
||||
{ id: "qingniandaxuesheng", gender: "male", age: "young", tones: ["大学生", "青年", "student"] },
|
||||
{ id: "zixinnansheng", gender: "male", age: "young", tones: ["自信", "confident"] },
|
||||
{ id: "elegantgentle-female", gender: "female", age: "young", tones: ["气质", "温婉", "professional"] },
|
||||
{ id: "livelybreezy-female", gender: "female", age: "teen", tones: ["活力", "轻快", "upbeat"] },
|
||||
{ id: "jingdiannvsheng", gender: "female", age: "middle", tones: ["经典", "classic", "成熟"] },
|
||||
{ id: "wenroushunv", gender: "female", age: "middle", tones: ["温柔", "熟女", "mature"] },
|
||||
{ id: "tianmeinvsheng", gender: "female", age: "young", tones: ["甜美", "sweet"] },
|
||||
{ id: "qingchunshaonv", gender: "female", age: "teen", tones: ["清纯", "少女", "pure"] },
|
||||
{ id: "yuanqishaonv", gender: "female", age: "teen", tones: ["元气", "少女", "活力", "energetic"] },
|
||||
{ id: "linjiajiejie", gender: "female", age: "young", tones: ["邻家", "姐姐"] },
|
||||
{ id: "jilingshaonv", gender: "female", age: "teen", tones: ["机灵", "灵动", "少女"] },
|
||||
{ id: "ruanmengnvsheng", gender: "female", age: "teen", tones: ["软萌", "可爱", "稚嫩", "甜软"] },
|
||||
{ id: "youyanvsheng", gender: "female", age: "young", tones: ["优雅", "elegant"] },
|
||||
{ id: "lengyanyujie", gender: "female", age: "middle", tones: ["冷艳", "御姐", "高冷"] },
|
||||
{ id: "shuangkuaijiejie", gender: "female", age: "young", tones: ["爽快", "姐姐", "干脆"] },
|
||||
{ id: "wenjingxuejie", gender: "female", age: "young", tones: ["文静", "学姐", "安静"] },
|
||||
{ id: "linjiameimei", gender: "female", age: "teen", tones: ["邻家", "妹妹"] },
|
||||
{ id: "zhixingjiejie", gender: "female", age: "young", tones: ["知性", "姐姐", "聪慧"] },
|
||||
{ id: "ganliannvsheng", gender: "female", age: "middle", tones: ["干练", "sharp", "professional"] },
|
||||
{ id: "qinhenvsheng", gender: "female", age: "young", tones: ["亲和", "warm", "亲切"] },
|
||||
{ id: "huolinvsheng", gender: "female", age: "young", tones: ["活力", "lively", "活泼"] },
|
||||
{ id: "qinqienvsheng", gender: "female", age: "middle", tones: ["亲切", "温暖"] },
|
||||
{ id: "wenrounvsheng", gender: "female", age: "young", tones: ["温柔", "tender", "柔和"] },
|
||||
];
|
||||
// step-tts-2 / stepaudio-2.5-tts). The JSON is the single source of truth —
|
||||
// shared by the scorer here, the CharacterDesigner prompt (via
|
||||
// formatStepfunCatalogForPrompt), the /api/tts-provider route's validity
|
||||
// check, and the offline enrich script. Adding more later is safe — the
|
||||
// scorer degrades gracefully when an unknown id is picked.
|
||||
// (catalogData is cast to PresetVoice[] at the import above; kept as
|
||||
// PRESET_VOICES so existing references stay unchanged.)
|
||||
|
||||
/** All valid preset voice ids — for validation by the CharacterDesigner
|
||||
* (discard an out-of-catalog LLM pick) and the enrich script. */
|
||||
export const STEPFUN_PRESET_VOICE_IDS: string[] = PRESET_VOICES.map(
|
||||
(v) => v.id,
|
||||
);
|
||||
|
||||
const STEPFUN_ID_SET = new Set(STEPFUN_PRESET_VOICE_IDS);
|
||||
|
||||
/** True iff `id` is one of the 32 catalog presets. Used to drop LLM-hallucinated
|
||||
* ids before they reach StepFun (which would otherwise 4xx on synth). */
|
||||
export function isValidStepfunVoiceId(id: string | null | undefined): boolean {
|
||||
return !!id && STEPFUN_ID_SET.has(id);
|
||||
}
|
||||
|
||||
/** Render the catalog as a 中文 prompt-friendly list, one line per preset,
|
||||
* so the CharacterDesigner and the enrich script can ask the LLM to pick a
|
||||
* matching voice id. Each line: `id — desc（gender/age）`. */
|
||||
export function formatStepfunCatalogForPrompt(): string {
|
||||
return PRESET_VOICES.map(
|
||||
(v) => `- ${v.id}：${v.desc}（${v.gender}/${v.age}）`,
|
||||
).join("\n");
|
||||
}
|
||||
|
||||
// Cheap deterministic 32-bit hash — used only to spread similar descriptions
|
||||
// across the top-N candidate voices so two "温柔女声" characters don't collide.
|
||||
@@ -139,12 +155,28 @@ export function pickStepfunVoiceId(description: string, salt = ""): string {
|
||||
// We mirror xiaomiProvision's async signature so the router stays uniform.
|
||||
// The optional `salt` (character name) spreads two characters that share
|
||||
// archetype keywords across the top-N candidate presets.
|
||||
//
|
||||
// `opts.stepfunVoiceId` — when the CharacterDesigner already picked a preset
|
||||
// (it sees the same catalog via formatStepfunCatalogForPrompt), honor it if
|
||||
// valid; otherwise fall back to the keyword scorer. This keeps StepFun
|
||||
// provisioning a pure function (zero network cost) while lifting voice-id
|
||||
// selection quality to LLM-grade on the live path.
|
||||
export type StepfunProvisionOptions = {
|
||||
/** LLM-selected preset id from the CharacterDesigner; validated against the
|
||||
* catalog and ignored when out of range (hallucination guard). */
|
||||
stepfunVoiceId?: string;
|
||||
};
|
||||
|
||||
export async function stepfunProvision(
|
||||
cfg: TtsConfig,
|
||||
description: string,
|
||||
salt?: string,
|
||||
opts?: StepfunProvisionOptions,
|
||||
): Promise<CharacterVoice> {
|
||||
const voiceId = pickStepfunVoiceId(description, salt);
|
||||
const voiceId =
|
||||
opts && isValidStepfunVoiceId(opts.stepfunVoiceId)
|
||||
? opts.stepfunVoiceId!
|
||||
: pickStepfunVoiceId(description, salt);
|
||||
return {
|
||||
provider: "stepfun",
|
||||
voiceId,
|
||||
|
||||
Reference in New Issue
Block a user