feat(tts): StepFun voice selection via CharacterDesigner + provider-aware beat-audio
Make homepage cards and live sessions produce sound when the server is configured for StepFun TTS, instead of silently failing (the prebaked Xiaomi voice was useless on a StepFun server, and wasted ~220KB/beat in Fast Origin Transfer). Three coordinated changes: 1. CharacterDesigner now picks a StepFun preset voice id directly from the 32-entry catalog in the SAME LLM call that designs the character — zero extra latency, LLM-grade match quality. The Xiaomi prompt path is byte-identical to history (verified programmatically) so cache hit rate and voice quality are preserved. pickStepfunVoiceId (keyword scorer) remains the fallback for orphan speakers / invalid LLM picks. 2. The 32-preset catalog moves to lib/tts-client/stepfun-voices.json as the single source of truth, shared by the scorer, the CharacterDesigner prompt, /api/tts-provider, and the offline enrich script. 3. A new GET /api/tts-provider endpoint lets the client probe the server's TTS provider at /play mount. fetchBeatAudio then shapes its request body: on a StepFun server it sends the lightweight stepfunVoiceId / voiceDescription and omits the ~220KB Xiaomi reference audio (FOT saving ~13MB per protagonist per session on prebaked cards). requestBeatAudio re-provisions on a provider mismatch before synth, so audio never goes silent on a cross-provider replay or mid-session provider flip. New type fields are all optional and backward-compatible: Character.stepfunVoiceId, BeatAudioRequest.voiceDescription/characterName/stepfunVoiceId, voice made optional. AGENTS.md updated for the new route, type fields, dependency map, and StepFun voice-selection flow.
This commit is contained in:
yuanzonghao
+79
-47
@@ -1,4 +1,28 @@
|
||||
import type { CharacterVoice, TtsConfig } from "@infiplot/types";
|
||||
import catalogData from "./stepfun-voices.json";
|
||||
|
||||
// Preset voice record. The 32 presets live in stepfun-voices.json (the single
|
||||
// source of truth — shared with the CharacterDesigner prompt, /api/tts-provider
|
||||
// validity check, and the offline enrich script). gender/age are discriminant
|
||||
// unions so detectGender / detectAge scoring stays type-safe.
|
||||
export type PresetVoice = {
|
||||
id: string;
|
||||
gender: "male" | "female";
|
||||
age: "teen" | "young" | "middle";
|
||||
/** Keywords (中文 or English) that, when present in the LLM's voice
|
||||
* description, boost this preset's score. Drawn from StepFun's published
|
||||
* voice name + recommended scenario. */
|
||||
tones: string[];
|
||||
/** 中文人设短语，供 LLM（设定师 prompt / enrich 脚本）在选音色时理解每个
|
||||
* 预设适合的角色类型。打分函数（pickStepfunVoiceId）仍只用 tones。 */
|
||||
desc: string;
|
||||
};
|
||||
|
||||
// JSON literals widen gender/age to `string`; cast back to the discriminant
|
||||
// unions. The catalog is a build-time-checked asset (touched rarely), and
|
||||
// pickStepfunVoiceId / isValidStepfunVoiceId tolerate anything we ship, so a
|
||||
// wrong entry surfaces as a bad voice pick rather than a crash.
|
||||
const PRESET_VOICES = catalogData as unknown as PresetVoice[];
|
||||
|
||||
// StepFun TTS uses an OpenAI-compatible /v1/audio/speech endpoint with PRESET
|
||||
// voice IDs only — there is no "design a new voice from text description"
|
||||
@@ -8,6 +32,14 @@ import type { CharacterVoice, TtsConfig } from "@infiplot/types";
|
||||
// top-N candidates so multiple similar characters don't collapse onto the
|
||||
// same voice. Provision is a pure function — no network call needed.
|
||||
|
||||
/** Provider detection — shared by /api/tts-provider, orchestrator fallback,
|
||||
* and the client (via the route). StepFun is inferred from a *.stepfun.com
|
||||
* host in the base URL, matching lib/tts-client/index.ts. Exported so every
|
||||
* caller agrees on the same rule. */
|
||||
export function isStepfun(cfg: TtsConfig): boolean {
|
||||
return /(^|[./])stepfun\.com\b/i.test(cfg.baseUrl);
|
||||
}
|
||||
|
||||
function arrayBufferToBase64(buffer: ArrayBuffer): string {
|
||||
const bytes = new Uint8Array(buffer);
|
||||
let binary = "";
|
||||
@@ -21,53 +53,37 @@ function arrayBufferToBase64(buffer: ArrayBuffer): string {
|
||||
const OUTPUT_FORMAT = "mp3";
|
||||
const OUTPUT_MIME = "audio/mpeg";
|
||||
|
||||
type PresetVoice = {
|
||||
id: string;
|
||||
gender: "male" | "female";
|
||||
age: "teen" | "young" | "middle";
|
||||
/** Keywords (中文 or English) that, when present in the LLM's voice
|
||||
* description, boost this preset's score. Drawn from StepFun's published
|
||||
* voice name + recommended scenario. */
|
||||
tones: string[];
|
||||
};
|
||||
|
||||
// Full catalog from StepFun's docs (32 presets across step-tts-mini /
|
||||
// step-tts-2 / stepaudio-2.5-tts). Adding more later is safe — the scorer
|
||||
// degrades gracefully when an unknown id is picked.
|
||||
const PRESET_VOICES: PresetVoice[] = [
|
||||
{ id: "cixingnansheng", gender: "male", age: "young", tones: ["磁性", "成熟", "narrative"] },
|
||||
{ id: "wenrounansheng", gender: "male", age: "young", tones: ["温柔", "gentle", "supportive"] },
|
||||
{ id: "wenrougongzi", gender: "male", age: "young", tones: ["温柔", "公子", "tender"] },
|
||||
{ id: "yuanqinansheng", gender: "male", age: "teen", tones: ["元气", "energetic", "阳光"] },
|
||||
{ id: "zhengpaiqingnian", gender: "male", age: "young", tones: ["正派", "正气", "earnest"] },
|
||||
{ id: "shuangkuainansheng", gender: "male", age: "young", tones: ["爽快", "干脆", "brisk"] },
|
||||
{ id: "boyinnansheng", gender: "male", age: "middle", tones: ["播音", "broadcast", "稳重"] },
|
||||
{ id: "ruyananshi", gender: "male", age: "middle", tones: ["儒雅", "斯文", "refined"] },
|
||||
{ id: "shenchennanyin", gender: "male", age: "middle", tones: ["深沉", "低沉", "deep"] },
|
||||
{ id: "qingniandaxuesheng", gender: "male", age: "young", tones: ["大学生", "青年", "student"] },
|
||||
{ id: "zixinnansheng", gender: "male", age: "young", tones: ["自信", "confident"] },
|
||||
{ id: "elegantgentle-female", gender: "female", age: "young", tones: ["气质", "温婉", "professional"] },
|
||||
{ id: "livelybreezy-female", gender: "female", age: "teen", tones: ["活力", "轻快", "upbeat"] },
|
||||
{ id: "jingdiannvsheng", gender: "female", age: "middle", tones: ["经典", "classic", "成熟"] },
|
||||
{ id: "wenroushunv", gender: "female", age: "middle", tones: ["温柔", "熟女", "mature"] },
|
||||
{ id: "tianmeinvsheng", gender: "female", age: "young", tones: ["甜美", "sweet"] },
|
||||
{ id: "qingchunshaonv", gender: "female", age: "teen", tones: ["清纯", "少女", "pure"] },
|
||||
{ id: "yuanqishaonv", gender: "female", age: "teen", tones: ["元气", "少女", "活力", "energetic"] },
|
||||
{ id: "linjiajiejie", gender: "female", age: "young", tones: ["邻家", "姐姐"] },
|
||||
{ id: "jilingshaonv", gender: "female", age: "teen", tones: ["机灵", "灵动", "少女"] },
|
||||
{ id: "ruanmengnvsheng", gender: "female", age: "teen", tones: ["软萌", "可爱", "稚嫩", "甜软"] },
|
||||
{ id: "youyanvsheng", gender: "female", age: "young", tones: ["优雅", "elegant"] },
|
||||
{ id: "lengyanyujie", gender: "female", age: "middle", tones: ["冷艳", "御姐", "高冷"] },
|
||||
{ id: "shuangkuaijiejie", gender: "female", age: "young", tones: ["爽快", "姐姐", "干脆"] },
|
||||
{ id: "wenjingxuejie", gender: "female", age: "young", tones: ["文静", "学姐", "安静"] },
|
||||
{ id: "linjiameimei", gender: "female", age: "teen", tones: ["邻家", "妹妹"] },
|
||||
{ id: "zhixingjiejie", gender: "female", age: "young", tones: ["知性", "姐姐", "聪慧"] },
|
||||
{ id: "ganliannvsheng", gender: "female", age: "middle", tones: ["干练", "sharp", "professional"] },
|
||||
{ id: "qinhenvsheng", gender: "female", age: "young", tones: ["亲和", "warm", "亲切"] },
|
||||
{ id: "huolinvsheng", gender: "female", age: "young", tones: ["活力", "lively", "活泼"] },
|
||||
{ id: "qinqienvsheng", gender: "female", age: "middle", tones: ["亲切", "温暖"] },
|
||||
{ id: "wenrounvsheng", gender: "female", age: "young", tones: ["温柔", "tender", "柔和"] },
|
||||
];
|
||||
// step-tts-2 / stepaudio-2.5-tts). The JSON is the single source of truth —
|
||||
// shared by the scorer here, the CharacterDesigner prompt (via
|
||||
// formatStepfunCatalogForPrompt), the /api/tts-provider route's validity
|
||||
// check, and the offline enrich script. Adding more later is safe — the
|
||||
// scorer degrades gracefully when an unknown id is picked.
|
||||
// (catalogData is cast to PresetVoice[] at the import above; kept as
|
||||
// PRESET_VOICES so existing references stay unchanged.)
|
||||
|
||||
/** All valid preset voice ids — for validation by the CharacterDesigner
|
||||
* (discard an out-of-catalog LLM pick) and the enrich script. */
|
||||
export const STEPFUN_PRESET_VOICE_IDS: string[] = PRESET_VOICES.map(
|
||||
(v) => v.id,
|
||||
);
|
||||
|
||||
const STEPFUN_ID_SET = new Set(STEPFUN_PRESET_VOICE_IDS);
|
||||
|
||||
/** True iff `id` is one of the 32 catalog presets. Used to drop LLM-hallucinated
|
||||
* ids before they reach StepFun (which would otherwise 4xx on synth). */
|
||||
export function isValidStepfunVoiceId(id: string | null | undefined): boolean {
|
||||
return !!id && STEPFUN_ID_SET.has(id);
|
||||
}
|
||||
|
||||
/** Render the catalog as a 中文 prompt-friendly list, one line per preset,
|
||||
* so the CharacterDesigner and the enrich script can ask the LLM to pick a
|
||||
* matching voice id. Each line: `id — desc（gender/age）`. */
|
||||
export function formatStepfunCatalogForPrompt(): string {
|
||||
return PRESET_VOICES.map(
|
||||
(v) => `- ${v.id}：${v.desc}（${v.gender}/${v.age}）`,
|
||||
).join("\n");
|
||||
}
|
||||
|
||||
// Cheap deterministic 32-bit hash — used only to spread similar descriptions
|
||||
// across the top-N candidate voices so two "温柔女声" characters don't collide.
|
||||
@@ -139,12 +155,28 @@ export function pickStepfunVoiceId(description: string, salt = ""): string {
|
||||
// We mirror xiaomiProvision's async signature so the router stays uniform.
|
||||
// The optional `salt` (character name) spreads two characters that share
|
||||
// archetype keywords across the top-N candidate presets.
|
||||
//
|
||||
// `opts.stepfunVoiceId` — when the CharacterDesigner already picked a preset
|
||||
// (it sees the same catalog via formatStepfunCatalogForPrompt), honor it if
|
||||
// valid; otherwise fall back to the keyword scorer. This keeps StepFun
|
||||
// provisioning a pure function (zero network cost) while lifting voice-id
|
||||
// selection quality to LLM-grade on the live path.
|
||||
export type StepfunProvisionOptions = {
|
||||
/** LLM-selected preset id from the CharacterDesigner; validated against the
|
||||
* catalog and ignored when out of range (hallucination guard). */
|
||||
stepfunVoiceId?: string;
|
||||
};
|
||||
|
||||
export async function stepfunProvision(
|
||||
cfg: TtsConfig,
|
||||
description: string,
|
||||
salt?: string,
|
||||
opts?: StepfunProvisionOptions,
|
||||
): Promise<CharacterVoice> {
|
||||
const voiceId = pickStepfunVoiceId(description, salt);
|
||||
const voiceId =
|
||||
opts && isValidStepfunVoiceId(opts.stepfunVoiceId)
|
||||
? opts.stepfunVoiceId!
|
||||
: pickStepfunVoiceId(description, salt);
|
||||
return {
|
||||
provider: "stepfun",
|
||||
voiceId,
|
||||
|
||||
Reference in New Issue
Block a user