feat(tts): StepFun voice selection via CharacterDesigner + provider-aware beat-audio

Make homepage cards and live sessions produce sound when the server is
configured for StepFun TTS, instead of silently failing (the prebaked
Xiaomi voice was useless on a StepFun server, and wasted ~220KB/beat in
Fast Origin Transfer).

Three coordinated changes:

1. CharacterDesigner now picks a StepFun preset voice id directly from the
   32-entry catalog in the SAME LLM call that designs the character — zero
   extra latency, LLM-grade match quality. The Xiaomi prompt path is
   byte-identical to history (verified programmatically) so cache hit rate
   and voice quality are preserved. pickStepfunVoiceId (keyword scorer)
   remains the fallback for orphan speakers / invalid LLM picks.

2. The 32-preset catalog moves to lib/tts-client/stepfun-voices.json as the
   single source of truth, shared by the scorer, the CharacterDesigner
   prompt, /api/tts-provider, and the offline enrich script.

3. A new GET /api/tts-provider endpoint lets the client probe the server's
   TTS provider at /play mount. fetchBeatAudio then shapes its request body:
   on a StepFun server it sends the lightweight stepfunVoiceId /
   voiceDescription and omits the ~220KB Xiaomi reference audio (FOT saving
   ~13MB per protagonist per session on prebaked cards). requestBeatAudio
   re-provisions on a provider mismatch before synth, so audio never goes
   silent on a cross-provider replay or mid-session provider flip.

New type fields are all optional and backward-compatible: Character.stepfunVoiceId,
BeatAudioRequest.voiceDescription/characterName/stepfunVoiceId, voice made
optional. AGENTS.md updated for the new route, type fields, dependency map,
and StepFun voice-selection flow.
This commit is contained in:
yuanzonghao
2026-06-15 12:49:25 +08:00
parent da191dd7a2
commit ca73a41a0b
15 changed files with 754 additions and 90 deletions
+36 -4
View File
@@ -1,5 +1,10 @@
import { chat, generateImage } from "@infiplot/ai-client";
import { provisionVoice } from "@infiplot/tts-client";
import {
isStepfun,
isValidStepfunVoiceId,
provisionVoice,
type ProvisionVoiceOptions,
} from "@infiplot/tts-client";
import type {
Character,
CharacterVoice,
@@ -9,7 +14,7 @@ import type {
import { parseJsonLoose } from "../jsonParser";
import { mockImageDataUri } from "../mockImage";
import {
CHARACTER_DESIGNER_SYSTEM,
buildCharacterDesignerSystem,
buildCharacterDesignerUserMessage,
buildCharacterPortraitPrompt,
} from "../prompts";
@@ -34,6 +39,10 @@ import {
type CharacterDesignOutput = {
visualDescription?: string;
voiceDescription?: string;
/** Only present on the StepFun path (the system prompt asks for it when
* stepfun:true). Hallucinated / out-of-catalog ids are dropped before
* they reach provisioning, falling back to pickStepfunVoiceId. */
stepfunVoiceId?: string;
};
// TEMP: per-phase timing for latency diagnosis. Same convention as the
@@ -50,7 +59,7 @@ async function runDesignLLM(
const raw = await chat(
config.text,
[
{ role: "system", content: CHARACTER_DESIGNER_SYSTEM },
{ role: "system", content: buildCharacterDesignerSystem({ stepfun: stepfunEnabled(config) }) },
{
role: "user",
content: buildCharacterDesignerUserMessage(charName, session),
@@ -61,6 +70,13 @@ async function runDesignLLM(
return parseJsonLoose<CharacterDesignOutput>(raw);
}
/** True when the server's TTS config points at StepFun (so the CharacterDesigner
* should also pick a preset voice id). Returns false when TTS is off or on the
* Xiaomi path — keeping the Xiaomi prompt byte-identical to history. */
function stepfunEnabled(config: EngineConfig): boolean {
return !!config.tts && isStepfun(config.tts);
}
// Generate the per-character base portrait. The portrait is a "concept
// sheet" — single character, neutral pose, plain background — so it works
// well as a Runware referenceImages anchor for later scenes.
@@ -105,10 +121,11 @@ export async function provisionCharacterVoice(
config: EngineConfig,
voiceDescription: string,
charName: string,
opts?: ProvisionVoiceOptions,
): Promise<CharacterVoice | undefined> {
if (!config.tts) return undefined;
try {
return await provisionVoice(config.tts, voiceDescription, charName);
return await provisionVoice(config.tts, voiceDescription, charName, opts);
} catch (err) {
const msg = err instanceof Error ? err.message : String(err);
console.error(`[characterDesigner] voice provision failed for ${charName}: ${msg}`);
@@ -120,10 +137,18 @@ export async function provisionCharacterVoice(
// call. The director then schedules renderCharacterPortrait /
// provisionCharacterVoice around the Painter. Multiple new characters in the
// same scene run this stage in parallel at the director level.
//
// On the StepFun path the same call ALSO yields stepfunVoiceId (the model
// picks from the 32-preset catalog it sees in the system prompt). An invalid
// pick is dropped here so the downstream provision falls back to the keyword
// scorer — never trust an LLM-hallucinated id at the synth boundary.
export type CharacterCard = {
name: string;
visualDescription?: string;
voiceDescription: string;
/** Only set on the StepFun path AND only when the LLM picked a valid catalog
* id. Threads through provisionCharacterVoice → stepfunProvision. */
stepfunVoiceId?: string;
};
export async function designCharacterCard(
@@ -135,12 +160,19 @@ export async function designCharacterCard(
const design = await runDesignLLM(config, session, charName);
tlog(`[charDesigner ${charName}] design LLM`, tDesign);
// Drop invalid catalog picks before they reach provision/synth. A hallucinated
// id would 4xx at synth time; better to fall back to pickStepfunVoiceId now.
const stepfunVoiceId = isValidStepfunVoiceId(design.stepfunVoiceId)
? design.stepfunVoiceId
: undefined;
return {
name: charName,
visualDescription: design.visualDescription?.trim() || undefined,
voiceDescription:
design.voiceDescription?.trim() ||
`请根据角色名「${charName}」推断其性别、年龄与气质,生成最贴合的音色。所属世界观:${session.worldSetting}`,
stepfunVoiceId,
};
}