Files
infiplot-web/apps/web/app/play/page.tsx
T
yuanzonghao 9cedfa66e4 feat: prefetch, vision split, provider adapter, UI polish
Engine
- Split /api/vision out from /api/interact so client can drive
  prefetch + cache lookup independently of click interpretation
- Image client switched to chat-completions+modalities API (OpenRouter/
  provider style), supporting markdown image URL responses
- annotateClick now resizes to 768w before composite to keep vision
  payloads small and avoid CDN timeouts
- Prompts updated to mention "JSON" in user messages (required by
  Gemini's strict JSON mode)
- Shared fetchWithRetry helper: 2 retries for chat/image, 0 for vision
  (with 60s hard timeout)

Client
- Parallel prefetch of all three choice branches on each new frame
- Effect deliberately excludes phase from deps so user-click doesn't
  abort in-flight prefetches
- Cache hit/miss/free-form fallback handled in handleClick
- PlayCanvas reads img naturalWidth/Height and adapts container to
  whatever aspect AI returns (no more cropped third choice)
- max-width raised to 560px, max-height calc(100dvh - 200px)

Misc
- README env-path corrected to apps/web/.env.local
- users.md: BGM/TTS idea note
- .env.example moved into apps/web alongside next config

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-12 19:38:03 +08:00

327 lines
11 KiB
TypeScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"use client";
import Link from "next/link";
import { useRouter, useSearchParams } from "next/navigation";
import { Suspense, useEffect, useRef, useState } from "react";
import { PlayCanvas, type Phase } from "@/components/PlayCanvas";
import { PRESETS } from "@/lib/presets";
import type {
ClickIntent,
InteractResponse,
Session,
StartResponse,
StoryFrame,
VisionResponse,
} from "@dada/types";
function PlayInner() {
const router = useRouter();
const params = useSearchParams();
const [phase, setPhase] = useState<Phase>("loading-first");
const [session, setSession] = useState<Session | null>(null);
const [imageBase64, setImageBase64] = useState<string | null>(null);
const [frame, setFrame] = useState<StoryFrame | null>(null);
const [intent, setIntent] = useState<ClickIntent | null>(null);
const [pendingClick, setPendingClick] = useState<{
x: number;
y: number;
} | null>(null);
const [turnNum, setTurnNum] = useState(0);
const [error, setError] = useState<string | null>(null);
const startedRef = useRef(false);
const prefetchAbortRef = useRef<AbortController | null>(null);
const prefetchRef = useRef<Record<string, Promise<InteractResponse>>>({});
useEffect(() => {
if (startedRef.current) return;
startedRef.current = true;
let payload: { worldSetting: string; styleGuide: string } | null = null;
const presetId = params.get("preset");
if (presetId) {
const p = PRESETS.find((x) => x.id === presetId);
if (p) {
payload = { worldSetting: p.worldSetting, styleGuide: p.styleGuide };
}
} else if (params.get("custom") === "1") {
const stored = sessionStorage.getItem("dada:custom");
if (stored) {
try {
payload = JSON.parse(stored);
} catch {
payload = null;
}
}
}
if (!payload) {
router.replace("/");
return;
}
const finalPayload = payload;
fetch("/api/start", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify(finalPayload),
})
.then(async (r) => {
if (!r.ok) {
const j = (await r.json().catch(() => ({}))) as { error?: string };
throw new Error(j.error ?? r.statusText);
}
return r.json() as Promise<StartResponse>;
})
.then((data) => {
setSession({
id: data.sessionId,
createdAt: Date.now(),
worldSetting: finalPayload.worldSetting,
styleGuide: finalPayload.styleGuide,
history: [{ frame: data.frame }],
});
setFrame(data.frame);
setImageBase64(data.imageBase64);
setPhase("ready");
setTurnNum(1);
})
.catch((e) => setError(String(e)));
}, [params, router]);
// Prefetch next-frame candidates whenever current frame becomes ready.
// All three fire in parallel for fastest cache fill. NOT depending on
// `phase` — we don't want to abort in-flight prefetches just because
// the user clicked. They should continue so handleClick can await them.
useEffect(() => {
if (!session || !frame) return;
prefetchAbortRef.current?.abort();
const ctrl = new AbortController();
prefetchAbortRef.current = ctrl;
const choices = frame.uiElements.filter((e) => e.kind === "choice");
const promises: Record<string, Promise<InteractResponse>> = {};
for (const choice of choices) {
const syntheticIntent: ClickIntent = {
targetId: choice.id,
targetLabel: choice.label,
reasoning: "prefetch",
};
const p = fetch("/api/interact", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ session, intent: syntheticIntent }),
signal: ctrl.signal,
}).then(async (r) => {
if (!r.ok) {
const j = (await r.json().catch(() => ({}))) as { error?: string };
throw new Error(j.error ?? r.statusText);
}
return r.json() as Promise<InteractResponse>;
});
p.catch(() => {});
promises[choice.id] = p;
}
prefetchRef.current = promises;
return () => {
ctrl.abort();
};
}, [frame?.id, session?.id]);
async function handleClick(click: { x: number; y: number }) {
if (phase !== "ready" || !session || !imageBase64) return;
setPhase("interacting");
setPendingClick(click);
setIntent(null);
const cacheSnapshot = prefetchRef.current;
try {
// Step 1: Vision (~4s) — figure out what the user actually clicked
const visionRes = await fetch("/api/vision", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({
session,
prevImageBase64: imageBase64,
click,
}),
});
if (!visionRes.ok) {
const j = (await visionRes.json().catch(() => ({}))) as {
error?: string;
};
throw new Error(j.error ?? visionRes.statusText);
}
const { intent: clickIntent } =
(await visionRes.json()) as VisionResponse;
// Step 2: Cache lookup
const cached = clickIntent.targetId
? cacheSnapshot[clickIntent.targetId]
: undefined;
let result: InteractResponse;
if (cached) {
// Cache hit — await the prefetched promise (mostly already resolved)
result = await cached;
// Overwrite the synthetic prefetch intent on history with the real one
const lastIdx = result.session.history.length - 1;
result = {
...result,
intent: clickIntent,
session: {
...result.session,
history: result.session.history.map((entry, idx) =>
idx === lastIdx
? { ...entry, click, intent: clickIntent }
: entry,
),
},
};
} else {
// Cache miss (free-form click) — abort wasted prefetches, run live
prefetchAbortRef.current?.abort();
const liveRes = await fetch("/api/interact", {
method: "POST",
headers: { "Content-Type": "application/json" },
body: JSON.stringify({ session, intent: clickIntent, click }),
});
if (!liveRes.ok) {
const j = (await liveRes.json().catch(() => ({}))) as {
error?: string;
};
throw new Error(j.error ?? liveRes.statusText);
}
result = (await liveRes.json()) as InteractResponse;
}
// Apply the result: append new frame to history
const updatedHistory = [...result.session.history, { frame: result.frame }];
setSession({ ...result.session, history: updatedHistory });
setFrame(result.frame);
setImageBase64(result.imageBase64);
setIntent(clickIntent);
setPendingClick(null);
setTurnNum((t) => t + 1);
setPhase("ready");
} catch (e) {
setError(String(e));
setPendingClick(null);
setPhase("ready");
}
}
if (error) {
return (
<div className="min-h-screen flex flex-col items-center justify-center px-8">
<div className="max-w-md text-center animate-fade-in">
<p className="text-[10px] smallcaps text-clay-500 mb-6">
An · error · occurred
</p>
<p className="font-serif italic text-clay-900 text-lg leading-[1.7] mb-10">
{error}
</p>
<Link
href="/"
className="text-[10px] smallcaps text-clay-700 hover:text-ember-500 transition-colors inline-flex items-center gap-3"
>
<i className="fa-solid fa-arrow-left text-[9px]" />
Return
</Link>
</div>
</div>
);
}
return (
<div className="min-h-screen flex flex-col">
<header className="px-5 md:px-12 pt-6 md:pt-8 flex items-center justify-between">
<Link
href="/"
className="text-[10px] smallcaps text-clay-600 hover:text-clay-900 transition-colors flex items-center gap-2"
>
<i className="fa-solid fa-arrow-left text-[9px]" />
Dada
</Link>
<div className="flex items-center gap-3 text-[10px] smallcaps text-clay-500 num">
<span>Frame · {String(turnNum).padStart(3, "0")}</span>
<span className="text-clay-300">·</span>
<span className="hidden sm:inline truncate max-w-[180px]">
{session?.id.slice(2, 14) ?? "—"}
</span>
</div>
</header>
<main className="flex-1 flex flex-col items-center justify-center px-4 md:px-8 py-6 md:py-10">
<PlayCanvas
imageBase64={imageBase64}
phase={phase}
pendingClick={pendingClick}
onClick={handleClick}
/>
<div className="mt-7 md:mt-9 max-w-md w-full text-center min-h-[64px] flex items-center justify-center">
{phase === "loading-first" && (
<p className="text-[10px] smallcaps text-clay-500 animate-slow-pulse">
Summoning · the · first · frame
</p>
)}
{phase === "interacting" && (
<div className="flex flex-col items-center gap-2 animate-fade-in">
<p className="text-[10px] smallcaps text-clay-500 animate-slow-pulse">
AI · is · painting · the · next · moment
</p>
<p className="font-serif italic text-clay-400 text-xs">
cached choices resolve in seconds · free-form takes longer
</p>
</div>
)}
{phase === "ready" && intent?.targetLabel && (
<p className="font-serif italic text-clay-500 text-base leading-relaxed animate-fade-in max-w-[320px]">
<span className="text-[9px] smallcaps not-italic text-clay-400 mr-2 align-middle">
Last · move ·
</span>
<span className="align-middle">{intent.targetLabel}</span>
</p>
)}
{phase === "ready" && !intent && turnNum > 0 && (
<p className="text-[10px] smallcaps text-clay-400 animate-fade-in">
Click · anywhere · to · respond
</p>
)}
</div>
</main>
<footer className="px-5 md:px-12 pb-6">
<div className="text-[9px] smallcaps text-clay-400 text-center num">
·
</div>
</footer>
</div>
);
}
export default function PlayPage() {
return (
<Suspense
fallback={
<div className="min-h-screen flex items-center justify-center">
<span className="text-[10px] smallcaps text-clay-500 animate-slow-pulse">
Loading
</span>
</div>
}
>
<PlayInner />
</Suspense>
);
}