宜博 / h3livestream

Commit 12aff609a16a489213bbcdb15555a2d034c8982f

Authored by yibo
1 parent a68fd8b3

Fix: make generation poll loop wall-clock instead of fixed poll count 

H3's 4 GPUs run one sharded model (H3-CONSOLE.md /v1/status), not 4
independent workers, so a batch of N jobs is served ~serially. The
poll loop's old fixed 200-iteration budget (~1000s at the 5s default
interval) didn't account for queue depth: submitting a 9-job batch hit
this live and 9/11 jobs got marked "generation timed out" locally
while still sitting healthy in H3's queue (queue_position confirmed
via GET /v1/videos/:id). Also treats H3's "cancelled" status as
terminal, matching "error"/"failed".

Recovered the wrongly-failed jobs by resetting them to "generating" in
generation-jobs.json (their h3JobId was already set, so
resumeDressupGenerations() picked them back up without resubmitting to
H3) rather than resubmitting and burning the queue further.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011Deddn3sM3oQnmvqWSMb4y
Inline Side-by-side
Showing 2 changed files with 125 additions and 2 deletions
dressup-generate.test.js
  View file @12aff60
... ... @@ -289,3 +289,116 @@ test(&quot;batch-gen: rejects missing characterId or empty garmentIds&quot;, async () =&gt; {
289 289 const res = await fetch(`${BASE}/api/admin/batch-gen`, authed({ method: "POST", body: JSON.stringify({ characterId: "nope", garmentIds: [] }) }));
290 290 assert.equal(res.status, 400);
291 291 });
  292 +
  293 +// Regression for a real incident (2026-09-02): a 9-job batch against H3's
  294 +// single sharded-model backend serializes almost entirely — the last job's
  295 +// queue wait alone blew past the old fixed 200-poll budget (~1000s) even
  296 +// though nothing had actually failed. generateDanceVideo's poll loop is now
  297 +// wall-clock (GENERATION_TIMEOUT_MS), independent of GENERATION_POLL_INTERVAL_MS,
  298 +// so a job stuck "queued" must be given the full configured budget before
  299 +// it's declared failed — not bailed out after a fixed number of polls.
  300 +test("generate: poll loop honors GENERATION_TIMEOUT_MS wall-clock, not a fixed poll count", async () => {
  301 + let neverDoneMockH3;
  302 + const port = await new Promise((resolve) => {
  303 + neverDoneMockH3 = http.createServer((req, res) => {
  304 + req.url = req.url.replace(/^\/+/, "/");
  305 + const chunks = [];
  306 + req.on("data", (c) => chunks.push(c));
  307 + req.on("end", () => {
  308 + if (req.method === "POST" && req.url === "/v1/assets") {
  309 + res.writeHead(200, { "Content-Type": "application/json" });
  310 + res.end(JSON.stringify({ id: "asset-x", url: "http://mock-h3/asset/x", kind: "image" }));
  311 + return;
  312 + }
  313 + if (req.method === "POST" && req.url === "/v1/videos") {
  314 + res.writeHead(200, { "Content-Type": "application/json" });
  315 + res.end(JSON.stringify({ id: "job-x" }));
  316 + return;
  317 + }
  318 + if (req.method === "GET" && req.url === "/v1/videos/job-x") {
  319 + // Always "queued" — simulates sitting behind a deep batch queue.
  320 + res.writeHead(200, { "Content-Type": "application/json" });
  321 + res.end(JSON.stringify({ id: "job-x", status: "queued" }));
  322 + return;
  323 + }
  324 + res.writeHead(404, { "Content-Type": "application/json" });
  325 + res.end(JSON.stringify({ error: "not found in mock" }));
  326 + });
  327 + });
  328 + neverDoneMockH3.listen(0, () => resolve(neverDoneMockH3.address().port));
  329 + });
  330 +
  331 + const port2 = 22934 + Math.floor(Math.random() * 1000);
  332 + const base2 = `http://localhost:${port2}`;
  333 + const cwd2 = fs.mkdtempSync(path.join(os.tmpdir(), "dressup-timeout-"));
  334 + for (const f of ["server.js", "ingest.js", "uploads.js", "store.js", "dressup-store.js", "prompt-template.js"]) {
  335 + fs.copyFileSync(path.join(__dirname, f), path.join(cwd2, f));
  336 + }
  337 + fs.mkdirSync(path.join(cwd2, "uploads"), { recursive: true });
  338 + fs.writeFileSync(path.join(cwd2, "uploads", "person.png"), "fake-png-bytes");
  339 + fs.writeFileSync(path.join(cwd2, "uploads", "garment.png"), "fake-png-bytes");
  340 + fs.writeFileSync(path.join(cwd2, "uploads", "ref.mp4"), "fake-mp4-bytes");
  341 +
  342 + const child2 = spawn(process.execPath, ["server.js"], {
  343 + cwd: cwd2,
  344 + env: {
  345 + ...process.env,
  346 + PORT: String(port2),
  347 + H3_CONSOLE_BASE: `http://localhost:${port}`,
  348 + H3_CONSOLE_TOKEN: "mock-token",
  349 + ADMIN_PASSWORD,
  350 + GENERATION_POLL_INTERVAL_MS: "20",
  351 + GENERATION_TIMEOUT_MS: "150",
  352 + },
  353 + stdio: "pipe",
  354 + });
  355 + try {
  356 + await waitForServer(base2);
  357 + const loginRes = await fetch(`${base2}/api/admin/login`, {
  358 + method: "POST",
  359 + headers: { "Content-Type": "application/json" },
  360 + body: JSON.stringify({ password: ADMIN_PASSWORD }),
  361 + });
  362 + const cookie2 = (loginRes.headers.get("set-cookie") || "").split(";")[0];
  363 + const authed2 = (opts = {}) => ({ ...opts, headers: { "Content-Type": "application/json", Cookie: cookie2, ...opts.headers } });
  364 +
  365 + const character = await fetch(
  366 + `${base2}/api/admin/characters`,
  367 + authed2({ method: "POST", body: JSON.stringify({ name: "小美", imageUrl: "/uploads/person.png", appearanceDescriptor: "black hair" }) })
  368 + ).then((r) => r.json());
  369 + const garment = await fetch(
  370 + `${base2}/api/admin/garments`,
  371 + authed2({ method: "POST", body: JSON.stringify({ name: "红旗袍", imageUrl: "/uploads/garment.png", appearanceDescriptor: "red qipao" }) })
  372 + ).then((r) => r.json());
  373 + await fetch(
  374 + `${base2}/api/admin/dance-refs`,
  375 + authed2({ method: "POST", body: JSON.stringify({ name: "拉丁舞", videoUrl: "/uploads/ref.mp4", motionDescriptor: "hip sways" }) })
  376 + );
  377 +
  378 + const submitted = await fetch(
  379 + `${base2}/api/admin/generate`,
  380 + authed2({ method: "POST", body: JSON.stringify({ characterId: character.id, garmentId: garment.id }) })
  381 + ).then((r) => r.json());
  382 +
  383 + // Still "queued" well past the old 200-poll budget's equivalent
  384 + // wall-clock (200 * 20ms = 4000ms) would have been fine either way at
  385 + // this poll interval — the real proof is that it survives on "queued"
  386 + // for longer than a short timeout would allow, then correctly fails
  387 + // once GENERATION_TIMEOUT_MS (150ms) actually elapses.
  388 + const start = Date.now();
  389 + let job;
  390 + for (let i = 0; i < 50; i++) {
  391 + job = await fetch(`${base2}/api/admin/generate/${submitted.jobId}`, authed2()).then((r) => r.json());
  392 + if (job.status === "failed") break;
  393 + await new Promise((r) => setTimeout(r, 20));
  394 + }
  395 + const elapsed = Date.now() - start;
  396 + assert.equal(job.status, "failed");
  397 + assert.equal(job.error, "generation timed out");
  398 + assert.ok(elapsed >= 150, `should not fail before the configured 150ms deadline (took ${elapsed}ms)`);
  399 + } finally {
  400 + child2.kill();
  401 + neverDoneMockH3.close();
  402 + fs.rmSync(cwd2, { recursive: true, force: true });
  403 + }
  404 +});
... ...
server.js
  View file @12aff60
... ... @@ -771,7 +771,15 @@ async function generateDanceVideo(job, opts = {}) {
771 771 job = generationJobs.update(job.id, { h3JobId: submitted.id, status: "generating" });
772 772 }
773 773  
774   - for (let i = 0; i < 200; i++) {
  774 + // ponytail: H3's 4 GPUs run one sharded model, not 4 independent
  775 + // workers (H3-CONSOLE.md /v1/status), so a batch of N jobs is served
  776 + // ~serially — the Nth job in queue_position can sit "queued" for
  777 + // N * ~700s before it even starts. A fixed poll-count budget doesn't
  778 + // know about queue depth, so this is wall-clock and generous instead;
  779 + // upgrade path if batches grow past a couple dozen: read `eta_seconds`/
  780 + // `queue_position` off the poll response and size the deadline to that.
  781 + const deadline = Date.now() + (opts.timeoutMs || Number(process.env.GENERATION_TIMEOUT_MS) || 2 * 60 * 60 * 1000);
  782 + while (Date.now() < deadline) {
775 783 await new Promise((r) => setTimeout(r, pollIntervalMs));
776 784 const polled = await h3ConsoleFetch(`/v1/videos/${job.h3JobId}`);
777 785 if (polled.deleted_at) throw new Error("job was deleted on the H3 console before it finished");
... ... @@ -808,7 +816,9 @@ async function generateDanceVideo(job, opts = {}) {
808 816 generationJobs.update(job.id, { status: "done", generatedVideoId: clip.id });
809 817 return clip;
810 818 }
811   - if (polled.status === "error" || polled.status === "failed") throw new Error(polled.error || "generation failed");
  819 + if (polled.status === "error" || polled.status === "failed" || polled.status === "cancelled") {
  820 + throw new Error(polled.error || `generation ${polled.status}`);
  821 + }
812 822 }
813 823 throw new Error("generation timed out");
814 824 } catch (err) {
... ...