From 12aff609a16a489213bbcdb15555a2d034c8982f Mon Sep 17 00:00:00 2001 From: yibo Date: Wed, 2 Sep 2026 11:48:12 +0800 Subject: [PATCH] Fix: make generation poll loop wall-clock instead of fixed poll count --- dressup-generate.test.js | 113 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ server.js | 14 ++++++++++++-- 2 files changed, 125 insertions(+), 2 deletions(-) diff --git a/dressup-generate.test.js b/dressup-generate.test.js index b429dff..9367da0 100644 --- a/dressup-generate.test.js +++ b/dressup-generate.test.js @@ -289,3 +289,116 @@ test("batch-gen: rejects missing characterId or empty garmentIds", async () => { const res = await fetch(`${BASE}/api/admin/batch-gen`, authed({ method: "POST", body: JSON.stringify({ characterId: "nope", garmentIds: [] }) })); assert.equal(res.status, 400); }); + +// Regression for a real incident (2026-09-02): a 9-job batch against H3's +// single sharded-model backend serializes almost entirely — the last job's +// queue wait alone blew past the old fixed 200-poll budget (~1000s) even +// though nothing had actually failed. generateDanceVideo's poll loop is now +// wall-clock (GENERATION_TIMEOUT_MS), independent of GENERATION_POLL_INTERVAL_MS, +// so a job stuck "queued" must be given the full configured budget before +// it's declared failed — not bailed out after a fixed number of polls. +test("generate: poll loop honors GENERATION_TIMEOUT_MS wall-clock, not a fixed poll count", async () => { + let neverDoneMockH3; + const port = await new Promise((resolve) => { + neverDoneMockH3 = http.createServer((req, res) => { + req.url = req.url.replace(/^\/+/, "/"); + const chunks = []; + req.on("data", (c) => chunks.push(c)); + req.on("end", () => { + if (req.method === "POST" && req.url === "/v1/assets") { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ id: "asset-x", url: "http://mock-h3/asset/x", kind: "image" })); + return; + } + if (req.method === "POST" && req.url === "/v1/videos") { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ id: "job-x" })); + return; + } + if (req.method === "GET" && req.url === "/v1/videos/job-x") { + // Always "queued" — simulates sitting behind a deep batch queue. + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ id: "job-x", status: "queued" })); + return; + } + res.writeHead(404, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: "not found in mock" })); + }); + }); + neverDoneMockH3.listen(0, () => resolve(neverDoneMockH3.address().port)); + }); + + const port2 = 22934 + Math.floor(Math.random() * 1000); + const base2 = `http://localhost:${port2}`; + const cwd2 = fs.mkdtempSync(path.join(os.tmpdir(), "dressup-timeout-")); + for (const f of ["server.js", "ingest.js", "uploads.js", "store.js", "dressup-store.js", "prompt-template.js"]) { + fs.copyFileSync(path.join(__dirname, f), path.join(cwd2, f)); + } + fs.mkdirSync(path.join(cwd2, "uploads"), { recursive: true }); + fs.writeFileSync(path.join(cwd2, "uploads", "person.png"), "fake-png-bytes"); + fs.writeFileSync(path.join(cwd2, "uploads", "garment.png"), "fake-png-bytes"); + fs.writeFileSync(path.join(cwd2, "uploads", "ref.mp4"), "fake-mp4-bytes"); + + const child2 = spawn(process.execPath, ["server.js"], { + cwd: cwd2, + env: { + ...process.env, + PORT: String(port2), + H3_CONSOLE_BASE: `http://localhost:${port}`, + H3_CONSOLE_TOKEN: "mock-token", + ADMIN_PASSWORD, + GENERATION_POLL_INTERVAL_MS: "20", + GENERATION_TIMEOUT_MS: "150", + }, + stdio: "pipe", + }); + try { + await waitForServer(base2); + const loginRes = await fetch(`${base2}/api/admin/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ password: ADMIN_PASSWORD }), + }); + const cookie2 = (loginRes.headers.get("set-cookie") || "").split(";")[0]; + const authed2 = (opts = {}) => ({ ...opts, headers: { "Content-Type": "application/json", Cookie: cookie2, ...opts.headers } }); + + const character = await fetch( + `${base2}/api/admin/characters`, + authed2({ method: "POST", body: JSON.stringify({ name: "小美", imageUrl: "/uploads/person.png", appearanceDescriptor: "black hair" }) }) + ).then((r) => r.json()); + const garment = await fetch( + `${base2}/api/admin/garments`, + authed2({ method: "POST", body: JSON.stringify({ name: "红旗袍", imageUrl: "/uploads/garment.png", appearanceDescriptor: "red qipao" }) }) + ).then((r) => r.json()); + await fetch( + `${base2}/api/admin/dance-refs`, + authed2({ method: "POST", body: JSON.stringify({ name: "拉丁舞", videoUrl: "/uploads/ref.mp4", motionDescriptor: "hip sways" }) }) + ); + + const submitted = await fetch( + `${base2}/api/admin/generate`, + authed2({ method: "POST", body: JSON.stringify({ characterId: character.id, garmentId: garment.id }) }) + ).then((r) => r.json()); + + // Still "queued" well past the old 200-poll budget's equivalent + // wall-clock (200 * 20ms = 4000ms) would have been fine either way at + // this poll interval — the real proof is that it survives on "queued" + // for longer than a short timeout would allow, then correctly fails + // once GENERATION_TIMEOUT_MS (150ms) actually elapses. + const start = Date.now(); + let job; + for (let i = 0; i < 50; i++) { + job = await fetch(`${base2}/api/admin/generate/${submitted.jobId}`, authed2()).then((r) => r.json()); + if (job.status === "failed") break; + await new Promise((r) => setTimeout(r, 20)); + } + const elapsed = Date.now() - start; + assert.equal(job.status, "failed"); + assert.equal(job.error, "generation timed out"); + assert.ok(elapsed >= 150, `should not fail before the configured 150ms deadline (took ${elapsed}ms)`); + } finally { + child2.kill(); + neverDoneMockH3.close(); + fs.rmSync(cwd2, { recursive: true, force: true }); + } +}); diff --git a/server.js b/server.js index fea8b34..b522398 100644 --- a/server.js +++ b/server.js @@ -771,7 +771,15 @@ async function generateDanceVideo(job, opts = {}) { job = generationJobs.update(job.id, { h3JobId: submitted.id, status: "generating" }); } - for (let i = 0; i < 200; i++) { + // ponytail: H3's 4 GPUs run one sharded model, not 4 independent + // workers (H3-CONSOLE.md /v1/status), so a batch of N jobs is served + // ~serially — the Nth job in queue_position can sit "queued" for + // N * ~700s before it even starts. A fixed poll-count budget doesn't + // know about queue depth, so this is wall-clock and generous instead; + // upgrade path if batches grow past a couple dozen: read `eta_seconds`/ + // `queue_position` off the poll response and size the deadline to that. + const deadline = Date.now() + (opts.timeoutMs || Number(process.env.GENERATION_TIMEOUT_MS) || 2 * 60 * 60 * 1000); + while (Date.now() < deadline) { await new Promise((r) => setTimeout(r, pollIntervalMs)); const polled = await h3ConsoleFetch(`/v1/videos/${job.h3JobId}`); if (polled.deleted_at) throw new Error("job was deleted on the H3 console before it finished"); @@ -808,7 +816,9 @@ async function generateDanceVideo(job, opts = {}) { generationJobs.update(job.id, { status: "done", generatedVideoId: clip.id }); return clip; } - if (polled.status === "error" || polled.status === "failed") throw new Error(polled.error || "generation failed"); + if (polled.status === "error" || polled.status === "failed" || polled.status === "cancelled") { + throw new Error(polled.error || `generation ${polled.status}`); + } } throw new Error("generation timed out"); } catch (err) { -- libgit2 0.21.4