Fix: make generation poll loop wall-clock instead of fixed poll count

H3's 4 GPUs run one sharded model (H3-CONSOLE.md /v1/status), not 4 independent workers, so a batch of N jobs is served ~serially. The poll loop's old fixed 200-iteration budget (~1000s at the 5s default interval) didn't account for queue depth: submitting a 9-job batch hit this live and 9/11 jobs got marked "generation timed out" locally while still sitting healthy in H3's queue (queue_position confirmed via GET /v1/videos/:id). Also treats H3's "cancelled" status as terminal, matching "error"/"failed". Recovered the wrongly-failed jobs by resetting them to "generating" in generation-jobs.json (their h3JobId was already set, so resumeDressupGenerations() picked them back up without resubmitting to H3) rather than resubmitting and burning the queue further. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011Deddn3sM3oQnmvqWSMb4y

Fix: make generation poll loop wall-clock instead of fixed poll count
H3's 4 GPUs run one sharded model (H3-CONSOLE.md /v1/status), not 4 independent workers, so a batch of N jobs is served ~serially. The poll loop's old fixed 200-iteration budget (~1000s at the 5s default interval) didn't account for queue depth: submitting a 9-job batch hit this live and 9/11 jobs got marked "generation timed out" locally while still sitting healthy in H3's queue (queue_position confirmed via GET /v1/videos/:id). Also treats H3's "cancelled" status as terminal, matching "error"/"failed". Recovered the wrongly-failed jobs by resetting them to "generating" in generation-jobs.json (their h3JobId was already set, so resumeDressupGenerations() picked them back up without resubmitting to H3) rather than resubmitting and burning the queue further. Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_011Deddn3sM3oQnmvqWSMb4y
yibo · 宜博
1 parent a68fd8b3
Showing 2 changed files with 125 additions and 2 deletions
dressup-generate.test.js
server.js
@@ -289,3 +289,116 @@ test(&quot;batch-gen: rejects missing characterId or empty garmentIds&quot;, async () =&gt; {
   const res = await fetch(`${BASE}/api/admin/batch-gen`, authed({ method: "POST", body: JSON.stringify({ characterId: "nope", garmentIds: [] }) }));
   assert.equal(res.status, 400);
 });
+
+// Regression for a real incident (2026-09-02): a 9-job batch against H3's
+// single sharded-model backend serializes almost entirely — the last job's
+// queue wait alone blew past the old fixed 200-poll budget (~1000s) even
+// though nothing had actually failed. generateDanceVideo's poll loop is now
+// wall-clock (GENERATION_TIMEOUT_MS), independent of GENERATION_POLL_INTERVAL_MS,
+// so a job stuck "queued" must be given the full configured budget before
+// it's declared failed — not bailed out after a fixed number of polls.
+test("generate: poll loop honors GENERATION_TIMEOUT_MS wall-clock, not a fixed poll count", async () => {
+  let neverDoneMockH3;
+  const port = await new Promise((resolve) => {
+    neverDoneMockH3 = http.createServer((req, res) => {
+      req.url = req.url.replace(/^\/+/, "/");
+      const chunks = [];
+      req.on("data", (c) => chunks.push(c));
+      req.on("end", () => {
+        if (req.method === "POST" && req.url === "/v1/assets") {
+          res.writeHead(200, { "Content-Type": "application/json" });
+          res.end(JSON.stringify({ id: "asset-x", url: "http://mock-h3/asset/x", kind: "image" }));
+          return;
+        }
+        if (req.method === "POST" && req.url === "/v1/videos") {
+          res.writeHead(200, { "Content-Type": "application/json" });
+          res.end(JSON.stringify({ id: "job-x" }));
+          return;
+        }
+        if (req.method === "GET" && req.url === "/v1/videos/job-x") {
+          // Always "queued" — simulates sitting behind a deep batch queue.
+          res.writeHead(200, { "Content-Type": "application/json" });
+          res.end(JSON.stringify({ id: "job-x", status: "queued" }));
+          return;
+        }
+        res.writeHead(404, { "Content-Type": "application/json" });
+        res.end(JSON.stringify({ error: "not found in mock" }));
+      });
+    });
+    neverDoneMockH3.listen(0, () => resolve(neverDoneMockH3.address().port));
+  });
+
+  const port2 = 22934 + Math.floor(Math.random() * 1000);
+  const base2 = `http://localhost:${port2}`;
+  const cwd2 = fs.mkdtempSync(path.join(os.tmpdir(), "dressup-timeout-"));
+  for (const f of ["server.js", "ingest.js", "uploads.js", "store.js", "dressup-store.js", "prompt-template.js"]) {
+    fs.copyFileSync(path.join(__dirname, f), path.join(cwd2, f));
+  }
+  fs.mkdirSync(path.join(cwd2, "uploads"), { recursive: true });
+  fs.writeFileSync(path.join(cwd2, "uploads", "person.png"), "fake-png-bytes");
+  fs.writeFileSync(path.join(cwd2, "uploads", "garment.png"), "fake-png-bytes");
+  fs.writeFileSync(path.join(cwd2, "uploads", "ref.mp4"), "fake-mp4-bytes");
+
+  const child2 = spawn(process.execPath, ["server.js"], {
+    cwd: cwd2,
+    env: {
+      ...process.env,
+      PORT: String(port2),
+      H3_CONSOLE_BASE: `http://localhost:${port}`,
+      H3_CONSOLE_TOKEN: "mock-token",
+      ADMIN_PASSWORD,
+      GENERATION_POLL_INTERVAL_MS: "20",
+      GENERATION_TIMEOUT_MS: "150",
+    },
+    stdio: "pipe",
+  });
+  try {
+    await waitForServer(base2);
+    const loginRes = await fetch(`${base2}/api/admin/login`, {
+      method: "POST",
+      headers: { "Content-Type": "application/json" },
+      body: JSON.stringify({ password: ADMIN_PASSWORD }),
+    });
+    const cookie2 = (loginRes.headers.get("set-cookie") || "").split(";")[0];
+    const authed2 = (opts = {}) => ({ ...opts, headers: { "Content-Type": "application/json", Cookie: cookie2, ...opts.headers } });
+
+    const character = await fetch(
+      `${base2}/api/admin/characters`,
+      authed2({ method: "POST", body: JSON.stringify({ name: "小美", imageUrl: "/uploads/person.png", appearanceDescriptor: "black hair" }) })
+    ).then((r) => r.json());
+    const garment = await fetch(
+      `${base2}/api/admin/garments`,
+      authed2({ method: "POST", body: JSON.stringify({ name: "红旗袍", imageUrl: "/uploads/garment.png", appearanceDescriptor: "red qipao" }) })
+    ).then((r) => r.json());
+    await fetch(
+      `${base2}/api/admin/dance-refs`,
+      authed2({ method: "POST", body: JSON.stringify({ name: "拉丁舞", videoUrl: "/uploads/ref.mp4", motionDescriptor: "hip sways" }) })
+    );
+
+    const submitted = await fetch(
+      `${base2}/api/admin/generate`,
+      authed2({ method: "POST", body: JSON.stringify({ characterId: character.id, garmentId: garment.id }) })
+    ).then((r) => r.json());
+
+    // Still "queued" well past the old 200-poll budget's equivalent
+    // wall-clock (200 * 20ms = 4000ms) would have been fine either way at
+    // this poll interval — the real proof is that it survives on "queued"
+    // for longer than a short timeout would allow, then correctly fails
+    // once GENERATION_TIMEOUT_MS (150ms) actually elapses.
+    const start = Date.now();
+    let job;
+    for (let i = 0; i < 50; i++) {
+      job = await fetch(`${base2}/api/admin/generate/${submitted.jobId}`, authed2()).then((r) => r.json());
+      if (job.status === "failed") break;
+      await new Promise((r) => setTimeout(r, 20));
+    }
+    const elapsed = Date.now() - start;
+    assert.equal(job.status, "failed");
+    assert.equal(job.error, "generation timed out");
+    assert.ok(elapsed >= 150, `should not fail before the configured 150ms deadline (took ${elapsed}ms)`);
+  } finally {
+    child2.kill();
+    neverDoneMockH3.close();
+    fs.rmSync(cwd2, { recursive: true, force: true });
+  }
+});
@@ -771,7 +771,15 @@ async function generateDanceVideo(job, opts = {}) {
       job = generationJobs.update(job.id, { h3JobId: submitted.id, status: "generating" });
     }
-    for (let i = 0; i < 200; i++) {
+    // ponytail: H3's 4 GPUs run one sharded model, not 4 independent
+    // workers (H3-CONSOLE.md /v1/status), so a batch of N jobs is served
+    // ~serially — the Nth job in queue_position can sit "queued" for
+    // N * ~700s before it even starts. A fixed poll-count budget doesn't
+    // know about queue depth, so this is wall-clock and generous instead;
+    // upgrade path if batches grow past a couple dozen: read `eta_seconds`/
+    // `queue_position` off the poll response and size the deadline to that.
+    const deadline = Date.now() + (opts.timeoutMs || Number(process.env.GENERATION_TIMEOUT_MS) || 2 * 60 * 60 * 1000);
+    while (Date.now() < deadline) {
       await new Promise((r) => setTimeout(r, pollIntervalMs));
       const polled = await h3ConsoleFetch(`/v1/videos/${job.h3JobId}`);
       if (polled.deleted_at) throw new Error("job was deleted on the H3 console before it finished");
@@ -808,7 +816,9 @@ async function generateDanceVideo(job, opts = {}) {
         generationJobs.update(job.id, { status: "done", generatedVideoId: clip.id });
         return clip;
       }
-      if (polled.status === "error" || polled.status === "failed") throw new Error(polled.error || "generation failed");
+      if (polled.status === "error" || polled.status === "failed" || polled.status === "cancelled") {
+        throw new Error(polled.error || `generation ${polled.status}`);
+      }
     }
     throw new Error("generation timed out");
   } catch (err) {