From 29f24a7074f777ec2e4b39ffb7a878157dc4193f Mon Sep 17 00:00:00 2001 From: yibo Date: Wed, 2 Sep 2026 19:16:39 +0800 Subject: [PATCH] Fix: a transient poll error no longer fails an in-progress generation --- dressup-generate.test.js | 116 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ server.js | 14 +++++++++++++- 2 files changed, 129 insertions(+), 1 deletion(-) diff --git a/dressup-generate.test.js b/dressup-generate.test.js index 9367da0..e7b8c0c 100644 --- a/dressup-generate.test.js +++ b/dressup-generate.test.js @@ -402,3 +402,119 @@ test("generate: poll loop honors GENERATION_TIMEOUT_MS wall-clock, not a fixed p fs.rmSync(cwd2, { recursive: true, force: true }); } }); + +// Regression for a real incident (2026-09-02): a 20-job batch's concurrent +// polling briefly 500'd H3 Console (non-JSON "Internal Server Error" body), +// and every single job got marked "failed" over that one bad poll even +// though H3 still had them queued/running fine. A transient poll error must +// not end the job — only a genuine terminal status (done/error/cancelled) +// or the deadline should. +test("generate: a transient non-JSON poll response doesn't fail the job, it just retries", async () => { + let flakyMockH3; + let pollCount = 0; + const port = await new Promise((resolve) => { + flakyMockH3 = http.createServer((req, res) => { + req.url = req.url.replace(/^\/+/, "/"); + const chunks = []; + req.on("data", (c) => chunks.push(c)); + req.on("end", () => { + if (req.method === "POST" && req.url === "/v1/assets") { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ id: "asset-x", url: "http://mock-h3/asset/x", kind: "image" })); + return; + } + if (req.method === "POST" && req.url === "/v1/videos") { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ id: "job-x" })); + return; + } + if (req.method === "GET" && req.url === "/v1/videos/job-x") { + pollCount++; + if (pollCount <= 2) { + // H3 Console's own flavor of a transient failure: an HTML/text + // 500 page, not JSON. + res.writeHead(500, { "Content-Type": "text/html" }); + res.end("Internal Server Error"); + return; + } + res.writeHead(200, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ id: "job-x", status: "done", partition: "ref2va", req: { width: 480, height: 832, frames: 240 } })); + return; + } + res.writeHead(404, { "Content-Type": "application/json" }); + res.end(JSON.stringify({ error: "not found in mock" })); + }); + }); + flakyMockH3.listen(0, () => resolve(flakyMockH3.address().port)); + }); + + const port2 = 23934 + Math.floor(Math.random() * 1000); + const base2 = `http://localhost:${port2}`; + const cwd2 = fs.mkdtempSync(path.join(os.tmpdir(), "dressup-flaky-poll-")); + for (const f of ["server.js", "ingest.js", "uploads.js", "store.js", "dressup-store.js", "prompt-template.js"]) { + fs.copyFileSync(path.join(__dirname, f), path.join(cwd2, f)); + } + fs.mkdirSync(path.join(cwd2, "uploads"), { recursive: true }); + fs.writeFileSync(path.join(cwd2, "uploads", "person.png"), "fake-png-bytes"); + fs.writeFileSync(path.join(cwd2, "uploads", "garment.png"), "fake-png-bytes"); + fs.writeFileSync(path.join(cwd2, "uploads", "ref.mp4"), "fake-mp4-bytes"); + + const child2 = spawn(process.execPath, ["server.js"], { + cwd: cwd2, + env: { + ...process.env, + PORT: String(port2), + H3_CONSOLE_BASE: `http://localhost:${port}`, + H3_CONSOLE_TOKEN: "mock-token", + ADMIN_PASSWORD, + GENERATION_POLL_INTERVAL_MS: "20", + }, + stdio: "pipe", + }); + try { + await waitForServer(base2); + const loginRes = await fetch(`${base2}/api/admin/login`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ password: ADMIN_PASSWORD }), + }); + const cookie2 = (loginRes.headers.get("set-cookie") || "").split(";")[0]; + const authed2 = (opts = {}) => ({ ...opts, headers: { "Content-Type": "application/json", Cookie: cookie2, ...opts.headers } }); + + const character = await fetch( + `${base2}/api/admin/characters`, + authed2({ method: "POST", body: JSON.stringify({ name: "小美", imageUrl: "/uploads/person.png", appearanceDescriptor: "black hair" }) }) + ).then((r) => r.json()); + const garment = await fetch( + `${base2}/api/admin/garments`, + authed2({ method: "POST", body: JSON.stringify({ name: "红旗袍", imageUrl: "/uploads/garment.png", appearanceDescriptor: "red qipao" }) }) + ).then((r) => r.json()); + await fetch( + `${base2}/api/admin/dance-refs`, + authed2({ method: "POST", body: JSON.stringify({ name: "拉丁舞", videoUrl: "/uploads/ref.mp4", motionDescriptor: "hip sways" }) }) + ); + + const submitted = await fetch( + `${base2}/api/admin/generate`, + authed2({ method: "POST", body: JSON.stringify({ characterId: character.id, garmentId: garment.id }) }) + ).then((r) => r.json()); + + const job = await waitForJobDoneOn(base2, authed2, submitted.jobId); + assert.equal(job.status, "done", `expected the job to survive the two flaky polls and finish, got: ${JSON.stringify(job)}`); + assert.ok(pollCount > 2, "the mock should have been polled past the two flaky responses"); + } finally { + child2.kill(); + flakyMockH3.close(); + fs.rmSync(cwd2, { recursive: true, force: true }); + } +}); + +async function waitForJobDoneOn(base, authed, jobId, timeoutMs = 5000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + const job = await fetch(`${base}/api/admin/generate/${jobId}`, authed()).then((r) => r.json()); + if (job.status === "done" || job.status === "failed") return job; + await new Promise((r) => setTimeout(r, 30)); + } + throw new Error("job did not finish in time"); +} diff --git a/server.js b/server.js index b522398..d435631 100644 --- a/server.js +++ b/server.js @@ -781,7 +781,19 @@ async function generateDanceVideo(job, opts = {}) { const deadline = Date.now() + (opts.timeoutMs || Number(process.env.GENERATION_TIMEOUT_MS) || 2 * 60 * 60 * 1000); while (Date.now() < deadline) { await new Promise((r) => setTimeout(r, pollIntervalMs)); - const polled = await h3ConsoleFetch(`/v1/videos/${job.h3JobId}`); + let polled; + try { + polled = await h3ConsoleFetch(`/v1/videos/${job.h3JobId}`); + } catch { + // A single bad poll (H3 Console hiccup — non-JSON 500 page, dropped + // connection) isn't a generation failure; the job itself is still + // fine on H3's side. Skip this tick and retry on the next one, + // bounded by the same deadline as everything else. Confirmed live + // 2026-09-02: a 20-job batch's concurrent polling briefly 500'd H3 + // Console, and every job got marked "failed" over a transient + // response even though H3 still had them queued/running. + continue; + } if (polled.deleted_at) throw new Error("job was deleted on the H3 console before it finished"); if (polled.status === "done") { const clip = { -- libgit2 0.21.4