宜博 / h3livestream

Commit 29f24a7074f777ec2e4b39ffb7a878157dc4193f

Authored by yibo
1 parent 700c1c46

Fix: a transient poll error no longer fails an in-progress generation 

h3ConsoleFetch blindly JSON.parses the response body regardless of
status code; a momentary non-JSON 500 from H3 Console during polling
(GET /v1/videos/:id) rejected the promise, which the poll loop let
propagate straight to the outer catch — permanently marking the job
"failed" even though the job was still fine on H3's side.

Hit this for real today: a 20-job batch's concurrent polling briefly
500'd H3 Console's HTTP layer, and all 20 jobs got marked failed
locally while every one of them was still queued/running on H3
(confirmed via direct GET /v1/videos/:id against the real console).

Now a failed poll attempt is swallowed and retried on the next tick,
bounded by the same wall-clock deadline as everything else (12aff60).
Only a genuine terminal status (done/error/failed/cancelled) or
deleted_at ends the job.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_011Deddn3sM3oQnmvqWSMb4y
Inline Side-by-side
Showing 2 changed files with 129 additions and 1 deletions
dressup-generate.test.js
  View file @29f24a7
... ... @@ -402,3 +402,119 @@ test(&quot;generate: poll loop honors GENERATION_TIMEOUT_MS wall-clock, not a fixed p
402 402 fs.rmSync(cwd2, { recursive: true, force: true });
403 403 }
404 404 });
  405 +
  406 +// Regression for a real incident (2026-09-02): a 20-job batch's concurrent
  407 +// polling briefly 500'd H3 Console (non-JSON "Internal Server Error" body),
  408 +// and every single job got marked "failed" over that one bad poll even
  409 +// though H3 still had them queued/running fine. A transient poll error must
  410 +// not end the job — only a genuine terminal status (done/error/cancelled)
  411 +// or the deadline should.
  412 +test("generate: a transient non-JSON poll response doesn't fail the job, it just retries", async () => {
  413 + let flakyMockH3;
  414 + let pollCount = 0;
  415 + const port = await new Promise((resolve) => {
  416 + flakyMockH3 = http.createServer((req, res) => {
  417 + req.url = req.url.replace(/^\/+/, "/");
  418 + const chunks = [];
  419 + req.on("data", (c) => chunks.push(c));
  420 + req.on("end", () => {
  421 + if (req.method === "POST" && req.url === "/v1/assets") {
  422 + res.writeHead(200, { "Content-Type": "application/json" });
  423 + res.end(JSON.stringify({ id: "asset-x", url: "http://mock-h3/asset/x", kind: "image" }));
  424 + return;
  425 + }
  426 + if (req.method === "POST" && req.url === "/v1/videos") {
  427 + res.writeHead(200, { "Content-Type": "application/json" });
  428 + res.end(JSON.stringify({ id: "job-x" }));
  429 + return;
  430 + }
  431 + if (req.method === "GET" && req.url === "/v1/videos/job-x") {
  432 + pollCount++;
  433 + if (pollCount <= 2) {
  434 + // H3 Console's own flavor of a transient failure: an HTML/text
  435 + // 500 page, not JSON.
  436 + res.writeHead(500, { "Content-Type": "text/html" });
  437 + res.end("Internal Server Error");
  438 + return;
  439 + }
  440 + res.writeHead(200, { "Content-Type": "application/json" });
  441 + res.end(JSON.stringify({ id: "job-x", status: "done", partition: "ref2va", req: { width: 480, height: 832, frames: 240 } }));
  442 + return;
  443 + }
  444 + res.writeHead(404, { "Content-Type": "application/json" });
  445 + res.end(JSON.stringify({ error: "not found in mock" }));
  446 + });
  447 + });
  448 + flakyMockH3.listen(0, () => resolve(flakyMockH3.address().port));
  449 + });
  450 +
  451 + const port2 = 23934 + Math.floor(Math.random() * 1000);
  452 + const base2 = `http://localhost:${port2}`;
  453 + const cwd2 = fs.mkdtempSync(path.join(os.tmpdir(), "dressup-flaky-poll-"));
  454 + for (const f of ["server.js", "ingest.js", "uploads.js", "store.js", "dressup-store.js", "prompt-template.js"]) {
  455 + fs.copyFileSync(path.join(__dirname, f), path.join(cwd2, f));
  456 + }
  457 + fs.mkdirSync(path.join(cwd2, "uploads"), { recursive: true });
  458 + fs.writeFileSync(path.join(cwd2, "uploads", "person.png"), "fake-png-bytes");
  459 + fs.writeFileSync(path.join(cwd2, "uploads", "garment.png"), "fake-png-bytes");
  460 + fs.writeFileSync(path.join(cwd2, "uploads", "ref.mp4"), "fake-mp4-bytes");
  461 +
  462 + const child2 = spawn(process.execPath, ["server.js"], {
  463 + cwd: cwd2,
  464 + env: {
  465 + ...process.env,
  466 + PORT: String(port2),
  467 + H3_CONSOLE_BASE: `http://localhost:${port}`,
  468 + H3_CONSOLE_TOKEN: "mock-token",
  469 + ADMIN_PASSWORD,
  470 + GENERATION_POLL_INTERVAL_MS: "20",
  471 + },
  472 + stdio: "pipe",
  473 + });
  474 + try {
  475 + await waitForServer(base2);
  476 + const loginRes = await fetch(`${base2}/api/admin/login`, {
  477 + method: "POST",
  478 + headers: { "Content-Type": "application/json" },
  479 + body: JSON.stringify({ password: ADMIN_PASSWORD }),
  480 + });
  481 + const cookie2 = (loginRes.headers.get("set-cookie") || "").split(";")[0];
  482 + const authed2 = (opts = {}) => ({ ...opts, headers: { "Content-Type": "application/json", Cookie: cookie2, ...opts.headers } });
  483 +
  484 + const character = await fetch(
  485 + `${base2}/api/admin/characters`,
  486 + authed2({ method: "POST", body: JSON.stringify({ name: "小美", imageUrl: "/uploads/person.png", appearanceDescriptor: "black hair" }) })
  487 + ).then((r) => r.json());
  488 + const garment = await fetch(
  489 + `${base2}/api/admin/garments`,
  490 + authed2({ method: "POST", body: JSON.stringify({ name: "红旗袍", imageUrl: "/uploads/garment.png", appearanceDescriptor: "red qipao" }) })
  491 + ).then((r) => r.json());
  492 + await fetch(
  493 + `${base2}/api/admin/dance-refs`,
  494 + authed2({ method: "POST", body: JSON.stringify({ name: "拉丁舞", videoUrl: "/uploads/ref.mp4", motionDescriptor: "hip sways" }) })
  495 + );
  496 +
  497 + const submitted = await fetch(
  498 + `${base2}/api/admin/generate`,
  499 + authed2({ method: "POST", body: JSON.stringify({ characterId: character.id, garmentId: garment.id }) })
  500 + ).then((r) => r.json());
  501 +
  502 + const job = await waitForJobDoneOn(base2, authed2, submitted.jobId);
  503 + assert.equal(job.status, "done", `expected the job to survive the two flaky polls and finish, got: ${JSON.stringify(job)}`);
  504 + assert.ok(pollCount > 2, "the mock should have been polled past the two flaky responses");
  505 + } finally {
  506 + child2.kill();
  507 + flakyMockH3.close();
  508 + fs.rmSync(cwd2, { recursive: true, force: true });
  509 + }
  510 +});
  511 +
  512 +async function waitForJobDoneOn(base, authed, jobId, timeoutMs = 5000) {
  513 + const deadline = Date.now() + timeoutMs;
  514 + while (Date.now() < deadline) {
  515 + const job = await fetch(`${base}/api/admin/generate/${jobId}`, authed()).then((r) => r.json());
  516 + if (job.status === "done" || job.status === "failed") return job;
  517 + await new Promise((r) => setTimeout(r, 30));
  518 + }
  519 + throw new Error("job did not finish in time");
  520 +}
... ...
server.js
  View file @29f24a7
... ... @@ -781,7 +781,19 @@ async function generateDanceVideo(job, opts = {}) {
781 781 const deadline = Date.now() + (opts.timeoutMs || Number(process.env.GENERATION_TIMEOUT_MS) || 2 * 60 * 60 * 1000);
782 782 while (Date.now() < deadline) {
783 783 await new Promise((r) => setTimeout(r, pollIntervalMs));
784   - const polled = await h3ConsoleFetch(`/v1/videos/${job.h3JobId}`);
  784 + let polled;
  785 + try {
  786 + polled = await h3ConsoleFetch(`/v1/videos/${job.h3JobId}`);
  787 + } catch {
  788 + // A single bad poll (H3 Console hiccup — non-JSON 500 page, dropped
  789 + // connection) isn't a generation failure; the job itself is still
  790 + // fine on H3's side. Skip this tick and retry on the next one,
  791 + // bounded by the same deadline as everything else. Confirmed live
  792 + // 2026-09-02: a 20-job batch's concurrent polling briefly 500'd H3
  793 + // Console, and every job got marked "failed" over a transient
  794 + // response even though H3 still had them queued/running.
  795 + continue;
  796 + }
785 797 if (polled.deleted_at) throw new Error("job was deleted on the H3 console before it finished");
786 798 if (polled.status === "done") {
787 799 const clip = {
... ...