宜博 / h3livestream

// Integration test for the T8 single-generation orchestration
// (POST/GET /api/admin/generate): submit -> forward assets to H3 /v1/assets
// -> POST /v1/videos -> poll -> land a clip in library.json with
// characterId/garmentId/refDanceVideoId. Runs server.js against a mock H3
// Console (not the real one) so it's fast and offline.
const test = require("node:test");
const assert = require("node:assert/strict");
const fs = require("node:fs");
const path = require("node:path");
const os = require("node:os");
const http = require("node:http");
const { spawn } = require("node:child_process");

const PORT = 21934 + Math.floor(Math.random() * 1000);
const BASE = `http://localhost:${PORT}`;
const ADMIN_PASSWORD = "test-password-only-for-this-run";
let cookie;
let child;
let cwd;
let mockH3;
let mockH3Port;
let videoPollCounts; // h3JobId -> number of GETs seen, so we can return "generating" once then "done"

function waitForServer(url, timeoutMs = 10000) {
  const deadline = Date.now() + timeoutMs;
  return (async function poll() {
    while (Date.now() < deadline) {
      try {
        await fetch(url);
        return;
      } catch {
        await new Promise((r) => setTimeout(r, 100));
      }
    }
    throw new Error("server did not come up in time");
  })();
}

function authed(opts = {}) {
  return { ...opts, headers: { "Content-Type": "application/json", Cookie: cookie, ...opts.headers } };
}

// Minimal stand-in for H3 Console: enough of /v1/assets, /v1/videos and
// /v1/videos/:id to exercise generateDanceVideo end to end.
function startMockH3() {
  videoPollCounts = new Map();
  let nextId = 1;
  return new Promise((resolve) => {
    mockH3 = http.createServer((req, res) => {
      // h3ConsoleFetch builds paths as `${new URL(H3_CONSOLE_BASE).pathname}${path}`;
      // with a bare "http://host:port" base (pathname "/") that yields a
      // leading "//v1/...", which real deployments avoid by pointing
      // H3_CONSOLE_BASE at a sub-path (e.g. ".../console"). Normalize here
      // so the mock matches routes regardless.
      req.url = req.url.replace(/^\/+/, "/");
      const chunks = [];
      req.on("data", (c) => chunks.push(c));
      req.on("end", () => {
        if (req.method === "POST" && req.url === "/v1/assets") {
          const id = `asset-${nextId++}`;
          res.writeHead(200, { "Content-Type": "application/json" });
          res.end(JSON.stringify({ id, url: `http://mock-h3/asset/${id}`, kind: "image" }));
          return;
        }
        if (req.method === "POST" && req.url === "/v1/videos") {
          const id = `job-${nextId++}`;
          res.writeHead(200, { "Content-Type": "application/json" });
          res.end(JSON.stringify({ id }));
          return;
        }
        const pollMatch = req.method === "GET" && req.url.match(/^\/v1\/videos\/(.+)$/);
        if (pollMatch) {
          const id = pollMatch[1];
          const seen = (videoPollCounts.get(id) || 0) + 1;
          videoPollCounts.set(id, seen);
          if (seen < 2) {
            res.writeHead(200, { "Content-Type": "application/json" });
            res.end(JSON.stringify({ id, status: "generating" }));
            return;
          }
          res.writeHead(200, { "Content-Type": "application/json" });
          res.end(
            JSON.stringify({
              id,
              status: "done",
              partition: "ref2va",
              req: { width: 480, height: 832, frames: 240 },
            })
          );
          return;
        }
        res.writeHead(404, { "Content-Type": "application/json" });
        res.end(JSON.stringify({ error: "not found in mock" }));
      });
    });
    mockH3.listen(0, () => {
      mockH3Port = mockH3.address().port;
      resolve();
    });
  });
}

test.before(async () => {
  await startMockH3();

  cwd = fs.mkdtempSync(path.join(os.tmpdir(), "dressup-generate-"));
  for (const f of ["server.js", "ingest.js", "uploads.js", "store.js", "dressup-store.js", "prompt-template.js", "db.js"]) {
    fs.copyFileSync(path.join(__dirname, f), path.join(cwd, f));
  }
  // ensureH3AssetUrl reads these off disk (path.join(__dirname, record.imageUrl))
  // before forwarding to /v1/assets, so they need to actually exist.
  fs.mkdirSync(path.join(cwd, "uploads"), { recursive: true });
  fs.writeFileSync(path.join(cwd, "uploads", "person.png"), "fake-png-bytes");
  fs.writeFileSync(path.join(cwd, "uploads", "garment.png"), "fake-png-bytes");
  fs.writeFileSync(path.join(cwd, "uploads", "ref.mp4"), "fake-mp4-bytes");

  child = spawn(process.execPath, ["server.js"], {
    cwd,
    env: {
      ...process.env,
      PORT: String(PORT),
      H3_CONSOLE_BASE: `http://localhost:${mockH3Port}`,
      H3_CONSOLE_TOKEN: "mock-token",
      ADMIN_PASSWORD,
      GENERATION_POLL_INTERVAL_MS: "20",
    },
    stdio: "pipe",
  });
  await waitForServer(BASE);

  const loginRes = await fetch(`${BASE}/api/admin/login`, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ password: ADMIN_PASSWORD }),
  });
  cookie = (loginRes.headers.get("set-cookie") || "").split(";")[0];
});

test.after(() => {
  child.kill();
  mockH3.close();
  fs.rmSync(cwd, { recursive: true, force: true });
});

async function waitForJobDone(jobId, timeoutMs = 5000) {
  const deadline = Date.now() + timeoutMs;
  while (Date.now() < deadline) {
    const job = await fetch(`${BASE}/api/admin/generate/${jobId}`, authed()).then((r) => r.json());
    if (job.status === "done" || job.status === "failed") return job;
    await new Promise((r) => setTimeout(r, 30));
  }
  throw new Error("job did not finish in time");
}

test("generate: full happy path lands a clip with character/garment/dance-ref traceability", async () => {
  const character = await fetch(
    `${BASE}/api/admin/characters`,
    authed({ method: "POST", body: JSON.stringify({ name: "小美", imageUrl: "/uploads/person.png", appearanceDescriptor: "black hair" }) })
  ).then((r) => r.json());
  const garment = await fetch(
    `${BASE}/api/admin/garments`,
    authed({ method: "POST", body: JSON.stringify({ name: "红旗袍", imageUrl: "/uploads/garment.png", appearanceDescriptor: "red qipao" }) })
  ).then((r) => r.json());
  const danceRef = await fetch(
    `${BASE}/api/admin/dance-refs`,
    authed({ method: "POST", body: JSON.stringify({ name: "拉丁舞", videoUrl: "/uploads/ref.mp4", motionDescriptor: "hip sways" }) })
  ).then((r) => r.json());

  const submitRes = await fetch(
    `${BASE}/api/admin/generate`,
    authed({ method: "POST", body: JSON.stringify({ characterId: character.id, garmentId: garment.id }) })
  );
  assert.equal(submitRes.status, 200);
  const submitted = await submitRes.json();
  assert.equal(submitted.cached, false);
  assert.ok(submitted.jobId);

  const job = await waitForJobDone(submitted.jobId);
  assert.equal(job.status, "done");
  assert.equal(job.refDanceVideoId, danceRef.id); // auto-picked since none was given
  assert.ok(job.generatedVideoId);

  const library = await fetch(`${BASE}/api/library`).then((r) => r.json());
  const clip = library.find((c) => c.id === job.generatedVideoId);
  assert.ok(clip, "generated clip should show up in /api/library");
  assert.equal(clip.characterId, character.id);
  assert.equal(clip.garmentId, garment.id);
  assert.equal(clip.refDanceVideoId, danceRef.id);
  assert.equal(clip.channel, "dance");
  // ingest.js's syncLibrary() only carries a library entry forward as-is
  // when `edited` is true; otherwise its periodic auto-sync (every 5 min,
  // server.js) rebuilds the record from the raw H3 job and silently drops
  // channel/characterId/garmentId/refDanceVideoId — breaking both the M.7
  // dedupe cache and pickNextClip()'s dance-channel exclusion.
  assert.equal(clip.edited, true, "dance clip must be edited:true or the next library sync wipes its traceability fields");

  // T10's admin.html traceability column joins clip.characterId/garmentId/
  // refDanceVideoId back against the three admin lists to show "谁的脸 +
  // 谁的衣服 + 哪条参考视频" — confirm that join actually resolves to the
  // real names, not just matching ids.
  const [charList, garmentList, danceRefList] = await Promise.all([
    fetch(`${BASE}/api/admin/characters`, authed()).then((r) => r.json()),
    fetch(`${BASE}/api/admin/garments`, authed()).then((r) => r.json()),
    fetch(`${BASE}/api/admin/dance-refs`, authed()).then((r) => r.json()),
  ]);
  assert.equal(charList.find((x) => x.id === clip.characterId).name, "小美");
  assert.equal(garmentList.find((x) => x.id === clip.garmentId).name, "红旗袍");
  assert.equal(danceRefList.find((x) => x.id === clip.refDanceVideoId).name, "拉丁舞");

  // Re-submitting the same character x garment combo should hit the M.7
  // dedupe cache instead of generating again.
  const repeatRes = await fetch(
    `${BASE}/api/admin/generate`,
    authed({ method: "POST", body: JSON.stringify({ characterId: character.id, garmentId: garment.id }) })
  );
  const repeat = await repeatRes.json();
  assert.equal(repeat.cached, true);
  assert.equal(repeat.clip.id, clip.id);
});

test("generate: rejects unknown character/garment ids", async () => {
  const res = await fetch(`${BASE}/api/admin/generate`, authed({ method: "POST", body: JSON.stringify({ characterId: "nope", garmentId: "nope" }) }));
  assert.equal(res.status, 400);
});

test("generate: 404 for unknown job id", async () => {
  const res = await fetch(`${BASE}/api/admin/generate/nonexistent`, authed());
  assert.equal(res.status, 404);
});

async function waitForBatchDone(batchId, expectedJobCount, timeoutMs = 5000) {
  const deadline = Date.now() + timeoutMs;
  while (Date.now() < deadline) {
    const jobs = await fetch(`${BASE}/api/admin/batch-gen/status?batchId=${batchId}`, authed()).then((r) => r.json());
    if (jobs.length >= expectedJobCount && jobs.every((j) => j.status === "done" || j.status === "failed")) return jobs;
    await new Promise((r) => setTimeout(r, 30));
  }
  throw new Error("batch did not finish in time");
}

test("batch-gen: 1 character x N garments expands to N jobs, progress trackable by batchId, dedupe skips repeats", async () => {
  const character = await fetch(
    `${BASE}/api/admin/characters`,
    authed({ method: "POST", body: JSON.stringify({ name: "批量小美", imageUrl: "/uploads/person.png", appearanceDescriptor: "black hair" }) })
  ).then((r) => r.json());
  const garmentNames = ["红旗袍", "校服", "婚纱"];
  const garments = [];
  for (const name of garmentNames) {
    garments.push(
      await fetch(
        `${BASE}/api/admin/garments`,
        authed({ method: "POST", body: JSON.stringify({ name, imageUrl: "/uploads/garment.png", appearanceDescriptor: name }) })
      ).then((r) => r.json())
    );
  }
  await fetch(
    `${BASE}/api/admin/dance-refs`,
    authed({ method: "POST", body: JSON.stringify({ name: "批量拉丁舞", videoUrl: "/uploads/ref.mp4", motionDescriptor: "hip sways" }) })
  );

  const batchRes = await fetch(
    `${BASE}/api/admin/batch-gen`,
    authed({ method: "POST", body: JSON.stringify({ characterId: character.id, garmentIds: garments.map((g) => g.id) }) })
  );
  assert.equal(batchRes.status, 200);
  const batch = await batchRes.json();
  assert.equal(batch.results.length, 3);
  assert.ok(batch.results.every((r) => r.cached === false && r.jobId));

  const jobs = await waitForBatchDone(batch.batchId, 3);
  assert.equal(jobs.length, 3);
  assert.ok(jobs.every((j) => j.status === "done"));
  assert.deepEqual(
    jobs.map((j) => j.garmentId).sort(),
    garments.map((g) => g.id).sort()
  );

  // Resubmitting the same character x garments batch should skip every
  // combo via the M.7 cache instead of generating again.
  const repeatRes = await fetch(
    `${BASE}/api/admin/batch-gen`,
    authed({ method: "POST", body: JSON.stringify({ characterId: character.id, garmentIds: garments.map((g) => g.id) }) })
  );
  const repeat = await repeatRes.json();
  assert.ok(repeat.results.every((r) => r.cached === true));
});

test("batch-gen: rejects missing characterId or empty garmentIds", async () => {
  const res = await fetch(`${BASE}/api/admin/batch-gen`, authed({ method: "POST", body: JSON.stringify({ characterId: "nope", garmentIds: [] }) }));
  assert.equal(res.status, 400);
});

// Regression for a real incident (2026-09-02): a 9-job batch against H3's
// single sharded-model backend serializes almost entirely — the last job's
// queue wait alone blew past the old fixed 200-poll budget (~1000s) even
// though nothing had actually failed. generateDanceVideo's poll loop is now
// wall-clock (GENERATION_TIMEOUT_MS), independent of GENERATION_POLL_INTERVAL_MS,
// so a job stuck "queued" must be given the full configured budget before
// it's declared failed — not bailed out after a fixed number of polls.
test("generate: poll loop honors GENERATION_TIMEOUT_MS wall-clock, not a fixed poll count", async () => {
  let neverDoneMockH3;
  const port = await new Promise((resolve) => {
    neverDoneMockH3 = http.createServer((req, res) => {
      req.url = req.url.replace(/^\/+/, "/");
      const chunks = [];
      req.on("data", (c) => chunks.push(c));
      req.on("end", () => {
        if (req.method === "POST" && req.url === "/v1/assets") {
          res.writeHead(200, { "Content-Type": "application/json" });
          res.end(JSON.stringify({ id: "asset-x", url: "http://mock-h3/asset/x", kind: "image" }));
          return;
        }
        if (req.method === "POST" && req.url === "/v1/videos") {
          res.writeHead(200, { "Content-Type": "application/json" });
          res.end(JSON.stringify({ id: "job-x" }));
          return;
        }
        if (req.method === "GET" && req.url === "/v1/videos/job-x") {
          // Always "queued" — simulates sitting behind a deep batch queue.
          res.writeHead(200, { "Content-Type": "application/json" });
          res.end(JSON.stringify({ id: "job-x", status: "queued" }));
          return;
        }
        res.writeHead(404, { "Content-Type": "application/json" });
        res.end(JSON.stringify({ error: "not found in mock" }));
      });
    });
    neverDoneMockH3.listen(0, () => resolve(neverDoneMockH3.address().port));
  });

  const port2 = 22934 + Math.floor(Math.random() * 1000);
  const base2 = `http://localhost:${port2}`;
  const cwd2 = fs.mkdtempSync(path.join(os.tmpdir(), "dressup-timeout-"));
  for (const f of ["server.js", "ingest.js", "uploads.js", "store.js", "dressup-store.js", "prompt-template.js", "db.js"]) {
    fs.copyFileSync(path.join(__dirname, f), path.join(cwd2, f));
  }
  fs.mkdirSync(path.join(cwd2, "uploads"), { recursive: true });
  fs.writeFileSync(path.join(cwd2, "uploads", "person.png"), "fake-png-bytes");
  fs.writeFileSync(path.join(cwd2, "uploads", "garment.png"), "fake-png-bytes");
  fs.writeFileSync(path.join(cwd2, "uploads", "ref.mp4"), "fake-mp4-bytes");

  const child2 = spawn(process.execPath, ["server.js"], {
    cwd: cwd2,
    env: {
      ...process.env,
      PORT: String(port2),
      H3_CONSOLE_BASE: `http://localhost:${port}`,
      H3_CONSOLE_TOKEN: "mock-token",
      ADMIN_PASSWORD,
      GENERATION_POLL_INTERVAL_MS: "20",
      GENERATION_TIMEOUT_MS: "150",
    },
    stdio: "pipe",
  });
  try {
    await waitForServer(base2);
    const loginRes = await fetch(`${base2}/api/admin/login`, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ password: ADMIN_PASSWORD }),
    });
    const cookie2 = (loginRes.headers.get("set-cookie") || "").split(";")[0];
    const authed2 = (opts = {}) => ({ ...opts, headers: { "Content-Type": "application/json", Cookie: cookie2, ...opts.headers } });

    const character = await fetch(
      `${base2}/api/admin/characters`,
      authed2({ method: "POST", body: JSON.stringify({ name: "小美", imageUrl: "/uploads/person.png", appearanceDescriptor: "black hair" }) })
    ).then((r) => r.json());
    const garment = await fetch(
      `${base2}/api/admin/garments`,
      authed2({ method: "POST", body: JSON.stringify({ name: "红旗袍", imageUrl: "/uploads/garment.png", appearanceDescriptor: "red qipao" }) })
    ).then((r) => r.json());
    await fetch(
      `${base2}/api/admin/dance-refs`,
      authed2({ method: "POST", body: JSON.stringify({ name: "拉丁舞", videoUrl: "/uploads/ref.mp4", motionDescriptor: "hip sways" }) })
    );

    const submitted = await fetch(
      `${base2}/api/admin/generate`,
      authed2({ method: "POST", body: JSON.stringify({ characterId: character.id, garmentId: garment.id }) })
    ).then((r) => r.json());

    // Still "queued" well past the old 200-poll budget's equivalent
    // wall-clock (200 * 20ms = 4000ms) would have been fine either way at
    // this poll interval — the real proof is that it survives on "queued"
    // for longer than a short timeout would allow, then correctly fails
    // once GENERATION_TIMEOUT_MS (150ms) actually elapses.
    const start = Date.now();
    let job;
    for (let i = 0; i < 50; i++) {
      job = await fetch(`${base2}/api/admin/generate/${submitted.jobId}`, authed2()).then((r) => r.json());
      if (job.status === "failed") break;
      await new Promise((r) => setTimeout(r, 20));
    }
    const elapsed = Date.now() - start;
    assert.equal(job.status, "failed");
    assert.equal(job.error, "generation timed out");
    assert.ok(elapsed >= 150, `should not fail before the configured 150ms deadline (took ${elapsed}ms)`);
  } finally {
    child2.kill();
    neverDoneMockH3.close();
    fs.rmSync(cwd2, { recursive: true, force: true });
  }
});

// Regression for a real incident (2026-09-02): a 20-job batch's concurrent
// polling briefly 500'd H3 Console (non-JSON "Internal Server Error" body),
// and every single job got marked "failed" over that one bad poll even
// though H3 still had them queued/running fine. A transient poll error must
// not end the job — only a genuine terminal status (done/error/cancelled)
// or the deadline should.
test("generate: a transient non-JSON poll response doesn't fail the job, it just retries", async () => {
  let flakyMockH3;
  let pollCount = 0;
  const port = await new Promise((resolve) => {
    flakyMockH3 = http.createServer((req, res) => {
      req.url = req.url.replace(/^\/+/, "/");
      const chunks = [];
      req.on("data", (c) => chunks.push(c));
      req.on("end", () => {
        if (req.method === "POST" && req.url === "/v1/assets") {
          res.writeHead(200, { "Content-Type": "application/json" });
          res.end(JSON.stringify({ id: "asset-x", url: "http://mock-h3/asset/x", kind: "image" }));
          return;
        }
        if (req.method === "POST" && req.url === "/v1/videos") {
          res.writeHead(200, { "Content-Type": "application/json" });
          res.end(JSON.stringify({ id: "job-x" }));
          return;
        }
        if (req.method === "GET" && req.url === "/v1/videos/job-x") {
          pollCount++;
          if (pollCount <= 2) {
            // H3 Console's own flavor of a transient failure: an HTML/text
            // 500 page, not JSON.
            res.writeHead(500, { "Content-Type": "text/html" });
            res.end("Internal Server Error");
            return;
          }
          res.writeHead(200, { "Content-Type": "application/json" });
          res.end(JSON.stringify({ id: "job-x", status: "done", partition: "ref2va", req: { width: 480, height: 832, frames: 240 } }));
          return;
        }
        res.writeHead(404, { "Content-Type": "application/json" });
        res.end(JSON.stringify({ error: "not found in mock" }));
      });
    });
    flakyMockH3.listen(0, () => resolve(flakyMockH3.address().port));
  });

  const port2 = 23934 + Math.floor(Math.random() * 1000);
  const base2 = `http://localhost:${port2}`;
  const cwd2 = fs.mkdtempSync(path.join(os.tmpdir(), "dressup-flaky-poll-"));
  for (const f of ["server.js", "ingest.js", "uploads.js", "store.js", "dressup-store.js", "prompt-template.js", "db.js"]) {
    fs.copyFileSync(path.join(__dirname, f), path.join(cwd2, f));
  }
  fs.mkdirSync(path.join(cwd2, "uploads"), { recursive: true });
  fs.writeFileSync(path.join(cwd2, "uploads", "person.png"), "fake-png-bytes");
  fs.writeFileSync(path.join(cwd2, "uploads", "garment.png"), "fake-png-bytes");
  fs.writeFileSync(path.join(cwd2, "uploads", "ref.mp4"), "fake-mp4-bytes");

  const child2 = spawn(process.execPath, ["server.js"], {
    cwd: cwd2,
    env: {
      ...process.env,
      PORT: String(port2),
      H3_CONSOLE_BASE: `http://localhost:${port}`,
      H3_CONSOLE_TOKEN: "mock-token",
      ADMIN_PASSWORD,
      GENERATION_POLL_INTERVAL_MS: "20",
    },
    stdio: "pipe",
  });
  try {
    await waitForServer(base2);
    const loginRes = await fetch(`${base2}/api/admin/login`, {
      method: "POST",
      headers: { "Content-Type": "application/json" },
      body: JSON.stringify({ password: ADMIN_PASSWORD }),
    });
    const cookie2 = (loginRes.headers.get("set-cookie") || "").split(";")[0];
    const authed2 = (opts = {}) => ({ ...opts, headers: { "Content-Type": "application/json", Cookie: cookie2, ...opts.headers } });

    const character = await fetch(
      `${base2}/api/admin/characters`,
      authed2({ method: "POST", body: JSON.stringify({ name: "小美", imageUrl: "/uploads/person.png", appearanceDescriptor: "black hair" }) })
    ).then((r) => r.json());
    const garment = await fetch(
      `${base2}/api/admin/garments`,
      authed2({ method: "POST", body: JSON.stringify({ name: "红旗袍", imageUrl: "/uploads/garment.png", appearanceDescriptor: "red qipao" }) })
    ).then((r) => r.json());
    await fetch(
      `${base2}/api/admin/dance-refs`,
      authed2({ method: "POST", body: JSON.stringify({ name: "拉丁舞", videoUrl: "/uploads/ref.mp4", motionDescriptor: "hip sways" }) })
    );

    const submitted = await fetch(
      `${base2}/api/admin/generate`,
      authed2({ method: "POST", body: JSON.stringify({ characterId: character.id, garmentId: garment.id }) })
    ).then((r) => r.json());

    const job = await waitForJobDoneOn(base2, authed2, submitted.jobId);
    assert.equal(job.status, "done", `expected the job to survive the two flaky polls and finish, got: ${JSON.stringify(job)}`);
    assert.ok(pollCount > 2, "the mock should have been polled past the two flaky responses");
  } finally {
    child2.kill();
    flakyMockH3.close();
    fs.rmSync(cwd2, { recursive: true, force: true });
  }
});

async function waitForJobDoneOn(base, authed, jobId, timeoutMs = 5000) {
  const deadline = Date.now() + timeoutMs;
  while (Date.now() < deadline) {
    const job = await fetch(`${base}/api/admin/generate/${jobId}`, authed()).then((r) => r.json());
    if (job.status === "done" || job.status === "failed") return job;
    await new Promise((r) => setTimeout(r, 30));
  }
  throw new Error("job did not finish in time");
}