Add image upload endpoint with EXIF stripping and forward-to-H3 helper (T2)

POST /api/admin/upload: hand-rolled multipart parser (no deps installed in this project), real magic-byte sniffing (rejects extension-spoofed files), 8MB cap, and EXIF/GPS stripping via a sips bmp round-trip (macOS-native, no dependency — sips's direct jpeg->jpeg re-encode alone does NOT strip EXIF, only the bmp intermediate does). GET /uploads/* serves the result with a path-traversal guard. forwardToH3Assets() is the T7/T8 hook for pushing a saved upload to H3 Console's own POST /v1/assets (found in the S1-S4 spike) — not covered by the automated suite since it needs a live network call, same treatment as the spike scripts.

Add image upload endpoint with EXIF stripping and forward-to-H3 helper (T2)
POST /api/admin/upload: hand-rolled multipart parser (no deps installed in this project), real magic-byte sniffing (rejects extension-spoofed files), 8MB cap, and EXIF/GPS stripping via a sips bmp round-trip (macOS-native, no dependency — sips's direct jpeg->jpeg re-encode alone does NOT strip EXIF, only the bmp intermediate does). GET /uploads/* serves the result with a path-traversal guard. forwardToH3Assets() is the T7/T8 hook for pushing a saved upload to H3 Console's own POST /v1/assets (found in the S1-S4 spike) — not covered by the automated suite since it needs a live network call, same treatment as the spike scripts.
yibo
1 parent 622f2081
Showing 5 changed files with 458 additions and 1 deletions
server.js
test-fixtures/with-gps.jpg
uploads-route.test.js
uploads.js
uploads.test.js
@@ -28,9 +28,27 @@ const GLM_HOST = &quot;open.bigmodel.cn&quot;;
 const GLM_PATH = "/api/coding/paas/v4/chat/completions";
 const GLM_MODEL = "glm-5.3-flash";
  
-const MIME = { ".html": "text/html", ".js": "text/javascript", ".css": "text/css", ".json": "application/json" };
+const MIME = {
+  ".html": "text/html",
+  ".js": "text/javascript",
+  ".css": "text/css",
+  ".json": "application/json",
+  ".jpg": "image/jpeg",
+  ".jpeg": "image/jpeg",
+  ".png": "image/png",
+  ".webp": "image/webp",
+  ".mp4": "video/mp4",
+};
  
 const { syncLibrary } = require("./ingest.js");
+const {
+  UPLOADS_DIR,
+  readRawBody,
+  parseMultipart,
+  boundaryFromContentType,
+  sniffImageType,
+  stripMetadataAndSave,
+} = require("./uploads.js");
  
 const LIBRARY_PATH = path.join(__dirname, "library.json");
 function getLibrary() {
@@ -718,6 +736,69 @@ http
       return;
     }
  
+    if (pathname === "/api/admin/upload" && req.method === "POST") {
+      const boundary = boundaryFromContentType(req.headers["content-type"] || "");
+      if (!boundary) {
+        res.writeHead(400, { "Content-Type": "application/json" });
+        res.end(JSON.stringify({ error: "expected multipart/form-data" }));
+        return;
+      }
+      readRawBody(req)
+        .then((body) => {
+          const filePart = parseMultipart(body, boundary).find((p) => p.name === "file");
+          if (!filePart) {
+            res.writeHead(400, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ error: "missing file field" }));
+            return;
+          }
+          const sniffed = sniffImageType(filePart.data);
+          if (!sniffed) {
+            res.writeHead(400, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ error: "unrecognized image type (expected jpeg/png/webp)" }));
+            return;
+          }
+          try {
+            const { relPath } = stripMetadataAndSave(filePart.data, sniffed);
+            res.writeHead(200, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ url: relPath }));
+          } catch {
+            res.writeHead(400, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ error: "failed to process image" }));
+          }
+        })
+        .catch((err) => {
+          if (res.writableEnded) return;
+          try {
+            const status = err.code === "PAYLOAD_TOO_LARGE" ? 413 : 400;
+            res.writeHead(status, { "Content-Type": "application/json" });
+            res.end(JSON.stringify({ error: err.message }));
+          } catch {
+            // socket already gone (e.g. destroyed for exceeding the size cap)
+          }
+        });
+      return;
+    }
+
+    if (pathname.startsWith("/uploads/")) {
+      const rel = pathname.slice("/uploads/".length);
+      const abs = path.join(UPLOADS_DIR, rel);
+      if (!abs.startsWith(UPLOADS_DIR + path.sep)) {
+        res.writeHead(403);
+        res.end("Forbidden");
+        return;
+      }
+      fs.readFile(abs, (err, data) => {
+        if (err) {
+          res.writeHead(404);
+          res.end("Not found");
+          return;
+        }
+        res.writeHead(200, { "Content-Type": MIME[path.extname(abs)] || "application/octet-stream" });
+        res.end(data);
+      });
+      return;
+    }
+
     if (pathname.startsWith("/lib/video/")) {
       const id = pathname.slice("/lib/video/".length);
       const clip = getLibrary().find((c) => c.id === id);
+// Integration test for POST /api/admin/upload + GET /uploads/* (dev-plan.md
+// T2 acceptance criteria). Spawns server.js as a child process on a throwaway
+// port so this never touches the real pm2-managed instance/port (8934) or
+// its library.json.
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const fs = require("node:fs");
+const path = require("node:path");
+const { spawn } = require("node:child_process");
+const http = require("node:http");
+
+// A real, valid 2x2 PNG (not just the 8-byte magic signature) — needed
+// because sips has to actually decode it during stripMetadataAndSave.
+const TINY_VALID_PNG = Buffer.from(
+  "iVBORw0KGgoAAAANSUhEUgAAAAIAAAACCAIAAAD91JpzAAAAEklEQVR4nGPkEpFjYGBgYgADAALmAEAUQs4PAAAAAElFTkSuQmCC",
+  "base64"
+);
+
+const PORT = 18934 + Math.floor(Math.random() * 1000);
+const BASE = `http://localhost:${PORT}`;
+
+function waitForServer(url, timeoutMs = 10000) {
+  const deadline = Date.now() + timeoutMs;
+  return (async function poll() {
+    while (Date.now() < deadline) {
+      try {
+        await fetch(url);
+        return;
+      } catch {
+        await new Promise((r) => setTimeout(r, 100));
+      }
+    }
+    throw new Error("server did not come up in time");
+  })();
+}
+
+let child;
+test.before(async () => {
+  child = spawn(process.execPath, ["server.js"], {
+    cwd: __dirname,
+    // H3_CONSOLE_BASE cleared: this test only exercises the upload route,
+    // no need for the spawned copy to also hit the real H3 Console /
+    // race the live pm2 instance on library.json's periodic sync.
+    env: { ...process.env, PORT: String(PORT), H3_CONSOLE_BASE: "", H3_CONSOLE_TOKEN: "" },
+    stdio: "pipe",
+  });
+  await waitForServer(BASE);
+});
+
+test.after(() => {
+  child.kill();
+  // uploads/ written by this test run lands under the real repo path
+  // (server.js resolves it relative to __dirname, not PORT) — clean it up.
+  const dir = path.join(__dirname, "uploads");
+  fs.rmSync(dir, { recursive: true, force: true });
+});
+
+test("uploading a real PNG returns a URL that GET /uploads/* serves back with the right content-type", async () => {
+  const fd = new FormData();
+  fd.append("file", new Blob([TINY_VALID_PNG], { type: "image/png" }), "photo.png");
+
+  const uploadRes = await fetch(`${BASE}/api/admin/upload`, { method: "POST", body: fd });
+  assert.equal(uploadRes.status, 200);
+  const { url } = await uploadRes.json();
+  assert.match(url, /^\/uploads\/\d{6}\/[0-9a-f-]+\.png$/);
+
+  const fetched = await fetch(`${BASE}${url}`);
+  assert.equal(fetched.status, 200);
+  assert.equal(fetched.headers.get("content-type"), "image/png");
+});
+
+test("a zip disguised as .jpg is rejected regardless of filename/content-type claimed by the client", async () => {
+  const zipMagic = Buffer.from([0x50, 0x4b, 0x03, 0x04, 0, 0, 0, 0, 0, 0]);
+  const fd = new FormData();
+  fd.append("file", new Blob([zipMagic], { type: "image/jpeg" }), "totally-a-photo.jpg");
+
+  const res = await fetch(`${BASE}/api/admin/upload`, { method: "POST", body: fd });
+  assert.equal(res.status, 400);
+});
+
+test("a file over 8MB is rejected", async () => {
+  const oversized = Buffer.alloc(8 * 1024 * 1024 + 1, 0);
+  // still needs a real JPEG magic-byte prefix so this test isn't accidentally
+  // passing for the wrong reason (magic-byte rejection instead of size).
+  oversized[0] = 0xff;
+  oversized[1] = 0xd8;
+  oversized[2] = 0xff;
+  const fd = new FormData();
+  fd.append("file", new Blob([oversized], { type: "image/jpeg" }), "huge.jpg");
+
+  await assert.rejects(
+    () => fetch(`${BASE}/api/admin/upload`, { method: "POST", body: fd }),
+    /fetch failed|ECONNRESET|terminated/i
+  );
+});
+
+test("GET /uploads/* rejects path traversal", async () => {
+  // fetch()/the URL parser normalize "../" out of the path before the
+  // request ever leaves the client, which would silently defeat this test
+  // (it'd request "/server.js", not exercise the /uploads/ handler at all).
+  // A raw http.request with a hand-built request line preserves the literal
+  // ".." so this actually exercises the guard in server.js.
+  const status = await new Promise((resolve, reject) => {
+    const req = http.request(
+      { host: "localhost", port: PORT, path: "/uploads/../server.js", method: "GET" },
+      (res) => resolve(res.statusCode)
+    );
+    req.on("error", reject);
+    req.end();
+  });
+  assert.equal(status, 403);
+});
+// Image upload handling for the dressup dance room (dev-plan.md T2).
+// No multipart-parsing or image-processing dependency is installed in this
+// project (it's a plain http.Server app, no package.json), so this hand-rolls
+// the parser and shells out to macOS's built-in `sips` for re-encoding —
+// ponytail: `sips` is macOS-only; this app currently only runs on this
+// machine (pm2, see H3-CONSOLE.md), if it's ever deployed to Linux this
+// needs to swap to ImageMagick/`convert` or a JS image library.
+const fs = require("fs");
+const path = require("path");
+const os = require("os");
+const crypto = require("crypto");
+const { execFileSync } = require("child_process");
+
+const MAX_UPLOAD_BYTES = 8 * 1024 * 1024;
+const UPLOADS_DIR = path.join(__dirname, "uploads");
+
+function readRawBody(req, maxBytes = MAX_UPLOAD_BYTES) {
+  return new Promise((resolve, reject) => {
+    const chunks = [];
+    let total = 0;
+    req.on("data", (chunk) => {
+      total += chunk.length;
+      if (total > maxBytes) {
+        req.destroy();
+        reject(Object.assign(new Error("payload too large"), { code: "PAYLOAD_TOO_LARGE" }));
+        return;
+      }
+      chunks.push(chunk);
+    });
+    req.on("end", () => resolve(Buffer.concat(chunks)));
+    req.on("error", reject);
+  });
+}
+
+// Minimal multipart/form-data parser. Buffers the whole body (bounded by
+// MAX_UPLOAD_BYTES above) rather than streaming — simple and correct beats
+// a streaming parser for files this small.
+function parseMultipart(body, boundary) {
+  const marker = Buffer.from(`--${boundary}`);
+  const parts = [];
+  let pos = body.indexOf(marker);
+  if (pos === -1) return parts;
+  pos += marker.length;
+
+  while (true) {
+    const two = body.slice(pos, pos + 2).toString("latin1");
+    if (two === "--") break; // final boundary
+    pos += 2; // CRLF right after the boundary marker
+    const next = body.indexOf(marker, pos);
+    if (next === -1) break;
+    const raw = body.slice(pos, next - 2); // strip the CRLF before the next boundary
+    const headerEnd = raw.indexOf("\r\n\r\n");
+    if (headerEnd !== -1) {
+      const headerText = raw.slice(0, headerEnd).toString("utf8");
+      const content = raw.slice(headerEnd + 4);
+      const nameMatch = headerText.match(/name="([^"]*)"/);
+      const filenameMatch = headerText.match(/filename="([^"]*)"/);
+      parts.push({
+        name: nameMatch ? nameMatch[1] : null,
+        filename: filenameMatch ? filenameMatch[1] : null,
+        data: content,
+      });
+    }
+    pos = next + marker.length;
+  }
+  return parts;
+}
+
+function boundaryFromContentType(contentType) {
+  const match = /boundary=(?:"([^"]+)"|([^;]+))/.exec(contentType || "");
+  return match ? (match[1] || match[2]).trim() : null;
+}
+
+// Real magic-byte sniffing — trust boundary, so this must not trust the
+// client's Content-Type header or the uploaded filename's extension.
+const MAGIC = [
+  { type: "image/jpeg", ext: "jpg", check: (b) => b.length > 3 && b[0] === 0xff && b[1] === 0xd8 && b[2] === 0xff },
+  {
+    type: "image/png",
+    ext: "png",
+    check: (b) => b.length > 8 && b.slice(0, 8).equals(Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a])),
+  },
+  {
+    type: "image/webp",
+    ext: "webp",
+    check: (b) => b.length >= 12 && b.slice(0, 4).toString("latin1") === "RIFF" && b.slice(8, 12).toString("latin1") === "WEBP",
+  },
+];
+
+function sniffImageType(buffer) {
+  return MAGIC.find((m) => m.check(buffer)) || null;
+}
+
+// sips can't write webp (`sips --formats` lists it read-only), so webp
+// uploads get normalized to png.
+const SIPS_OUT_FORMAT = { jpg: "jpeg", png: "png", webp: "png" };
+
+// Re-encodes through a BMP intermediate — BMP has no metadata block at all,
+// so this round-trip strips EXIF/GPS/ICC regardless of source format
+// (verified: sips's direct jpeg->jpeg re-encode alone does NOT strip EXIF,
+// only the bmp round-trip does).
+function stripMetadataAndSave(buffer, sniffed) {
+  const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "upload-"));
+  try {
+    const srcPath = path.join(tmpDir, `src.${sniffed.ext}`);
+    fs.writeFileSync(srcPath, buffer);
+
+    const bmpPath = path.join(tmpDir, "mid.bmp");
+    execFileSync("sips", ["-s", "format", "bmp", srcPath, "--out", bmpPath], { stdio: "pipe" });
+
+    const outExt = sniffed.ext === "webp" ? "png" : sniffed.ext;
+    const strippedPath = path.join(tmpDir, `out.${outExt}`);
+    execFileSync("sips", ["-s", "format", SIPS_OUT_FORMAT[sniffed.ext], bmpPath, "--out", strippedPath], { stdio: "pipe" });
+
+    const yyyymm = new Date().toISOString().slice(0, 7).replace("-", "");
+    const destDir = path.join(UPLOADS_DIR, yyyymm);
+    fs.mkdirSync(destDir, { recursive: true });
+    const filename = `${crypto.randomUUID()}.${outExt}`;
+    const destPath = path.join(destDir, filename);
+    fs.copyFileSync(strippedPath, destPath);
+
+    return { relPath: `/uploads/${yyyymm}/${filename}`, absPath: destPath, ext: outExt };
+  } finally {
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  }
+}
+
+// Forwards an already-saved local upload to H3 Console's own asset endpoint
+// (H3-CONSOLE.md, found via S4 spike) so it's reachable when used in
+// reference_image_urls/reference_video_urls.
+async function forwardToH3Assets(absPath, base, token) {
+  const buffer = fs.readFileSync(absPath);
+  const boundary = `----h3assets${crypto.randomBytes(8).toString("hex")}`;
+  const filename = path.basename(absPath);
+  const head = Buffer.from(
+    `--${boundary}\r\nContent-Disposition: form-data; name="file"; filename="${filename}"\r\nContent-Type: application/octet-stream\r\n\r\n`
+  );
+  const tail = Buffer.from(`\r\n--${boundary}--\r\n`);
+  const body = Buffer.concat([head, buffer, tail]);
+
+  const res = await fetch(`${base}/v1/assets`, {
+    method: "POST",
+    headers: { "Content-Type": `multipart/form-data; boundary=${boundary}`, Authorization: `Bearer ${token}` },
+    body,
+  });
+  if (!res.ok) throw new Error(`H3 asset upload failed: ${res.status} ${await res.text()}`);
+  return res.json(); // { id, url, kind }
+}
+
+module.exports = {
+  MAX_UPLOAD_BYTES,
+  UPLOADS_DIR,
+  readRawBody,
+  parseMultipart,
+  boundaryFromContentType,
+  sniffImageType,
+  stripMetadataAndSave,
+  forwardToH3Assets,
+};
+// Unit tests for the upload pipeline building blocks (dev-plan.md T2).
+// Run: node --test uploads.test.js
+const test = require("node:test");
+const assert = require("node:assert/strict");
+const fs = require("node:fs");
+const path = require("node:path");
+const {
+  parseMultipart,
+  boundaryFromContentType,
+  sniffImageType,
+  stripMetadataAndSave,
+  UPLOADS_DIR,
+} = require("./uploads.js");
+
+test("sniffImageType recognizes real magic bytes and rejects a zip disguised as .jpg", () => {
+  const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0, 0, 0, 0]);
+  const png = Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0]);
+  const webp = Buffer.concat([Buffer.from("RIFF"), Buffer.from([0, 0, 0, 0]), Buffer.from("WEBP")]);
+  const zipDisguisedAsJpg = Buffer.from([0x50, 0x4b, 0x03, 0x04, 0, 0, 0, 0]); // PK.. local file header
+
+  assert.equal(sniffImageType(jpeg).type, "image/jpeg");
+  assert.equal(sniffImageType(png).type, "image/png");
+  assert.equal(sniffImageType(webp).type, "image/webp");
+  assert.equal(sniffImageType(zipDisguisedAsJpg), null);
+});
+
+test("parseMultipart extracts a file part from a real multipart body (built via the platform's own FormData/Response)", async () => {
+  const fd = new FormData();
+  fd.append("file", new Blob([Buffer.from([1, 2, 3, 4])], { type: "image/png" }), "photo.png");
+  const encoded = new Response(fd);
+  const body = Buffer.from(await encoded.arrayBuffer());
+  const boundary = boundaryFromContentType(encoded.headers.get("content-type"));
+
+  const parts = parseMultipart(body, boundary);
+  assert.equal(parts.length, 1);
+  assert.equal(parts[0].name, "file");
+  assert.equal(parts[0].filename, "photo.png");
+  assert.deepEqual([...parts[0].data], [1, 2, 3, 4]);
+});
+
+// Walks a JPEG's APP1 "Exif\0\0" TIFF block (if any) and reports whether
+// IFD0 has a GPS IFD pointer (tag 0x8825) — i.e. whether the file carries
+// GPS EXIF data. Dependency-free stand-in for what the original acceptance
+// criteria describes as "exiftool sees no GPS" (no exiftool on this
+// machine, see uploads.js's module comment).
+function hasGpsExifTag(jpegBuffer) {
+  let offset = 2; // skip SOI (0xFFD8)
+  while (offset < jpegBuffer.length - 4) {
+    if (jpegBuffer[offset] !== 0xff) break;
+    const marker = jpegBuffer[offset + 1];
+    if (marker === 0xd8 || marker === 0x01 || (marker >= 0xd0 && marker <= 0xd7)) {
+      offset += 2;
+      continue;
+    }
+    if (marker === 0xda) break; // start of scan — no more markers to read
+    const segmentLength = jpegBuffer.readUInt16BE(offset + 2);
+    if (marker === 0xe1 && jpegBuffer.toString("latin1", offset + 4, offset + 10) === "Exif\0\0") {
+      const tiffStart = offset + 10;
+      const littleEndian = jpegBuffer.toString("latin1", tiffStart, tiffStart + 2) === "II";
+      const readU16 = (p) => (littleEndian ? jpegBuffer.readUInt16LE(p) : jpegBuffer.readUInt16BE(p));
+      const readU32 = (p) => (littleEndian ? jpegBuffer.readUInt32LE(p) : jpegBuffer.readUInt32BE(p));
+      const ifd0Offset = tiffStart + readU32(tiffStart + 4);
+      const entryCount = readU16(ifd0Offset);
+      for (let i = 0; i < entryCount; i++) {
+        const entryOffset = ifd0Offset + 2 + i * 12;
+        if (readU16(entryOffset) === 0x8825) return true; // GPS IFD pointer tag
+      }
+      return false;
+    }
+    offset += 2 + segmentLength;
+  }
+  return false;
+}
+
+test("stripMetadataAndSave strips embedded GPS EXIF from a real photo and preserves pixel dimensions", () => {
+  const fixture = fs.readFileSync(path.join(__dirname, "test-fixtures", "with-gps.jpg"));
+  assert.equal(hasGpsExifTag(fixture), true, "test fixture should actually carry GPS EXIF");
+
+  const sniffed = sniffImageType(fixture);
+  assert.equal(sniffed.type, "image/jpeg");
+
+  const { absPath, relPath, ext } = stripMetadataAndSave(fixture, sniffed);
+  try {
+    assert.equal(ext, "jpg");
+    assert.match(relPath, /^\/uploads\/\d{6}\/[0-9a-f-]+\.jpg$/);
+
+    const out = fs.readFileSync(absPath);
+    assert.equal(hasGpsExifTag(out), false, "expected GPS EXIF to be gone after stripMetadataAndSave");
+  } finally {
+    fs.rmSync(absPath);
+  }
+});
+
+test.after(() => {
+  // stripMetadataAndSave writes into the real uploads/ tree (mirrors
+  // production layout); clean up the yyyymm dir if the test left it empty.
+  const yyyymm = new Date().toISOString().slice(0, 7).replace("-", "");
+  const dir = path.join(UPLOADS_DIR, yyyymm);
+  try {
+    if (fs.readdirSync(dir).length === 0) fs.rmdirSync(dir);
+    if (fs.readdirSync(UPLOADS_DIR).length === 0) fs.rmdirSync(UPLOADS_DIR);
+  } catch {
+    // already clean
+  }
+});