Commit 2c946632b4fbc0e2f88bebb8a248730cb1b976eb
1 parent
e4ac1cff
feat(device): 快照 stage_digest_media.py（日报媒体同源暂存）
collect_media_oss.py 只上传本地已有原图，微信多半只落缩略图 → oss_url 全空 → HTML 媒体墙塌缩（10-15 起）。本脚本把缩略图暂存到同源公开路由并按 md5 去重， 供 gen_digest_html 渲染。手机上新增，此处仅同步快照。 Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
Showing
1 changed file
with
170 additions
and
0 deletions
device/pixelphone/digest/wechat-group-digest/scripts/stage_digest_media.py
0 → 100644
| 1 | +#!/usr/bin/env python3 | ||
| 2 | +"""日报媒体同源暂存（2026-10-17）——修复日报 HTML 图片/视频不可点预览。 | ||
| 3 | + | ||
| 4 | +背景： | ||
| 5 | + collect_media_oss.py --upload 只上传「本地已有原图」，微信通常只落了缩略图 | ||
| 6 | + （|thumb），原图未下载 ⇒ 全部被跳过 ⇒ oss_url 全空 ⇒ gen_digest_html 的 | ||
| 7 | + build_media_section 渲染零条目，日报只剩文字计数行（10-15 起塌缩）。 | ||
| 8 | + | ||
| 9 | +本脚本： | ||
| 10 | + 1. 读 ~/media_digest_<组>_<日期>.json； | ||
| 11 | + 2. 把每个媒体文件（缩略图即可）用 `su cp` 批量暂存到 termux 可读位置； | ||
| 12 | + 3. 按内容 md5 去重（同图/同视频被 8 群转发只存一份，合并群名）； | ||
| 13 | + 4. 落位 ~/wechat-viewer/digests/media/<组>/<日期>/<md5>.<ext>， | ||
| 14 | + 回填 digest_url=/digest-assets/media/<组>/<日期>/<md5>.<ext>（同源、公开路由）； | ||
| 15 | + 5. 清单原地写回，供 gen_digest_html 渲染。 | ||
| 16 | + | ||
| 17 | +用法: | ||
| 18 | + python3 stage_digest_media.py --date 2026-10-16 --group tgo [--limit N] | ||
| 19 | + | ||
| 20 | +幂等：已有 digest_url 且目标文件存在的条目直接跳过。不依赖 OSS。 | ||
| 21 | +""" | ||
| 22 | +import argparse | ||
| 23 | +import hashlib | ||
| 24 | +import json | ||
| 25 | +import os | ||
| 26 | +import shutil | ||
| 27 | +import subprocess | ||
| 28 | +import sys | ||
| 29 | +import tempfile | ||
| 30 | + | ||
| 31 | +HOME = os.path.expanduser("~") | ||
| 32 | +DIGEST_MEDIA_DIR = os.path.join(HOME, "wechat-viewer", "digests", "media") | ||
| 33 | +SUDO_SH = os.environ.get("PREFIX", "/data/data/com.termux/files/usr") + "/tmp" | ||
| 34 | + | ||
| 35 | + | ||
| 36 | +def media_json_path(group: str, date: str) -> str: | ||
| 37 | + return os.path.join(HOME, f"media_digest_{group}_{date}.json") | ||
| 38 | + | ||
| 39 | + | ||
| 40 | +def batch_su_copy(pairs: list[tuple[str, str]]) -> set[str]: | ||
| 41 | + """一条 su 批量 cp+chmod。返回成功暂存的 dst 集合。""" | ||
| 42 | + if not pairs: | ||
| 43 | + return set() | ||
| 44 | + batch = tempfile.mkstemp(prefix="stage_copy_", suffix=".sh", dir=SUDO_SH)[1] | ||
| 45 | + with open(batch, "w") as f: | ||
| 46 | + f.write("#!/system/bin/sh\n") | ||
| 47 | + for src, dst in pairs: | ||
| 48 | + s = src.replace("'", "'\\''") | ||
| 49 | + d = dst.replace("'", "'\\''") | ||
| 50 | + f.write(f"cp '{s}' '{d}' 2>/dev/null && chmod 644 '{d}'\n") | ||
| 51 | + os.chmod(batch, 0o644) | ||
| 52 | + try: | ||
| 53 | + subprocess.run(["su", "-c", f"sh '{batch}'"], capture_output=True, timeout=300) | ||
| 54 | + finally: | ||
| 55 | + try: | ||
| 56 | + os.unlink(batch) | ||
| 57 | + except OSError: | ||
| 58 | + pass | ||
| 59 | + return {dst for _, dst in pairs if os.path.exists(dst)} | ||
| 60 | + | ||
| 61 | + | ||
| 62 | +def sniff_ext(head: bytes) -> str: | ||
| 63 | + if head.startswith(b"\xff\xd8"): | ||
| 64 | + return ".jpg" | ||
| 65 | + if head.startswith(b"\x89PNG"): | ||
| 66 | + return ".png" | ||
| 67 | + if head[:4] == b"\x1aE\xdf\xa3" or b"ftyp" in head[:12]: | ||
| 68 | + return ".mp4" | ||
| 69 | + return ".jpg" | ||
| 70 | + | ||
| 71 | + | ||
| 72 | +def main(): | ||
| 73 | + ap = argparse.ArgumentParser() | ||
| 74 | + ap.add_argument("--date", required=True) | ||
| 75 | + ap.add_argument("--group", default="tgo") | ||
| 76 | + ap.add_argument("--limit", type=int, default=0) | ||
| 77 | + args = ap.parse_args() | ||
| 78 | + | ||
| 79 | + mpath = media_json_path(args.group, args.date) | ||
| 80 | + if not os.path.exists(mpath): | ||
| 81 | + print(json.dumps({"ok": False, "error": f"清单不存在: {mpath}"}), file=sys.stderr) | ||
| 82 | + return 1 | ||
| 83 | + with open(mpath, encoding="utf-8") as f: | ||
| 84 | + data = json.load(f) | ||
| 85 | + media = data.get("media", []) | ||
| 86 | + | ||
| 87 | + out_dir = os.path.join(DIGEST_MEDIA_DIR, args.group, args.date) | ||
| 88 | + os.makedirs(out_dir, exist_ok=True) | ||
| 89 | + url_prefix = f"/digest-assets/media/{args.group}/{args.date}" | ||
| 90 | + | ||
| 91 | + # 1. 待暂存条目：本地路径解析 + 幂等跳过 | ||
| 92 | + todo = [] # (idx, src_path) | ||
| 93 | + skipped_done = missing = 0 | ||
| 94 | + for i, m in enumerate(media): | ||
| 95 | + if m.get("digest_url") and os.path.exists( | ||
| 96 | + os.path.join(HOME, "wechat-viewer", "digests", | ||
| 97 | + m["digest_url"].removeprefix("/digest-assets/"))): | ||
| 98 | + skipped_done += 1 | ||
| 99 | + continue | ||
| 100 | + lp = (m.get("local_path") or "").split("|")[0] | ||
| 101 | + if not lp or lp.startswith("THUMBNAIL_DIRPATH://"): | ||
| 102 | + missing += 1 | ||
| 103 | + continue | ||
| 104 | + if args.limit and len(todo) >= args.limit: | ||
| 105 | + break | ||
| 106 | + todo.append((i, lp)) | ||
| 107 | + | ||
| 108 | + # 2. su 批量暂存到 termux tmp | ||
| 109 | + tmp_dir = tempfile.mkdtemp(prefix=f"stage_media_{args.group}_{args.date}_", dir=SUDO_SH) | ||
| 110 | + pairs = [(lp, os.path.join(tmp_dir, f"{i}")) for i, lp in todo] | ||
| 111 | + staged = batch_su_copy(pairs) | ||
| 112 | + | ||
| 113 | + # 3. 内容 md5 去重 + 落位 | ||
| 114 | + by_md5: dict[str, dict] = {} # md5 -> media item（保留首条，合并群名） | ||
| 115 | + md5_of: dict[int, str] = {} # idx -> md5 | ||
| 116 | + staged_cnt = 0 | ||
| 117 | + for i, lp in todo: | ||
| 118 | + dst_tmp = os.path.join(tmp_dir, f"{i}") | ||
| 119 | + if dst_tmp not in staged: | ||
| 120 | + missing += 1 | ||
| 121 | + continue | ||
| 122 | + with open(dst_tmp, "rb") as fh: | ||
| 123 | + content = fh.read() | ||
| 124 | + md5 = hashlib.md5(content).hexdigest() | ||
| 125 | + m = media[i] | ||
| 126 | + if md5 in by_md5: | ||
| 127 | + first = by_md5[md5] | ||
| 128 | + for g in (m.get("group") or "").split(","): | ||
| 129 | + g = g.strip() | ||
| 130 | + if g and g not in first["group"]: | ||
| 131 | + first["group"] = f"{first['group']}, {g}" if first["group"] else g | ||
| 132 | + md5_of[i] = md5 | ||
| 133 | + continue | ||
| 134 | + ext = sniff_ext(content[:16]) | ||
| 135 | + final = os.path.join(out_dir, md5 + ext) | ||
| 136 | + if not os.path.exists(final): | ||
| 137 | + shutil.copyfile(dst_tmp, final) | ||
| 138 | + os.chmod(final, 0o644) | ||
| 139 | + m["digest_url"] = f"{url_prefix}/{md5}{ext}" | ||
| 140 | + by_md5[md5] = m | ||
| 141 | + md5_of[i] = md5 | ||
| 142 | + staged_cnt += 1 | ||
| 143 | + shutil.rmtree(tmp_dir, ignore_errors=True) | ||
| 144 | + | ||
| 145 | + # 4. 重复条目指向同一 URL（群名已并入首条，渲染前去重由生成器按 md5 键处理： | ||
| 146 | + # 这里直接把重复条目剔除，首条带合并群名，清单更干净） | ||
| 147 | + kept, seen = [], set() | ||
| 148 | + for i, m in enumerate(media): | ||
| 149 | + h = md5_of.get(i) | ||
| 150 | + if h is None: | ||
| 151 | + kept.append(m) | ||
| 152 | + elif h not in seen: | ||
| 153 | + seen.add(h) | ||
| 154 | + kept.append(m) | ||
| 155 | + data["media"] = kept | ||
| 156 | + data["staged"] = {"ok": True, "staged_new": staged_cnt, "unique": len(by_md5), | ||
| 157 | + "skipped_done": skipped_done, "missing_local": missing, | ||
| 158 | + "total_before": len(media), "total_after": len(kept)} | ||
| 159 | + with open(mpath, "w", encoding="utf-8") as f: | ||
| 160 | + json.dump(data, f, ensure_ascii=False, indent=1, default=str) | ||
| 161 | + | ||
| 162 | + print(json.dumps({"ok": True, "group": args.group, "date": args.date, | ||
| 163 | + "staged_new": staged_cnt, "unique": len(by_md5), | ||
| 164 | + "skipped_done": skipped_done, "missing_local": missing, | ||
| 165 | + "total": len(kept), "dir": out_dir}, ensure_ascii=False)) | ||
| 166 | + return 0 | ||
| 167 | + | ||
| 168 | + | ||
| 169 | +if __name__ == "__main__": | ||
| 170 | + sys.exit(main()) |