batch_summarize.py
3.51 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
#!/data/data/com.termux/files/usr/bin/python3 -u
"""批量文章摘要 — 最多 2 篇并行，逐批完成"""
import os, requests, time, sys
from datetime import datetime
BASE = "http://127.0.0.1:5001"
BATCH_SIZE = 2
POLL_INTERVAL = 8 # 每 8 秒检查一次
JOB_TIMEOUT = 360 # 单个任务超时 6 分钟
LOG = "/data/data/com.termux/files/home/wechat-viewer/batch_summarize.log"
# Ensure HOME is set so hermes subprocess can find ~/pixelphone
if not os.environ.get("HOME"):
os.environ["HOME"] = "/data/data/com.termux/files/home"
if not os.environ.get("HERMES_HOME"):
os.environ["HERMES_HOME"] = os.path.join(os.environ["HOME"], "pixelphone")
def log(msg):
ts = datetime.now().strftime("%H:%M:%S")
line = f"[{ts}] {msg}"
print(line, flush=True)
with open(LOG, "a") as f:
f.write(line + "\n")
def get_unsummarized(limit=200):
r = requests.get(f"{BASE}/api/articles/unsummarized?limit={limit}", timeout=10)
return r.json().get("data", [])
def submit_summary(article):
try:
r = requests.post(f"{BASE}/api/articles/summarize", json={
"url": article["url"], "title": article["title"]
}, timeout=10)
j = r.json()
if j.get("ok") and j.get("data", {}).get("status") == "started":
return j["data"]["task_id"]
if j.get("ok") and j.get("data", {}).get("status") == "cached":
return "cached"
except Exception as e:
log(f" 提交异常: {e}")
return None
def wait_for_job(task_id, timeout=JOB_TIMEOUT):
start = time.time()
while time.time() - start < timeout:
try:
r = requests.get(f"{BASE}/api/job/{task_id}", timeout=5)
j = r.json()
status = j.get("data", {}).get("status", "")
if status == "completed":
return True
elif status == "failed":
return False
except:
pass
time.sleep(POLL_INTERVAL)
return False
def main():
articles = get_unsummarized()
total = len(articles)
if total == 0:
log("没有待摘要的文章。")
return
log(f"开始批量摘要: 共 {total} 篇，每批 {BATCH_SIZE} 篇并行")
done, failed, skipped = 0, 0, 0
for i in range(0, total, BATCH_SIZE):
batch = articles[i:i + BATCH_SIZE]
batch_num = i // BATCH_SIZE + 1
total_batches = (total + BATCH_SIZE - 1) // BATCH_SIZE
# 提交本批次
tasks = []
for a in batch:
tid = submit_summary(a)
if tid == "cached":
log(f"[{batch_num}/{total_batches}] 已有摘要跳过: {a['title'][:40]}")
skipped += 1
elif tid:
tasks.append((tid, a["title"][:40]))
log(f"[{batch_num}/{total_batches}] 已提交: {a['title'][:40]}")
else:
log(f"[{batch_num}/{total_batches}] 提交失败: {a['title'][:40]}")
failed += 1
# 等待本批次完成
for tid, title in tasks:
log(f" 等待: {title}...")
ok = wait_for_job(tid)
if ok:
log(f" ✅ 完成: {title}")
done += 1
else:
log(f" ❌ 超时/失败: {title}")
failed += 1
progress = done + failed + skipped
log(f" 进度: {progress}/{total} (✅{done} ❌{failed} ⏭{skipped})")
log(f"全部完成: ✅{done} ❌{failed} ⏭{skipped} / {total}")
if __name__ == "__main__":
main()