reap_stuck_jobs.py
2.59 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
#!/data/data/com.termux/files/usr/bin/python3
"""Stuck job reaper — scan hermes_jobs for orphaned 'running' jobs and fix them."""
import sqlite3, os, sys, re
from datetime import datetime, timedelta
DB = "/data/data/com.termux/files/home/wechat-viewer/circle.db"
LOG_DIR = "/data/data/com.termux/files/home/tmp/hermes_jobs"
def main():
db = sqlite3.connect(DB)
# Find stuck jobs: status='running' and started > 5 min ago
cutoff = (datetime.now() - timedelta(minutes=5)).strftime("%Y-%m-%d %H:%M:%S")
stuck = db.execute(
"SELECT task_id, job_type, job_name, status, started_at, log_file "
"FROM hermes_jobs WHERE status='running' AND started_at < ?",
(cutoff,)
).fetchall()
if not stuck:
print("No stuck jobs found.")
db.close()
return
print(f"Found {len(stuck)} stuck jobs:")
completed_pattern = re.compile(r"Session:\s+\S+")
fixed_completed = 0
fixed_failed = 0
for task_id, job_type, job_name, status, started_at, log_file in stuck:
log_path = log_file if os.path.isabs(log_file) else os.path.join(LOG_DIR, os.path.basename(log_file))
# Check if log file exists and has completion evidence
new_status = "failed" # default
if os.path.exists(log_path):
try:
with open(log_path) as f:
content = f.read()
# Check for successful completion markers
if "Exit code: 0" in content or "Duration:" in content:
new_status = "completed"
elif "TIMEOUT" in content:
new_status = "failed"
elif completed_pattern.search(content):
new_status = "completed"
except Exception:
new_status = "failed"
else:
new_status = "failed"
db.execute("UPDATE hermes_jobs SET status=? WHERE task_id=?", (new_status, task_id))
print(f" {task_id[:8]}... {job_type}:{job_name[:30]} → {new_status}")
if new_status == "completed":
fixed_completed += 1
else:
fixed_failed += 1
db.commit()
db.close()
total_fixed = fixed_completed + fixed_failed
gate_trace = (
f"[node=reaper,attempt=1,gate=STUCK_JOBS,"
f"action={'CONTINUE' if total_fixed > 0 else 'PASS'}] "
f"fixed {fixed_completed} completed + {fixed_failed} failed out of {len(stuck)} stuck"
)
print(f"\nDone: {fixed_completed} completed, {fixed_failed} failed.")
print(gate_trace)
if __name__ == "__main__":
main()