#!/data/data/com.termux/files/usr/bin/python3 """Stuck job reaper — scan hermes_jobs for orphaned 'running' jobs and fix them.""" import sqlite3, os, sys, re from datetime import datetime, timedelta DB = "/data/data/com.termux/files/home/wechat-viewer/circle.db" LOG_DIR = "/data/data/com.termux/files/home/tmp/hermes_jobs" def main(): db = sqlite3.connect(DB) # Find stuck jobs: status='running' and started > 5 min ago cutoff = (datetime.now() - timedelta(minutes=5)).strftime("%Y-%m-%d %H:%M:%S") stuck = db.execute( "SELECT task_id, job_type, job_name, status, started_at, log_file " "FROM hermes_jobs WHERE status='running' AND started_at < ?", (cutoff,) ).fetchall() if not stuck: print("No stuck jobs found.") db.close() return print(f"Found {len(stuck)} stuck jobs:") completed_pattern = re.compile(r"Session:\s+\S+") fixed_completed = 0 fixed_failed = 0 for task_id, job_type, job_name, status, started_at, log_file in stuck: log_path = log_file if os.path.isabs(log_file) else os.path.join(LOG_DIR, os.path.basename(log_file)) # Check if log file exists and has completion evidence new_status = "failed" # default if os.path.exists(log_path): try: with open(log_path) as f: content = f.read() # Check for successful completion markers if "Exit code: 0" in content or "Duration:" in content: new_status = "completed" elif "TIMEOUT" in content: new_status = "failed" elif completed_pattern.search(content): new_status = "completed" except Exception: new_status = "failed" else: new_status = "failed" db.execute("UPDATE hermes_jobs SET status=? WHERE task_id=?", (new_status, task_id)) print(f" {task_id[:8]}... {job_type}:{job_name[:30]} → {new_status}") if new_status == "completed": fixed_completed += 1 else: fixed_failed += 1 db.commit() db.close() total_fixed = fixed_completed + fixed_failed gate_trace = ( f"[node=reaper,attempt=1,gate=STUCK_JOBS," f"action={'CONTINUE' if total_fixed > 0 else 'PASS'}] " f"fixed {fixed_completed} completed + {fixed_failed} failed out of {len(stuck)} stuck" ) print(f"\nDone: {fixed_completed} completed, {fixed_failed} failed.") print(gate_trace) if __name__ == "__main__": main()