deploy_supervisor.py
6.03 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
"""Deploy Supervisor process skeleton (U-036).
Independent OS process (started via `python3 -m supervisor.deploy_supervisor`),
pure stdlib, no LLM CLI dependency -- per docs/supervisor-auto-rollback-design.md
§2's "three independent processes, no shared failure domain" principle:
Deploy Supervisor crashing must not affect Live/Candidate, and Live/Candidate
crashing must not affect Deploy Supervisor's own ability to keep observing
and act.
PID/status file conventions adapted (NOT imported -- see CLAUDE.md's "do not
import graphloop code directly" rule) from graphloop's hermes_watchdog.py:
atomic writes (tempfile-in-same-dir + os.replace), liveness via
`os.kill(pid, 0)`, fail-closed (this skeleton never auto-restarts anything --
a crashing `poll_fn` stops the loop and marks status='stopped' rather than
silently dying while still holding the PID file, which would make
`is_running()` lie).
Scope: this Unit owns process lifecycle only (start/stop/status/PID file).
The actual "check for staged deployments and act" logic is U-037+'s job,
injected here as the `poll_fn` callback -- U-036 has no opinion about what
that logic does, only about running it on a schedule and surviving/reporting
its own crashes correctly.
"""
import argparse
import json
import os
import signal
import subprocess
import sys
import time
from src import paths
def runtime_dir():
d = os.path.join(paths.platform_home(), "deploy_supervisor")
os.makedirs(d, exist_ok=True)
return d
def pid_file_path():
return os.path.join(runtime_dir(), "supervisor.pid")
def status_file_path():
return os.path.join(runtime_dir(), "supervisor.status.json")
def _atomic_write_json(path, data):
tmp = path + ".{}.tmp".format(os.getpid())
with open(tmp, "w", encoding="utf-8") as handle:
json.dump(data, handle)
os.replace(tmp, path)
def is_alive(pid):
if not isinstance(pid, int) or pid <= 0:
return False
try:
os.kill(pid, 0)
except OSError:
return False
return True
def write_status(status, detail=None):
_atomic_write_json(status_file_path(), {
"status": status, "pid": os.getpid(), "updated_at": time.time(), "detail": detail,
})
def read_status():
try:
with open(status_file_path(), encoding="utf-8") as handle:
return json.load(handle)
except (OSError, json.JSONDecodeError):
return None
def is_running():
"""True if a Deploy Supervisor process is currently alive per the PID
file. Doesn't distinguish "never started" from "PID file stale" --
callers needing that distinction should also check `read_status()`."""
try:
with open(pid_file_path(), encoding="utf-8") as handle:
pid = int(handle.read().strip())
except (OSError, ValueError):
return False
return is_alive(pid)
class _StopRequested(Exception):
"""Internal: raised by the SIGTERM handler to break run_foreground's loop."""
def run_foreground(poll_fn=None, poll_interval=5.0, iterations=None):
"""Blocking main loop -- calls `poll_fn()` (if given) once per
`poll_interval` seconds, up to `iterations` cycles (None = forever, only
exits on SIGTERM). Writes a PID file + 'running' status on entry, and
unconditionally writes a 'stopped' status + removes the PID file on any
exit path (normal completion, SIGTERM, or an unhandled exception from
`poll_fn`) -- an exception from `poll_fn` propagates to the caller after
cleanup runs, it is not swallowed (fail-closed: a broken poll cycle
should surface, not vanish into an infinite silent retry).
"""
def _handle_sigterm(signum, frame):
raise _StopRequested()
previous_handler = signal.signal(signal.SIGTERM, _handle_sigterm)
with open(pid_file_path(), "w", encoding="utf-8") as handle:
handle.write(str(os.getpid()))
write_status("running")
try:
count = 0
while iterations is None or count < iterations:
if poll_fn is not None:
poll_fn()
count += 1
if iterations is not None and count >= iterations:
break
time.sleep(poll_interval)
except _StopRequested:
pass
finally:
signal.signal(signal.SIGTERM, previous_handler)
write_status("stopped")
if os.path.exists(pid_file_path()):
os.remove(pid_file_path())
def start_detached(poll_interval=5.0):
"""Launches `python3 -m supervisor.deploy_supervisor` as a real,
independent OS process (Popen, not a thread) -- crashing does not take
down whatever called this. Returns the child's pid. Raises RuntimeError
if a Deploy Supervisor is already running (per its own PID file)."""
if is_running():
raise RuntimeError("Deploy Supervisor is already running")
process = subprocess.Popen(
[sys.executable, "-m", "supervisor.deploy_supervisor", "--poll-interval", str(poll_interval)],
cwd=paths.repo_root(), stdin=subprocess.DEVNULL, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL,
)
return process.pid
def stop(timeout=10.0):
"""Sends SIGTERM to the running Deploy Supervisor (if any) and waits up
to `timeout` seconds for it to exit cleanly (PID file removed). Returns
True if it stopped, False if it was already not running or didn't stop
in time -- fail-closed: this does not SIGKILL on timeout, that's a
judgment call left to whoever's driving it, not silently forced here."""
try:
with open(pid_file_path(), encoding="utf-8") as handle:
pid = int(handle.read().strip())
except (OSError, ValueError):
return False
if not is_alive(pid):
return False
os.kill(pid, signal.SIGTERM)
deadline = time.time() + timeout
while time.time() < deadline:
if not is_running():
return True
time.sleep(0.1)
return False
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--poll-interval", type=float, default=5.0)
args = parser.parse_args()
run_foreground(poll_interval=args.poll_interval)
if __name__ == "__main__":
main()