deploy_rollback.py
9.07 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
"""c_deploy_supervisor_rollback (03-C §3.15 / 04-P §4.6 / design doc §7):
path A (pre-cutover, health check failed -- Live was never touched) and
path B (post-cutover, anomaly detected during monitoring -- Live IS the bad
version, has to actually be swapped back), plus the one fallback both paths
share: rollback itself failing.
**No unified `rollback(conn, deployment_id)` entry point (documented design
choice, matches this subsystem's established convention)**: `deployment`
persists `candidate_instance_dir` (U-037) but no PID column for either the
candidate or the live process -- `deploy_stage.stage_deployment()` and
`deploy_cutover.cutover()` both already require their caller to hand in the
relevant pid(s) explicitly rather than look them up from the DB, since
that's only ever meaningful within one continuous Deploy Supervisor process
lifetime (the one that actually spawned them). `rollback_path_a`/
`rollback_path_b` follow the same convention rather than inventing a
lookup mechanism the rest of this subsystem doesn't have either.
**`deployment.status` vs the ROLLBACK_FAILED return value (documented
interpretation)**: 03-C's `output_contract` for `c_deploy_supervisor_rollback`
lists only `{status: 'rolled_back' | 'health_check_failed' | 'rollback_failed'}`
-- `blocked_pending_human` never appears there, but IS the literal
`error_state` 04-P's own transition table names for this row, and IS one of
`deployment.status`'s valid enum values (01-M 1.15). Read together: 'rollback_
failed' is the ERROR NAME (what gets raised / what 03-C's contract lists),
while the actually-PERSISTED terminal `deployment.status` is
'blocked_pending_human' -- exactly the same pattern `deploy_stage.
CheckoutFailedError` and `deploy_cutover.CutoverFailedError` already use
(raise a *FailedError, persist status='blocked_pending_human'), so this
follows the same convention rather than introducing a new one.
**`git revert` conflict handling (design doc §10 open question 3, this
Unit's own explicit "有明确处理" Done requirement)**: any `git revert`
failure (conflict or otherwise) folds into the SAME `RollbackFailedError` ->
`blocked_pending_human` fallback both paths already have -- not a new
state-machine branch, per the design doc's own recommendation ("应该并入
7.3 兜底路径，而不是新开一个分支"). `git revert --abort` is run best-effort
afterward so a failed attempt doesn't leave the repo mid-conflict for
whoever investigates the resulting `blocked_pending_human`.
"""
import os
import subprocess
from supervisor import deploy_cutover, deploy_health_check, deploy_stage
from src import bug_record, paths
class RollbackFailedError(Exception):
"""ROLLBACK_FAILED -- 03-C §3.15: the automatic recovery mechanism
itself failed (old version won't start either, or `git revert`
conflicts) -- NOT a judgment about whose code is right. Not retried;
`deployment.status='blocked_pending_human'` is already applied by the
time this is raised."""
def _git_revert(repo_root, commit_hash):
subprocess.run(
["git", "revert", "--no-edit", commit_hash],
cwd=repo_root, check=True, capture_output=True, text=True,
)
def _abort_revert_if_in_progress(repo_root):
"""Best-effort: restores the pre-revert state so a failed attempt
doesn't leave the repo mid-conflict."""
subprocess.run(["git", "revert", "--abort"], cwd=repo_root, capture_output=True, text=True)
def _block(conn, deployment_id, reason):
conn.execute(
"UPDATE deployment SET status = 'blocked_pending_human', rollback_reason = ? WHERE deployment_id = ?",
(reason, deployment_id),
)
conn.commit()
def rollback_path_a(
conn, deployment_id, candidate_pid, candidate_db_path, git_commit_hash,
repo_root=None, description=None, task_id=None, stop_timeout=10, candidate_instance_dir=None,
):
"""Pre-cutover rollback (design doc §7.1): Live was never touched, this
is the cheapest path. Terminates the candidate, deletes its one-time DB
copy, removes its worktree checkout, `git revert`s the merge commit,
and records a `bug_record(root_cause_category=core_engine)` audit trail
with the failing signals as evidence. Returns {status: 'rolled_back'}.
**`candidate_instance_dir` cleanup (2026-08-03, real self-hosting run)**:
a real health-check failure exercised this path for the first time and
left the candidate's `git worktree add --detach` checkout permanently
on disk (and registered in `git worktree list`) -- a confirmed-bad
candidate is never needed again after path A, unlike a successful
cutover's candidate (which becomes `previous_instance_dir`, correctly
kept until the monitoring window closes). Optional (default None) so
existing tests that never used a real worktree checkout for their
candidate are unaffected; the real caller (self_hosting_e2e.py) now
passes it through from `stage_deployment()`'s own return value."""
repo_root = repo_root or paths.repo_root()
description = description or "health check failed for deployment {}".format(deployment_id)
try:
deploy_cutover._stop_process(candidate_pid, timeout=stop_timeout)
if candidate_db_path and os.path.exists(candidate_db_path):
os.remove(candidate_db_path)
if candidate_instance_dir:
subprocess.run(
["git", "worktree", "remove", "--force", candidate_instance_dir],
cwd=repo_root, check=True, capture_output=True, text=True,
)
_git_revert(repo_root, git_commit_hash)
except subprocess.CalledProcessError as e:
_abort_revert_if_in_progress(repo_root)
_block(conn, deployment_id, "rollback_failed (path A): git revert failed: {}".format(e.stderr))
raise RollbackFailedError(
"rollback path A failed for deployment {}: git revert error: {}".format(deployment_id, e.stderr)
) from e
bug_record.record_core_engine_change(conn, task_id, description)
conn.execute("UPDATE deployment SET status = 'rolled_back' WHERE deployment_id = ?", (deployment_id,))
conn.commit()
return {"status": "rolled_back"}
def rollback_path_b(
conn, deployment_id, live_pid, previous_instance_dir, prod_db_path, git_commit_hash,
repo_root=None, description=None, task_id=None, live_port=deploy_cutover.LIVE_PORT,
python_executable=None, stop_timeout=10, bind_timeout=15, startup_check_timeout=30,
):
"""Post-cutover rollback (design doc §7.2): Live IS the bad version, so
the previously-known-good checkout has to be revived and re-verified
before this can be trusted as the new Live. Returns {status:
'rolled_back'}. Raises RollbackFailedError (deployment already marked
blocked_pending_human) if the revived previous_instance_dir fails to
bind live_port OR fails its own re-verification (same three checks
c_deploy_supervisor_health_check runs, called directly here rather than
through that module's deployment-row-mutating wrapper -- this is a
distinct concern from the original candidate's own staged health check
trail and shouldn't be interleaved with it)."""
repo_root = repo_root or paths.repo_root()
description = description or "post-cutover anomaly detected for deployment {}".format(deployment_id)
conn.execute("UPDATE deployment SET status = 'rollback_in_progress' WHERE deployment_id = ?", (deployment_id,))
conn.commit()
try:
deploy_cutover._stop_process(live_pid, timeout=stop_timeout)
revived = deploy_stage._launch_candidate(previous_instance_dir, live_port, prod_db_path, python_executable)
if not deploy_cutover._wait_for_live_http(live_port, bind_timeout):
raise RollbackFailedError(
"previous_instance_dir {} never bound live_port {} during rollback".format(
previous_instance_dir, live_port
)
)
signals = [
deploy_health_check.check_process_alive(revived.pid, timeout=startup_check_timeout),
deploy_health_check.check_http_health(live_port),
deploy_health_check.check_canary_task(live_port),
]
failing = [s for s in signals if s["status"] != "ok"]
if failing:
raise RollbackFailedError(
"revived previous_instance_dir failed re-verification for deployment {}: {}".format(
deployment_id, failing
)
)
_git_revert(repo_root, git_commit_hash)
except subprocess.CalledProcessError as e:
_abort_revert_if_in_progress(repo_root)
_block(conn, deployment_id, "rollback_failed (path B): git revert failed: {}".format(e.stderr))
raise RollbackFailedError(
"rollback path B failed for deployment {}: git revert error: {}".format(deployment_id, e.stderr)
) from e
except RollbackFailedError as e:
_block(conn, deployment_id, "rollback_failed (path B): {}".format(e))
raise
bug_record.record_core_engine_change(conn, task_id, description)
conn.execute("UPDATE deployment SET status = 'rolled_back' WHERE deployment_id = ?", (deployment_id,))
conn.commit()
return {"status": "rolled_back"}