"""c_deploy_supervisor_rollback (03-C §3.15 / 04-P §4.6 / design doc §7): path A (pre-cutover, health check failed -- Live was never touched) and path B (post-cutover, anomaly detected during monitoring -- Live IS the bad version, has to actually be swapped back), plus the one fallback both paths share: rollback itself failing. **No unified `rollback(conn, deployment_id)` entry point (documented design choice, matches this subsystem's established convention)**: `deployment` persists `candidate_instance_dir` (U-037) but no PID column for either the candidate or the live process -- `deploy_stage.stage_deployment()` and `deploy_cutover.cutover()` both already require their caller to hand in the relevant pid(s) explicitly rather than look them up from the DB, since that's only ever meaningful within one continuous Deploy Supervisor process lifetime (the one that actually spawned them). `rollback_path_a`/ `rollback_path_b` follow the same convention rather than inventing a lookup mechanism the rest of this subsystem doesn't have either. **`deployment.status` vs the ROLLBACK_FAILED return value (documented interpretation)**: 03-C's `output_contract` for `c_deploy_supervisor_rollback` lists only `{status: 'rolled_back' | 'health_check_failed' | 'rollback_failed'}` -- `blocked_pending_human` never appears there, but IS the literal `error_state` 04-P's own transition table names for this row, and IS one of `deployment.status`'s valid enum values (01-M 1.15). Read together: 'rollback_ failed' is the ERROR NAME (what gets raised / what 03-C's contract lists), while the actually-PERSISTED terminal `deployment.status` is 'blocked_pending_human' -- exactly the same pattern `deploy_stage. CheckoutFailedError` and `deploy_cutover.CutoverFailedError` already use (raise a *FailedError, persist status='blocked_pending_human'), so this follows the same convention rather than introducing a new one. **`git revert` conflict handling (design doc §10 open question 3, this Unit's own explicit "有明确处理" Done requirement)**: any `git revert` failure (conflict or otherwise) folds into the SAME `RollbackFailedError` -> `blocked_pending_human` fallback both paths already have -- not a new state-machine branch, per the design doc's own recommendation ("应该并入 7.3 兜底路径,而不是新开一个分支"). `git revert --abort` is run best-effort afterward so a failed attempt doesn't leave the repo mid-conflict for whoever investigates the resulting `blocked_pending_human`. """ import os import subprocess from supervisor import deploy_cutover, deploy_health_check, deploy_stage from src import bug_record, paths class RollbackFailedError(Exception): """ROLLBACK_FAILED -- 03-C §3.15: the automatic recovery mechanism itself failed (old version won't start either, or `git revert` conflicts) -- NOT a judgment about whose code is right. Not retried; `deployment.status='blocked_pending_human'` is already applied by the time this is raised.""" def _git_revert(repo_root, commit_hash): subprocess.run( ["git", "revert", "--no-edit", commit_hash], cwd=repo_root, check=True, capture_output=True, text=True, ) def _abort_revert_if_in_progress(repo_root): """Best-effort: restores the pre-revert state so a failed attempt doesn't leave the repo mid-conflict.""" subprocess.run(["git", "revert", "--abort"], cwd=repo_root, capture_output=True, text=True) def _block(conn, deployment_id, reason): conn.execute( "UPDATE deployment SET status = 'blocked_pending_human', rollback_reason = ? WHERE deployment_id = ?", (reason, deployment_id), ) conn.commit() def rollback_path_a( conn, deployment_id, candidate_pid, candidate_db_path, git_commit_hash, repo_root=None, description=None, task_id=None, stop_timeout=10, ): """Pre-cutover rollback (design doc §7.1): Live was never touched, this is the cheapest path. Terminates the candidate, deletes its one-time DB copy, `git revert`s the merge commit, and records a `bug_record( root_cause_category=core_engine)` audit trail with the failing signals as evidence. Returns {status: 'rolled_back'}.""" repo_root = repo_root or paths.repo_root() description = description or "health check failed for deployment {}".format(deployment_id) try: deploy_cutover._stop_process(candidate_pid, timeout=stop_timeout) if candidate_db_path and os.path.exists(candidate_db_path): os.remove(candidate_db_path) _git_revert(repo_root, git_commit_hash) except subprocess.CalledProcessError as e: _abort_revert_if_in_progress(repo_root) _block(conn, deployment_id, "rollback_failed (path A): git revert failed: {}".format(e.stderr)) raise RollbackFailedError( "rollback path A failed for deployment {}: git revert error: {}".format(deployment_id, e.stderr) ) from e bug_record.record_core_engine_change(conn, task_id, description) conn.execute("UPDATE deployment SET status = 'rolled_back' WHERE deployment_id = ?", (deployment_id,)) conn.commit() return {"status": "rolled_back"} def rollback_path_b( conn, deployment_id, live_pid, previous_instance_dir, prod_db_path, git_commit_hash, repo_root=None, description=None, task_id=None, live_port=deploy_cutover.LIVE_PORT, python_executable=None, stop_timeout=10, bind_timeout=15, startup_check_timeout=30, ): """Post-cutover rollback (design doc §7.2): Live IS the bad version, so the previously-known-good checkout has to be revived and re-verified before this can be trusted as the new Live. Returns {status: 'rolled_back'}. Raises RollbackFailedError (deployment already marked blocked_pending_human) if the revived previous_instance_dir fails to bind live_port OR fails its own re-verification (same three checks c_deploy_supervisor_health_check runs, called directly here rather than through that module's deployment-row-mutating wrapper -- this is a distinct concern from the original candidate's own staged health check trail and shouldn't be interleaved with it).""" repo_root = repo_root or paths.repo_root() description = description or "post-cutover anomaly detected for deployment {}".format(deployment_id) conn.execute("UPDATE deployment SET status = 'rollback_in_progress' WHERE deployment_id = ?", (deployment_id,)) conn.commit() try: deploy_cutover._stop_process(live_pid, timeout=stop_timeout) revived = deploy_stage._launch_candidate(previous_instance_dir, live_port, prod_db_path, python_executable) if not deploy_cutover._wait_for_live_http(live_port, bind_timeout): raise RollbackFailedError( "previous_instance_dir {} never bound live_port {} during rollback".format( previous_instance_dir, live_port ) ) signals = [ deploy_health_check.check_process_alive(revived.pid, timeout=startup_check_timeout), deploy_health_check.check_http_health(live_port), deploy_health_check.check_canary_task(live_port), ] failing = [s for s in signals if s["status"] != "ok"] if failing: raise RollbackFailedError( "revived previous_instance_dir failed re-verification for deployment {}: {}".format( deployment_id, failing ) ) _git_revert(repo_root, git_commit_hash) except subprocess.CalledProcessError as e: _abort_revert_if_in_progress(repo_root) _block(conn, deployment_id, "rollback_failed (path B): git revert failed: {}".format(e.stderr)) raise RollbackFailedError( "rollback path B failed for deployment {}: git revert error: {}".format(deployment_id, e.stderr) ) from e except RollbackFailedError as e: _block(conn, deployment_id, "rollback_failed (path B): {}".format(e)) raise bug_record.record_core_engine_change(conn, task_id, description) conn.execute("UPDATE deployment SET status = 'rolled_back' WHERE deployment_id = ?", (deployment_id,)) conn.commit() return {"status": "rolled_back"}