|
|
@@ -0,0 +1,145 @@
|
|
|
+"""Executor supervisor: run the Motor-CAD task executor under a watchdog.
|
|
|
+
|
|
|
+Why this exists (incident 2026-09-04):
|
|
|
+ The executor once became a zombie - process alive, but all threads frozen
|
|
|
+ (a hung COM call holding the GIL is the prime suspect). No heartbeat, no
|
|
|
+ task polling, no traceback, no log. User-submitted scans sat unclaimed for
|
|
|
+ hours until someone noticed. In-process watchdogs cannot help when the GIL
|
|
|
+ itself is seized, so supervision must live in a SEPARATE process.
|
|
|
+
|
|
|
+What it does:
|
|
|
+ 1. Starts scripts/run_task_executor.py as a child process.
|
|
|
+ 2. Every CHECK_INTERVAL_S seconds:
|
|
|
+ - child dead -> restart (with backoff)
|
|
|
+ - child alive but its executor OFFLINE at the backend for longer than
|
|
|
+ OFFLINE_GRACE_S -> kill and restart (the zombie case)
|
|
|
+ 3. Logs everything to output/executor_logs/supervisor_*.log and stdout.
|
|
|
+
|
|
|
+Usage:
|
|
|
+ python scripts/executor_supervisor.py --config executor_config.json
|
|
|
+"""
|
|
|
+
|
|
|
+import argparse
|
|
|
+import logging
|
|
|
+import os
|
|
|
+import signal
|
|
|
+import subprocess
|
|
|
+import sys
|
|
|
+import time
|
|
|
+from datetime import datetime
|
|
|
+from pathlib import Path
|
|
|
+
|
|
|
+import requests
|
|
|
+
|
|
|
+CHECK_INTERVAL_S = 15
|
|
|
+OFFLINE_GRACE_S = 180 # how long "alive but offline" is tolerated
|
|
|
+RESTART_BACKOFF_S = 10 # delay between restarts
|
|
|
+BACKEND_URL = "http://127.0.0.1:8000"
|
|
|
+
|
|
|
+
|
|
|
+def setup_logger(log_path: Path) -> logging.Logger:
|
|
|
+ logger = logging.getLogger("supervisor")
|
|
|
+ logger.setLevel(logging.INFO)
|
|
|
+ fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
|
|
|
+ fh = logging.FileHandler(log_path, encoding="utf-8")
|
|
|
+ fh.setFormatter(fmt)
|
|
|
+ sh = logging.StreamHandler(sys.stdout)
|
|
|
+ sh.setFormatter(fmt)
|
|
|
+ logger.addHandler(fh)
|
|
|
+ logger.addHandler(sh)
|
|
|
+ return logger
|
|
|
+
|
|
|
+
|
|
|
+def executor_online(backend: str, pid: int):
|
|
|
+ """Return (reachable, online) for the executor belonging to child pid.
|
|
|
+
|
|
|
+ Executor ids are 'motorcad-executor-<pid>-<rand>'; match by pid segment.
|
|
|
+ reachable=False means the backend itself is down - not the child's fault,
|
|
|
+ so the caller must NOT treat it as a zombie.
|
|
|
+ """
|
|
|
+ try:
|
|
|
+ r = requests.get(f"{backend}/api/executor/status", timeout=5)
|
|
|
+ data = r.json()
|
|
|
+ except Exception:
|
|
|
+ return False, None
|
|
|
+ for e in data.get("executors", []):
|
|
|
+ if f"-{pid}-" in str(e.get("executor_id", "")):
|
|
|
+ return True, bool(e.get("online"))
|
|
|
+ return True, None # backend up, but our executor has not registered yet
|
|
|
+
|
|
|
+
|
|
|
+def main():
|
|
|
+ ap = argparse.ArgumentParser()
|
|
|
+ ap.add_argument("--config", default="executor_config.json")
|
|
|
+ args = ap.parse_args()
|
|
|
+
|
|
|
+ root = Path(__file__).resolve().parent.parent
|
|
|
+ log_dir = root / "output" / "executor_logs"
|
|
|
+ log_dir.mkdir(parents=True, exist_ok=True)
|
|
|
+ logger = setup_logger(log_dir / f"supervisor_{datetime.now():%Y%m%d_%H%M%S}.log")
|
|
|
+
|
|
|
+ child_cmd = [
|
|
|
+ sys.executable,
|
|
|
+ str(root / "scripts" / "run_task_executor.py"),
|
|
|
+ "--config", str(root / args.config),
|
|
|
+ ]
|
|
|
+ logger.info("supervisor started; child cmd: %s", " ".join(child_cmd))
|
|
|
+
|
|
|
+ child = None
|
|
|
+ offline_since = None
|
|
|
+ stopping = False
|
|
|
+
|
|
|
+ def stop_child(sig=None, frame=None):
|
|
|
+ nonlocal stopping
|
|
|
+ stopping = True
|
|
|
+ if child and child.poll() is None:
|
|
|
+ logger.info("stopping child pid=%s", child.pid)
|
|
|
+ child.terminate()
|
|
|
+ try:
|
|
|
+ child.wait(timeout=10)
|
|
|
+ except subprocess.TimeoutExpired:
|
|
|
+ child.kill()
|
|
|
+
|
|
|
+ signal.signal(signal.SIGINT, stop_child)
|
|
|
+ signal.signal(signal.SIGTERM, stop_child)
|
|
|
+
|
|
|
+ while not stopping:
|
|
|
+ if child is None or child.poll() is not None:
|
|
|
+ if child is not None:
|
|
|
+ logger.warning("child exited rc=%s; restarting in %ds",
|
|
|
+ child.returncode, RESTART_BACKOFF_S)
|
|
|
+ time.sleep(RESTART_BACKOFF_S)
|
|
|
+ child = subprocess.Popen(child_cmd, cwd=str(root))
|
|
|
+ offline_since = None
|
|
|
+ logger.info("child started pid=%s", child.pid)
|
|
|
+ continue
|
|
|
+
|
|
|
+ reachable, online = executor_online(BACKEND_URL, child.pid)
|
|
|
+ if not reachable:
|
|
|
+ # Backend down: nothing the child can do; just wait.
|
|
|
+ offline_since = None
|
|
|
+ elif online is False:
|
|
|
+ if offline_since is None:
|
|
|
+ offline_since = time.monotonic()
|
|
|
+ logger.warning("child pid=%s registered but OFFLINE; grace %ds",
|
|
|
+ child.pid, OFFLINE_GRACE_S)
|
|
|
+ elif time.monotonic() - offline_since > OFFLINE_GRACE_S:
|
|
|
+ logger.error("ZOMBIE: child pid=%s alive but offline >%ds; killing",
|
|
|
+ child.pid, OFFLINE_GRACE_S)
|
|
|
+ child.kill()
|
|
|
+ child.wait(timeout=10)
|
|
|
+ child = None
|
|
|
+ offline_since = None
|
|
|
+ else:
|
|
|
+ if offline_since is not None:
|
|
|
+ logger.info("child pid=%s back online", child.pid)
|
|
|
+ offline_since = None
|
|
|
+
|
|
|
+ time.sleep(CHECK_INTERVAL_S)
|
|
|
+
|
|
|
+ stop_child()
|
|
|
+ logger.info("supervisor stopped")
|
|
|
+
|
|
|
+
|
|
|
+if __name__ == "__main__":
|
|
|
+ main()
|
carlin