pcb-afm-simulation-system/scripts/executor_supervisor.py

"""Executor supervisor: run the Motor-CAD task executor under a watchdog.

Why this exists (incident 2026-09-04):
    The executor once became a zombie - process alive, but all threads frozen
    (a hung COM call holding the GIL is the prime suspect). No heartbeat, no
    task polling, no traceback, no log. User-submitted scans sat unclaimed for
    hours until someone noticed. In-process watchdogs cannot help when the GIL
    itself is seized, so supervision must live in a SEPARATE process.

What it does:
    1. Starts scripts/run_task_executor.py as a child process.
    2. Every CHECK_INTERVAL_S seconds:
       - child dead            -> restart (with backoff)
       - child alive but its executor OFFLINE at the backend for longer than
         OFFLINE_GRACE_S -> kill and restart (the zombie case)
    3. Logs everything to output/executor_logs/supervisor_*.log and stdout.

Usage:
    python scripts/executor_supervisor.py --config executor_config.json
"""

import argparse
import logging
import os
import signal
import subprocess
import sys
import time
from datetime import datetime
from pathlib import Path

import requests

CHECK_INTERVAL_S = 15
OFFLINE_GRACE_S = 180      # how long "alive but offline" is tolerated
RESTART_BACKOFF_S = 10     # delay between restarts
BACKEND_URL = "http://127.0.0.1:8000"


def setup_logger(log_path: Path) -> logging.Logger:
    logger = logging.getLogger("supervisor")
    logger.setLevel(logging.INFO)
    fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
    fh = logging.FileHandler(log_path, encoding="utf-8")
    fh.setFormatter(fmt)
    sh = logging.StreamHandler(sys.stdout)
    sh.setFormatter(fmt)
    logger.addHandler(fh)
    logger.addHandler(sh)
    return logger


def executor_online(backend: str, pid: int):
    """Return (reachable, online) for the executor belonging to child pid.

    Executor ids are 'motorcad-executor-<pid>-<rand>'; match by pid segment.
    reachable=False means the backend itself is down - not the child's fault,
    so the caller must NOT treat it as a zombie.
    """
    try:
        r = requests.get(f"{backend}/api/executor/status", timeout=5)
        data = r.json()
    except Exception:
        return False, None
    for e in data.get("executors", []):
        if f"-{pid}-" in str(e.get("executor_id", "")):
            return True, bool(e.get("online"))
    return True, None  # backend up, but our executor has not registered yet


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("--config", default="executor_config.json")
    args = ap.parse_args()

    root = Path(__file__).resolve().parent.parent
    log_dir = root / "output" / "executor_logs"
    log_dir.mkdir(parents=True, exist_ok=True)
    logger = setup_logger(log_dir / f"supervisor_{datetime.now():%Y%m%d_%H%M%S}.log")

    child_cmd = [
        sys.executable,
        str(root / "scripts" / "run_task_executor.py"),
        "--config", str(root / args.config),
    ]
    logger.info("supervisor started; child cmd: %s", " ".join(child_cmd))

    child = None
    offline_since = None
    stopping = False

    def stop_child(sig=None, frame=None):
        nonlocal stopping
        stopping = True
        if child and child.poll() is None:
            logger.info("stopping child pid=%s", child.pid)
            child.terminate()
            try:
                child.wait(timeout=10)
            except subprocess.TimeoutExpired:
                child.kill()

    signal.signal(signal.SIGINT, stop_child)
    signal.signal(signal.SIGTERM, stop_child)

    while not stopping:
        if child is None or child.poll() is not None:
            if child is not None:
                logger.warning("child exited rc=%s; restarting in %ds",
                               child.returncode, RESTART_BACKOFF_S)
                time.sleep(RESTART_BACKOFF_S)
            child = subprocess.Popen(child_cmd, cwd=str(root))
            offline_since = None
            logger.info("child started pid=%s", child.pid)
            continue

        reachable, online = executor_online(BACKEND_URL, child.pid)
        if not reachable:
            # Backend down: nothing the child can do; just wait.
            offline_since = None
        elif online is False:
            if offline_since is None:
                offline_since = time.monotonic()
                logger.warning("child pid=%s registered but OFFLINE; grace %ds",
                               child.pid, OFFLINE_GRACE_S)
            elif time.monotonic() - offline_since > OFFLINE_GRACE_S:
                logger.error("ZOMBIE: child pid=%s alive but offline >%ds; killing",
                             child.pid, OFFLINE_GRACE_S)
                child.kill()
                child.wait(timeout=10)
                child = None
                offline_since = None
        else:
            if offline_since is not None:
                logger.info("child pid=%s back online", child.pid)
            offline_since = None

        time.sleep(CHECK_INTERVAL_S)

    stop_child()
    logger.info("supervisor stopped")


if __name__ == "__main__":
    main()