fix(executor): 僵尸僵死双层防护——进程内线程/心跳看门狗(非零退出)+外部监管进程(offline超180s自动杀启); TEST-060杀进程自动重启实测通过

docs/TEST_RECORDS.md

@@ -2263,3 +2263,40 @@ solver.run_single_point(thermal_mode)（coupled 分支调 do_magnetic_thermal_ca
   2. 执行器新增 `_is_task_cancelled()`：每点前 best-effort 查 `GET /api/tasks/{id}`（网络失败不误判中断），cancelled 则设 `_task_cancelled` 中断循环，按 `cancelled` 上报已完成点
 - **测试**：新增 `scripts/test_executor_cancel.py`（6 项全过：本地模式不取消/网络异常不误判/cancelled 检出/running 不误判/中途取消只跑 1 点并上报 cancelled/未取消跑全部点）；回归 test_executor_m3 4 项、test_executor_p5m2 8 项、test_executor_config 20 项全过；前端 vue-tsc 0 错误
 - **注意**：取消在点边界生效（当前点跑完后中断），已完成的点保留上报
+
+---
+
+## TEST-060：执行器僵尸僵死事件复盘 + 双层防护（进程内看门狗 + 外部监管进程）
+
+| 项目 | 内容 |
+|---|---|
+| 测试日期 | 2026-09-04 |
+| 触发事件 | 执行器 11924-5fed 于 06:13 完成用户扫描任务 a4f6ad1d 后全程冻结：日志/心跳/取任务全停、无任何报错，进程僵尸至 12:06 才消失。期间用户 11:08/11:10 两次提交扫描均无人领取，只能取消。 |
+
+### 事件重建（证据链）
+
+- 最后心跳状态卡在 running → 冻结点在任务完成回调之后、下一次心跳之前。
+- 日志无任何异常 → 非 Python 异常，疑似 Motor-CAD COM 对象在 GC 时释放挂起且持有 GIL，冻结全部线程。
+- 进程级冻结使一切进程内看门狗失效 → 必须有独立进程的外部监管。
+
+### 修复（双层防护）
+
+1. **进程内**（task_executor.py / run_task_executor.py）：
+   - start_polling 返回全部线程（含心跳线程），主循环逐一监控，任一线程死亡 → 报错 + 非零退出；
+   - _send_heartbeat 记录成功时间戳，主循环看门狗：120s 无成功心跳 → 退出（exit 4）；
+   - 每个任务完成后做确定性 gc.collect()，前后打日志面包屑——若 COM 释放在此挂起，日志精确定位。
+2. **进程外**（新建 scripts/executor_supervisor.py）：
+   - 子进程方式运行执行器；每 15s 查后端 /api/executor/status；
+   - 子进程死 → 10s 退避后自动重启；
+   - 进程活但 offline 超 180s（僵尸态）→ 杀进程重启；
+   - 后端不可达时不误判。
+
+### 验证
+
+- 编译 + ASCII 全通过；
+- 看门狗实测：杀子进程 20304 → 61s 内自动重启新执行器 26664-27ea 上线。✅
+- 当前执行器已在看门狗保护下运行。
+
+### 使用方式
+
+今后启动执行器推荐：`python scripts/executor_supervisor.py --config executor_config.json`

scripts/executor_supervisor.py

@@ -0,0 +1,145 @@
+"""Executor supervisor: run the Motor-CAD task executor under a watchdog.
+
+Why this exists (incident 2026-09-04):
+    The executor once became a zombie - process alive, but all threads frozen
+    (a hung COM call holding the GIL is the prime suspect). No heartbeat, no
+    task polling, no traceback, no log. User-submitted scans sat unclaimed for
+    hours until someone noticed. In-process watchdogs cannot help when the GIL
+    itself is seized, so supervision must live in a SEPARATE process.
+
+What it does:
+    1. Starts scripts/run_task_executor.py as a child process.
+    2. Every CHECK_INTERVAL_S seconds:
+       - child dead            -> restart (with backoff)
+       - child alive but its executor OFFLINE at the backend for longer than
+         OFFLINE_GRACE_S -> kill and restart (the zombie case)
+    3. Logs everything to output/executor_logs/supervisor_*.log and stdout.
+
+Usage:
+    python scripts/executor_supervisor.py --config executor_config.json
+"""
+
+import argparse
+import logging
+import os
+import signal
+import subprocess
+import sys
+import time
+from datetime import datetime
+from pathlib import Path
+
+import requests
+
+CHECK_INTERVAL_S = 15
+OFFLINE_GRACE_S = 180      # how long "alive but offline" is tolerated
+RESTART_BACKOFF_S = 10     # delay between restarts
+BACKEND_URL = "http://127.0.0.1:8000"
+
+
+def setup_logger(log_path: Path) -> logging.Logger:
+    logger = logging.getLogger("supervisor")
+    logger.setLevel(logging.INFO)
+    fmt = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
+    fh = logging.FileHandler(log_path, encoding="utf-8")
+    fh.setFormatter(fmt)
+    sh = logging.StreamHandler(sys.stdout)
+    sh.setFormatter(fmt)
+    logger.addHandler(fh)
+    logger.addHandler(sh)
+    return logger
+
+
+def executor_online(backend: str, pid: int):
+    """Return (reachable, online) for the executor belonging to child pid.
+
+    Executor ids are 'motorcad-executor-<pid>-<rand>'; match by pid segment.
+    reachable=False means the backend itself is down - not the child's fault,
+    so the caller must NOT treat it as a zombie.
+    """
+    try:
+        r = requests.get(f"{backend}/api/executor/status", timeout=5)
+        data = r.json()
+    except Exception:
+        return False, None
+    for e in data.get("executors", []):
+        if f"-{pid}-" in str(e.get("executor_id", "")):
+            return True, bool(e.get("online"))
+    return True, None  # backend up, but our executor has not registered yet
+
+
+def main():
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--config", default="executor_config.json")
+    args = ap.parse_args()
+
+    root = Path(__file__).resolve().parent.parent
+    log_dir = root / "output" / "executor_logs"
+    log_dir.mkdir(parents=True, exist_ok=True)
+    logger = setup_logger(log_dir / f"supervisor_{datetime.now():%Y%m%d_%H%M%S}.log")
+
+    child_cmd = [
+        sys.executable,
+        str(root / "scripts" / "run_task_executor.py"),
+        "--config", str(root / args.config),
+    ]
+    logger.info("supervisor started; child cmd: %s", " ".join(child_cmd))
+
+    child = None
+    offline_since = None
+    stopping = False
+
+    def stop_child(sig=None, frame=None):
+        nonlocal stopping
+        stopping = True
+        if child and child.poll() is None:
+            logger.info("stopping child pid=%s", child.pid)
+            child.terminate()
+            try:
+                child.wait(timeout=10)
+            except subprocess.TimeoutExpired:
+                child.kill()
+
+    signal.signal(signal.SIGINT, stop_child)
+    signal.signal(signal.SIGTERM, stop_child)
+
+    while not stopping:
+        if child is None or child.poll() is not None:
+            if child is not None:
+                logger.warning("child exited rc=%s; restarting in %ds",
+                               child.returncode, RESTART_BACKOFF_S)
+                time.sleep(RESTART_BACKOFF_S)
+            child = subprocess.Popen(child_cmd, cwd=str(root))
+            offline_since = None
+            logger.info("child started pid=%s", child.pid)
+            continue
+
+        reachable, online = executor_online(BACKEND_URL, child.pid)
+        if not reachable:
+            # Backend down: nothing the child can do; just wait.
+            offline_since = None
+        elif online is False:
+            if offline_since is None:
+                offline_since = time.monotonic()
+                logger.warning("child pid=%s registered but OFFLINE; grace %ds",
+                               child.pid, OFFLINE_GRACE_S)
+            elif time.monotonic() - offline_since > OFFLINE_GRACE_S:
+                logger.error("ZOMBIE: child pid=%s alive but offline >%ds; killing",
+                             child.pid, OFFLINE_GRACE_S)
+                child.kill()
+                child.wait(timeout=10)
+                child = None
+                offline_since = None
+        else:
+            if offline_since is not None:
+                logger.info("child pid=%s back online", child.pid)
+            offline_since = None
+
+        time.sleep(CHECK_INTERVAL_S)
+
+    stop_child()
+    logger.info("supervisor stopped")
+
+
+if __name__ == "__main__":
+    main()

scripts/run_task_executor.py

@@ -148,14 +148,35 @@ def main():
     executors = build_executors(cfg)
     threads = []
     for ex in executors:
-        threads.append(ex.start_polling(interval=int(cfg["poll_interval"])))
+        threads.extend(ex.start_polling(interval=int(cfg["poll_interval"])))
         logger.info("executor started: %s", ex.executor_id)
 
     print("Task executor started. Model=%s" % cfg["model_path"], flush=True)
     print("Web base URL: %s" % cfg["web_base_url"], flush=True)
     print("Ctrl+C to stop.", flush=True)
+    # Watchdog: exit loudly if any worker thread dies (previously only the
+    # poll threads were tracked; a dead heartbeat thread left a zombie
+    # executor that looked alive locally but was OFFLINE at the backend,
+    # silently dropping user tasks - observed 2026-09-04).
+    hb_stale_s = 120.0  # 24x the 5s interval: generous, only catches real hangs
     try:
-        while any(t.is_alive() for t in threads):
+        while True:
+            alive = [t.is_alive() for t in threads]
+            if not all(alive):
+                dead = sum(1 for a in alive if not a)
+                logger.error("WATCHDOG: %d worker thread(s) died; exiting for restart", dead)
+                print("WATCHDOG: worker thread died, exiting for restart", flush=True)
+                sys.exit(3)
+            # Heartbeat freshness: if the GIL is seized by a hung COM call the
+            # heartbeat stops even though threads are technically "alive".
+            for ex in executors:
+                last = getattr(ex, "last_heartbeat_at", None)
+                if last is not None and time.monotonic() - last > hb_stale_s:
+                    logger.error(
+                        "WATCHDOG: no successful heartbeat for %.0fs (executor %s); exiting for restart",
+                        time.monotonic() - last, ex.executor_id)
+                    print("WATCHDOG: heartbeat stale, exiting for restart", flush=True)
+                    sys.exit(4)
             time.sleep(1)
     except KeyboardInterrupt:
         for ex in executors:

scripts/task_executor.py

@@ -5,6 +5,7 @@ simulations via RobustMotorCADSolver, reports progress and results.
 
 NOTE: All strings must be ASCII only. Chinese text uses \\uXXXX escapes.
 """
+import gc
 import json
 import os
 import sys
@@ -55,6 +56,12 @@ class TaskExecutor:
         # Set when the web side cancelled the currently-running task
         # (detected by polling GET /api/tasks/{id} between points).
         self._task_cancelled = False
+        # Heartbeat observability: monotonic timestamp of the last heartbeat
+        # that actually reached the backend, plus a handle on the heartbeat
+        # thread itself. A silently-dead heartbeat thread once left the
+        # executor running-but-OFFLINE (observed 2026-09-04).
+        self.last_heartbeat_at: Optional[float] = None
+        self._hb_thread: Optional[threading.Thread] = None
         if executor_id is not None:
             self.executor_id = executor_id
         else:
@@ -474,23 +481,33 @@ class TaskExecutor:
                 },
                 timeout=5,
             )
+            self.last_heartbeat_at = time.monotonic()
         except Exception:
             # Heartbeat failures are non-fatal
             pass
 
-    def start_polling(self, interval: int = 5) -> threading.Thread:
+    def start_polling(self, interval: int = 5) -> List[threading.Thread]:
         """Start background threads: one polls/executes tasks, one heartbeats.
 
         Heartbeat runs on its own thread so a long-running Motor-CAD point
         (~2 min each) never starves the heartbeat - otherwise the backend
         would mark this executor offline mid-task (observed 2026-09-04).
+
+        Returns BOTH threads so the caller can watch them; a silently dead
+        thread must surface as a loud failure, not a zombie executor.
         """
         self._running = True
         self._stop_event.clear()
 
         def heartbeat_loop():
             while self._running and not self._stop_event.is_set():
-                self._send_heartbeat()
+                try:
+                    self._send_heartbeat()
+                except Exception as e:
+                    # _send_heartbeat already swallows request errors; this
+                    # guards the unexpected so the thread never dies silently.
+                    if self.on_error:
+                        self.on_error(f"Heartbeat loop error: {str(e)}")
                 self._stop_event.wait(interval)
 
         def poll_loop():
@@ -501,6 +518,16 @@ class TaskExecutor:
                         if self._stop_event.is_set():
                             break
                         self.execute_task(task)
+                        # Post-task GC breadcrumb: COM objects released at GC
+                        # time can hang while holding the GIL, freezing every
+                        # thread (zombie executor observed 2026-09-04). Collect
+                        # at a known point so a hang here is pinpointed by the
+                        # surrounding log lines instead of being invisible.
+                        if self.on_progress:
+                            self.on_progress("post-task-gc:begin")
+                        gc.collect()
+                        if self.on_progress:
+                            self.on_progress("post-task-gc:end")
                 except Exception as e:
                     if self.on_error:
                         self.on_error(f"Poll loop error: {str(e)}")
@@ -508,9 +535,10 @@ class TaskExecutor:
 
         hb_thread = threading.Thread(target=heartbeat_loop, daemon=True)
         hb_thread.start()
+        self._hb_thread = hb_thread
         thread = threading.Thread(target=poll_loop, daemon=True)
         thread.start()
-        return thread
+        return [thread, hb_thread]
 
     def stop(self):
         """Stop the executor."""
@@ -633,8 +661,8 @@ if __name__ == "__main__":
     )
     print("Task executor started (mock mode). Press Ctrl+C to stop.")
     try:
-        thread = executor.start_polling(interval=5)
-        while thread.is_alive():
+        threads = executor.start_polling(interval=5)
+        while any(t.is_alive() for t in threads):
             time.sleep(1)
     except KeyboardInterrupt:
         executor.stop()