sudo apt update
sudo apt install supervisor -y

root@hzy-baidubcc:~# ls /etc/supervisor/
conf.d  supervisord.conf

# supervisor配置文件：/etc/supervisor/supervisord.conf

vim /etc/supervisor/supervisord.conf

# web端登陆的用户密码以及端口
[inet_http_server]
port=8080
username=admin
password=admin

sudo yum install epel-release -y
sudo yum install supervisor -y

# 安装依赖
pip install setuptools
# 下载源码
wget https://pypi.python.org/packages/source/s/supervisor/supervisor-4.2.5.tar.gz
tar zxvf supervisor-4.2.5.tar.gz
cd supervisor-4.2.5
# 安装
python setup.py install

;  Unix socket 配置（本地通信）
[unix_http_server]
file=/tmp/supervisor.sock   ; socket 文件路径
;chmod=0700                 ; 文件权限
;chown=nobody:nogroup       ; 所属用户组

;  supervisor 核心配置
[supervisord]
logfile=/var/log/supervisord.log  ; 日志文件路径
logfile_maxbytes=50MB             ; 单日志文件最大大小
logfile_backups=10                ; 日志备份数量
loglevel=info                     ; 日志级别（debug/info/warn/error/critical）
pidfile=/var/run/supervisord.pid  ; PID文件路径
nodaemon=false                    ; 是否前台运行（false为后台运行）
minfds=1024                       ; 最小文件描述符数量
minprocs=200                      ; 最小进程数
;user=www-data                    ; 运行用户（非root时指定）

;  supervisorctl 客户端配置
[supervisorctl]
serverurl=unix:///tmp/supervisor.sock  ; 与服务端通信的socket路径
;serverurl=http://127.0.0.1:8080       ; 也可使用HTTP连接

;  包含子进程配置文件
[include]
files = /etc/supervisor/conf.d/*.conf  ; 加载所有.conf结尾的配置文件

;  Web 管理界面配置
[inet_http_server]
port=0.0.0.0:8080           ; 监听地址和端口（0.0.0.0 表示允许所有IP访问）
username=admin              ; 登录用户名
password=admin              ; 登录密码

systemctl restart supervisor.service

[program:echo_time]  ; 进程名称（唯一标识）
command=sh /project/version1/echo_time.sh  ; 执行命令
directory=/project/version1                ; 工作目录
user=www-data                              ; 运行用户
autostart=true                             ; 随supervisor启动而启动
autorestart=true                           ; 进程退出时自动重启
startretries=3                             ; 启动失败重试次数
redirect_stderr=true                       ; 重定向 stderr 到 stdout
stdout_logfile=/var/log/echo_time.log      ; 标准输出日志路径
stdout_logfile_maxbytes=10MB               ; 日志文件最大大小
stdout_logfile_backups=5                   ; 日志备份数量
stopasgroup=true                           ; 停止进程时同时停止子进程
killasgroup=true                           ; 杀死进程时同时杀死子进程

; 环境变量设置
environment=
    PATH="/usr/local/bin:/usr/bin",
    DATABASE_URL="mysql://user:pass@localhost/db"
; 启动前执行的命令
;prestart=/path/to/prestart_script.sh
; 停止后执行的命令
;poststop=/path/to/poststop_script.sh
; 启动超时时间（秒）
startsecs=10
; 停止等待时间（秒）
stopwaitsecs=60
; 进程优先级（值越小优先级越高）
priority=999
; 仅当特定事件发生时才重启
;autorestart=unexpected  ; 仅在意外退出时重启
;exitcodes=0,2           ; 正常退出码（autorestart=unexpected时生效）

# 进入交互模式（可直接输入命令）
supervisorctl
# 查看所有进程状态
supervisorctl status
# 启动单个进程
supervisorctl start <进程名>
# 停止单个进程
supervisorctl stop <进程名>
# 重启单个进程
supervisorctl restart <进程名>
# 启动所有进程
supervisorctl start all
# 停止所有进程
supervisorctl stop all
# 重启所有进程
supervisorctl restart all
# 重新加载配置文件（新增进程时使用，进行测试）
supervisorctl reread
# 更新配置并重启受影响的进程
supervisorctl update
# 强制重启 supervisord 服务（高危命令）
supervisorctl reload

# 启动服务
sudo systemctl start supervisor
# 设置开机自启
sudo systemctl enable supervisor
# 停止服务
sudo systemctl stop supervisor
# 重启服务
sudo systemctl restart supervisor
# 查看服务状态
systemctl status supervisor

vim version1/echo_time.sh 
#/bin/bash

while true; do
    echo `date +%Y-%m-%d,%H:%m:%s` > /project/time.log
    sleep 2
done

vim /etc/supervisor/conf.d/echo_time.conf 

[program:echo_time.sh]
command=sh /project/version1/echo_time.sh
directory=/project/version1
autostart=true
autorestart=true
stdout_logfile=/var/log/test-service.log
stderr_logfile=/var/log/test-service.err.log

supervisorctl reread
supervisorctl update 
supervisorctl status

# supervisorctl status 
test                            RUNNING   pid 2385895, uptime 3:12:31

tail -f /project/time.log 

2025-10-11,11:10:1760155021
tail: /project/time.log: file truncated
2025-10-11,11:10:1760155023
tail: /project/time.log: file truncated
2025-10-11,11:10:1760155025
tail: /project/time.log: file truncated
2025-10-11,11:10:1760155027
tail: /project/time.log: file truncated
2025-10-11,11:10:1760155029

[program:test]
directory=/data/projects/test
command=java -Dfile.encoding=utf-8 -jar test-biz.jar
autostart=true
autorestart=true
startsecs=5
startretries=3
user=root
redirect_stderr=false
stdout_logfile=/data/logs/test.log
stdout_logfile_maxbytes=20MB
stdout_logfile_backups=3
stderr_logfile=/data/logs/test.err
stderr_logfile_maxbytes=20MB
stderr_logfile_backups=3

supervisorctl reread
supervisorctl update 
supervisorctl status

vim /etc/supervisor/conf.d/test.conf 

[program:test]
directory=/data/projects/test
command=/data/projects/test
autostart=true
autorestart=true
startsecs=5
startretries=3
user=root
redirect_stderr=false
stdout_logfile=/data/logs/test.log
stdout_logfile_maxbytes=20MB
stdout_logfile_backups=3
stderr_logfile=/data/logs/test.err
stderr_logfile_maxbytes=20MB
stderr_logfile_backups=3

# 创建用户：-m 创建家目录 / -s 指定shell
useradd -m -s /bin/bash svc_supervisor_view

# 创建SSH密钥目录（SSH免密登录必须）
mkdir -p /home/svc_supervisor_view/.ssh

# 权限700：仅当前用户能读写，SSH强制安全规则
chmod 700 /home/svc_supervisor_view/.ssh

# 归属权给专用用户，否则SSH认证失败
chown -R svc_supervisor_view:svc_supervisor_view /home/svc_supervisor_view/.ssh

cat >/usr/local/bin/supervisor_status_readonly.sh <<'EOF'
#!/bin/sh
set -u

sudo /usr/bin/supervisorctl status
rc=$?

# 集群互补部署或发布中的短暂停服，会让 supervisorctl 返回 3。
# 只要能拿到状态输出，就不要把它当成失败。
if [ "$rc" -eq 0 ] || [ "$rc" -eq 3 ]; then
  exit 0
fi

exit "$rc"
EOF

chmod 755 /usr/local/bin/supervisor_status_readonly.sh

cat >/etc/sudoers.d/svc_supervisor_view <<'EOF'
svc_supervisor_view ALL=(root) NOPASSWD: /usr/bin/supervisorctl status
EOF

chmod 440 /etc/sudoers.d/svc_supervisor_view
visudo -cf /etc/sudoers.d/svc_supervisor_view

cat >>/home/svc_supervisor_view/.ssh/authorized_keys <<'EOF'
from="Jenkins IP",no-agent-forwarding,no-port-forwarding,no-X11-forwarding,no-pty,no-user-rc,command="/usr/local/bin/supervisor_status_readonly.sh" ssh-ed25519 公钥COPY
EOF

chmod 600 /home/svc_supervisor_view/.ssh/authorized_keys
chown svc_supervisor_view:svc_supervisor_view /home/svc_supervisor_view/.ssh/authorized_keys

sudo -u svc_supervisor_view /usr/local/bin/supervisor_status_readonly.sh

# 存放前端页面
mkdir -p /data/ops-status/web

# 存放SSH密钥（权限700，最高安全）
mkdir -p /data/ops-status/.ssh
chmod 700 /data/ops-status/.ssh

# -t ed25519 安全算法 / -f 指定路径 / -N '' 无密码
ssh-keygen -t ed25519 -f /data/ops-status/.ssh/id_ed25519 -N ''

# 私钥权限600（必须！泄露会导致服务器被入侵）
chmod 600 /data/ops-status/.ssh/id_ed25519

# 公钥权限644
chmod 644 /data/ops-status/.ssh/id_ed25519.pub

# 创建known_hosts文件
touch /data/ops-status/.ssh/known_hosts
chmod 600 /data/ops-status/.ssh/known_hosts

# 扫描所有生产服务器IP（替换成你的真实IP）
ssh-keyscan -H 192.168.1.10 >> /data/ops-status/.ssh/known_hosts
ssh-keyscan -H 192.168.1.11 >> /data/ops-status/.ssh/known_hosts

from __future__ import annotations

import logging
import os
import re
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from pathlib import Path
from typing import Any

import paramiko
import yaml
from flask import Flask, jsonify

BASE_DIR = Path(__file__).resolve().parent
INVENTORY_PATH = Path(os.getenv("INVENTORY_PATH", BASE_DIR / "inventory.yml"))
CACHE_TTL = int(os.getenv("CACHE_TTL", "10"))
SSH_TIMEOUT = int(os.getenv("SSH_TIMEOUT", "8"))
MAX_WORKERS = int(os.getenv("MAX_WORKERS", "8"))

STATE_RE = re.compile(r"^(?P<name>\S+)\s+(?P<state>[A-Z_]+)\s*(?P<description>.*)$")
UPTIME_RE = re.compile(r"uptime\s+(?P<uptime>.+)$", re.IGNORECASE)

logging.basicConfig(
    level=os.getenv("LOG_LEVEL", "INFO"),
    format="%(asctime)s [%(levelname)s] %(name)s - %(message)s",
)
logger = logging.getLogger("ops-status")

app = Flask(__name__)
_cache_lock = threading.Lock()
_cache: dict[str, Any] = {"timestamp": 0.0, "data": None}


class InventoryError(RuntimeError):
    pass


def now_iso() -> str:
    return datetime.now(timezone.utc).astimezone().isoformat(timespec="seconds")


def load_inventory() -> dict[str, Any]:
    if not INVENTORY_PATH.exists():
        raise InventoryError(f"inventory file not found: {INVENTORY_PATH}")

    with INVENTORY_PATH.open("r", encoding="utf-8") as fp:
        data = yaml.safe_load(fp) or {}

    ssh_cfg = data.get("ssh")
    if not isinstance(ssh_cfg, dict):
        raise InventoryError("inventory.yml must contain an 'ssh' mapping")

    username = ssh_cfg.get("username")
    if not username:
        raise InventoryError("inventory.yml ssh.username is required")

    servers = data.get("servers")
    if not isinstance(servers, list) or not servers:
        raise InventoryError("inventory.yml must contain a non-empty 'servers' list")

    private_key = ssh_cfg.get("private_key")
    if not private_key:
        raise InventoryError("inventory.yml ssh.private_key is required")

    ssh_defaults = {
        "port": int(ssh_cfg.get("port", 22)),
        "username": username,
        "private_key": private_key,
        "known_hosts": ssh_cfg.get("known_hosts"),
        "allow_agent": bool(ssh_cfg.get("allow_agent", False)),
        "look_for_keys": bool(ssh_cfg.get("look_for_keys", False)),
        "allow_unknown_host_keys": bool(ssh_cfg.get("allow_unknown_host_keys", False)),
        "connect_timeout": int(ssh_cfg.get("connect_timeout", SSH_TIMEOUT)),
        "remote_command": ssh_cfg.get("remote_command", "/usr/local/bin/supervisor_status_readonly.sh"),
    }

    normalized_servers: list[dict[str, Any]] = []
    for idx, server in enumerate(servers, start=1):
        if not isinstance(server, dict):
            raise InventoryError(f"server entry #{idx} must be a mapping")

        host = server.get("host")
        if not host:
            raise InventoryError(f"server entry #{idx} missing host")

        normalized_servers.append(
            {
                "name": server.get("name") or host,
                "host": host,
                "port": int(server.get("port", ssh_defaults["port"])),
                "remote_command": server.get("remote_command", ssh_defaults["remote_command"]),
            }
        )

    return {"ssh": ssh_defaults, "servers": normalized_servers}


def build_ssh_client(ssh_cfg: dict[str, Any]) -> paramiko.SSHClient:
    client = paramiko.SSHClient()

    if ssh_cfg.get("allow_unknown_host_keys"):
        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
    else:
        known_hosts = ssh_cfg.get("known_hosts")
        if known_hosts:
            path = Path(str(known_hosts))
            if path.exists():
                client.load_host_keys(str(path))
            else:
                logger.warning("known_hosts not found: %s", path)
        client.set_missing_host_key_policy(paramiko.RejectPolicy())

    return client


def parse_supervisor_output(output: str) -> list[dict[str, str]]:
    services: list[dict[str, str]] = []
    for raw_line in output.splitlines():
        line = raw_line.rstrip()
        if not line:
            continue

        match = STATE_RE.match(line)
        if not match:
            continue

        state = match.group("state")
        if state != "RUNNING":
            continue

        description = (match.group("description") or "").strip()
        uptime_match = UPTIME_RE.search(description)
        uptime = uptime_match.group("uptime").strip() if uptime_match else ""

        services.append(
            {
                "name": match.group("name"),
                "uptime": uptime,
            }
        )

    services.sort(key=lambda item: item["name"].lower())
    return services


def collect_from_server(server: dict[str, Any], ssh_cfg: dict[str, Any]) -> dict[str, Any]:
    result = {
        "name": server["name"],
        "host": server["host"],
        "reachable": False,
        "error": "",
        "collected_at": now_iso(),
        "running_count": 0,
        "running_services": [],
    }

    client = None
    try:
        client = build_ssh_client(ssh_cfg)
        client.connect(
            hostname=server["host"],
            port=server["port"],
            username=ssh_cfg["username"],
            key_filename=str(ssh_cfg["private_key"]),
            timeout=ssh_cfg["connect_timeout"],
            banner_timeout=ssh_cfg["connect_timeout"],
            auth_timeout=ssh_cfg["connect_timeout"],
            allow_agent=ssh_cfg["allow_agent"],
            look_for_keys=ssh_cfg["look_for_keys"],
        )

        stdin, stdout, stderr = client.exec_command(
            server["remote_command"], timeout=ssh_cfg["connect_timeout"] + 10
        )
        stdout_text = stdout.read().decode("utf-8", errors="replace")
        stderr_text = stderr.read().decode("utf-8", errors="replace").strip()
        exit_code = stdout.channel.recv_exit_status()

        services = parse_supervisor_output(stdout_text)
        has_status_lines = any(
            STATE_RE.match(line.rstrip()) for line in stdout_text.splitlines() if line.strip()
        )

        if has_status_lines:
            result["reachable"] = True
            result["running_services"] = services
            result["running_count"] = len(services)

            # 集群互补部署或发布中的短暂停服，会让 supervisorctl 返回非 0。
            # 只要能拿到状态输出，就视为主机可达，不把整机判失败。
            if exit_code not in (0, 3) and stderr_text:
                result["error"] = f"status parsed with exit code {exit_code}: {stderr_text}"
            return result

        result["error"] = stderr_text or f"remote command failed with exit code {exit_code}"
        return result
    except Exception as exc:  # noqa: BLE001
        result["error"] = str(exc)
        return result
    finally:
        if client is not None:
            client.close()


def collect_status(force: bool = False) -> dict[str, Any]:
    now = time.time()
    with _cache_lock:
        if not force and _cache["data"] and now - _cache["timestamp"] < CACHE_TTL:
            return _cache["data"]

    inventory = load_inventory()
    ssh_cfg = inventory["ssh"]
    servers = inventory["servers"]

    results: list[dict[str, Any]] = []
    with ThreadPoolExecutor(max_workers=min(MAX_WORKERS, len(servers))) as executor:
        future_map = {executor.submit(collect_from_server, server, ssh_cfg): server for server in servers}
        for future in as_completed(future_map):
            results.append(future.result())

    results.sort(key=lambda item: item["name"].lower())
    payload = {
        "generated_at": now_iso(),
        "server_count": len(results),
        "running_total": sum(item["running_count"] for item in results),
        "servers": results,
    }

    with _cache_lock:
        _cache["timestamp"] = now
        _cache["data"] = payload

    return payload


@app.get("/health")
def health() -> Any:
    return jsonify({"ok": True, "time": now_iso()})


@app.get("/api/status")
def api_status() -> Any:
    return jsonify(collect_status(force=False))


@app.get("/api/status/refresh")
def api_status_refresh() -> Any:
    return jsonify(collect_status(force=True))


if __name__ == "__main__":
    app.run(host="127.0.0.1", port=int(os.getenv("PORT", "18081")), debug=False)

ssh:
  username: svc_supervisor_view
  port: 22
  private_key: /data/ops-status/.ssh/id_ed25519
  known_hosts: /data/ops-status/.ssh/known_hosts
  allow_agent: false
  look_for_keys: false
  allow_unknown_host_keys: false
  connect_timeout: 8
  remote_command: /usr/local/bin/supervisor_status_readonly.sh

servers:
  - name: pc1
    host: 1.2.3.4

  - name: pc2
    host: 5.6.7.8

<!DOCTYPE html>
<html lang="zh-CN">
<head>
  <meta charset="UTF-8" />
  <meta name="viewport" content="width=device-width, initial-scale=1.0" />
  <title>Supervisor 服务状态</title>
  <style>
    :root {
      --bg: #f8fafc;
      --panel: #ffffff;
      --text: #0f172a;
      --muted: #64748b;
      --border: #e2e8f0;
      --primary: #2563eb;
      --ok-bg: #ecfdf5;
      --ok-border: #bbf7d0;
      --ok-text: #166534;
      --error-bg: #fef2f2;
      --error-border: #fecaca;
      --error-text: #991b1b;
      --shadow: 0 14px 36px rgba(15, 23, 42, 0.10);
    }

    * { box-sizing: border-box; }

    body {
      margin: 0;
      background: var(--bg);
      color: var(--text);
      font: 14px/1.5 -apple-system, BlinkMacSystemFont, "Segoe UI", "PingFang SC",
            "Hiragino Sans GB", "Microsoft YaHei", sans-serif;
    }

    .page {
      max-width: 1100px;
      margin: 0 auto;
      padding: 18px;
    }

    .header {
      background: var(--panel);
      border: 1px solid var(--border);
      border-radius: 18px;
      box-shadow: var(--shadow);
      padding: 18px;
      margin-bottom: 16px;
    }

    .header-top {
      display: flex;
      justify-content: space-between;
      gap: 12px;
      align-items: flex-start;
      flex-wrap: wrap;
    }

    h1 {
      font-size: 22px;
      margin: 0 0 6px;
    }

    .muted {
      color: var(--muted);
      font-size: 13px;
    }

    .toolbar {
      display: flex;
      gap: 10px;
      margin-top: 14px;
      flex-wrap: wrap;
    }

    .toolbar input {
      flex: 1 1 280px;
      min-width: 220px;
      height: 40px;
      padding: 0 14px;
      border: 1px solid var(--border);
      border-radius: 12px;
      background: #fff;
      outline: none;
    }

    .toolbar button, .header a {
      height: 40px;
      padding: 0 16px;
      border: 0;
      border-radius: 12px;
      cursor: pointer;
      font-weight: 600;
      text-decoration: none;
      display: inline-flex;
      align-items: center;
      justify-content: center;
    }

    .btn-primary {
      background: var(--primary);
      color: #fff;
    }

    .btn-light {
      background: #eff6ff;
      color: #1d4ed8;
    }

    .summary {
      display: grid;
      grid-template-columns: repeat(auto-fit, minmax(180px, 1fr));
      gap: 12px;
      margin-top: 14px;
    }

    .stat {
      background: #fff;
      border: 1px solid var(--border);
      border-radius: 14px;
      padding: 14px;
    }

    .stat .label {
      color: var(--muted);
      font-size: 12px;
      margin-bottom: 4px;
    }

    .stat .value {
      font-size: 22px;
      font-weight: 700;
    }

    .server-grid {
      display: grid;
      grid-template-columns: repeat(auto-fit, minmax(320px, 1fr));
      gap: 14px;
    }

    .server-card {
      background: var(--panel);
      border: 1px solid var(--border);
      border-radius: 18px;
      box-shadow: var(--shadow);
      overflow: hidden;
    }

    .server-head {
      padding: 16px 18px;
      border-bottom: 1px solid var(--border);
      display: flex;
      align-items: flex-start;
      justify-content: space-between;
      gap: 10px;
    }

    .server-title {
      font-size: 18px;
      font-weight: 700;
      margin: 0 0 4px;
    }

    .server-subtitle {
      color: var(--muted);
      font-size: 13px;
    }

    .badge {
      display: inline-flex;
      align-items: center;
      justify-content: center;
      min-width: 80px;
      padding: 6px 10px;
      border-radius: 999px;
      font-size: 12px;
      font-weight: 700;
      background: var(--ok-bg);
      color: var(--ok-text);
      border: 1px solid var(--ok-border);
    }

    .server-error .badge {
      background: var(--error-bg);
      color: var(--error-text);
      border-color: var(--error-border);
    }

    .server-body {
      padding: 16px 18px;
      display: grid;
      gap: 10px;
    }

    .service-item {
      border: 1px solid var(--ok-border);
      background: var(--ok-bg);
      border-radius: 14px;
      padding: 12px 14px;
    }

    .service-name {
      font-weight: 700;
      margin-bottom: 4px;
      word-break: break-all;
    }

    .service-meta {
      color: var(--ok-text);
      font-size: 13px;
    }

    .empty-state, .error-state {
      border-radius: 14px;
      padding: 14px;
      font-size: 14px;
    }

    .empty-state {
      border: 1px dashed var(--border);
      color: var(--muted);
      background: #fff;
    }

    .error-state {
      border: 1px solid var(--error-border);
      color: var(--error-text);
      background: var(--error-bg);
      white-space: pre-wrap;
      word-break: break-word;
    }

    @media (max-width: 768px) {
      .page { padding: 12px; }
      .header { padding: 14px; }
    }
  </style>
</head>
<body>
  <div class="page">
    <section class="header">
      <div class="header-top">
        <div>
          <h1>Supervisor 服务状态面板</h1>
          <div class="muted" id="metaText">正在加载...</div>
        </div>
        <a href="#" class="btn-light" id="closePanelLink">收起窗口</a>
      </div>

      <div class="toolbar">
        <input id="searchInput" type="search" placeholder="按服务名筛选，仅显示运行中的服务" />
        <button class="btn-primary" id="refreshBtn" type="button">立即刷新</button>
      </div>

      <div class="summary" id="summary"></div>
    </section>

    <section class="server-grid" id="serverGrid"></section>
  </div>

  <script>
    const state = {
      query: '',
      data: null,
      isLoading: false,
    };

    function escapeHtml(value) {
      return String(value ?? '')
        .replaceAll('&', '&amp;')
        .replaceAll('<', '&lt;')
        .replaceAll('>', '&gt;')
        .replaceAll('"', '&quot;')
        .replaceAll("'", '&#39;');
    }

    function fmtTime(value) {
      if (!value) return '-';
      try {
        return new Date(value).toLocaleString('zh-CN', { hour12: false });
      } catch (e) {
        return value;
      }
    }

    function buildSummary(data) {
      const okServers = data.servers.filter(item => item.reachable).length;
      const cards = [
        { label: '主机数', value: data.server_count ?? data.servers.length },
        { label: '可连接主机', value: okServers },
        { label: '运行中服务总数', value: data.running_total ?? 0 },
      ];

      return cards.map(item => `
        <div class="stat">
          <div class="label">${escapeHtml(item.label)}</div>
          <div class="value">${escapeHtml(item.value)}</div>
        </div>
      `).join('');
    }

    function buildServerCard(server, query) {
      const q = query.trim().toLowerCase();
      const services = (server.running_services || []).filter(item => {
        if (!q) return true;
        return item.name.toLowerCase().includes(q);
      });

      const cardClass = server.reachable ? 'server-card' : 'server-card server-error';

      let bodyHtml = '';
      if (!server.reachable) {
        bodyHtml = `<div class="error-state">${escapeHtml(server.error || '连接失败')}</div>`;
      } else if (!services.length && q) {
        bodyHtml = `<div class="empty-state">当前主机没有匹配“${escapeHtml(query)}”的运行中服务。</div>`;
      } else if (!services.length) {
        bodyHtml = '<div class="empty-state">当前主机没有运行中的 Supervisor 服务。</div>';
      } else {
        bodyHtml = services.map(item => `
          <div class="service-item">
            <div class="service-name">${escapeHtml(item.name)}</div>
            <div class="service-meta">运行时长：${escapeHtml(item.uptime || '未知')}</div>
          </div>
        `).join('');
      }

      return `
        <article class="${cardClass}">
          <div class="server-head">
            <div>
              <div class="server-title">${escapeHtml(server.name)}</div>
              <div class="server-subtitle">${escapeHtml(server.host)} · 采集时间 ${escapeHtml(fmtTime(server.collected_at))}</div>
            </div>
            <div class="badge">${server.reachable ? `运行中 ${escapeHtml(server.running_count || 0)}` : '连接失败'}</div>
          </div>
          <div class="server-body">${bodyHtml}</div>
        </article>
      `;
    }

    function render() {
      const summary = document.getElementById('summary');
      const serverGrid = document.getElementById('serverGrid');
      const metaText = document.getElementById('metaText');

      if (!state.data) {
        summary.innerHTML = '';
        serverGrid.innerHTML = '<div class="empty-state">暂无数据。</div>';
        metaText.textContent = state.isLoading ? '正在加载...' : '暂无数据';
        return;
      }

      summary.innerHTML = buildSummary(state.data);
      metaText.textContent = `最近更新时间：${fmtTime(state.data.generated_at)}，仅展示 RUNNING 的 Supervisor 服务。`;
      serverGrid.innerHTML = state.data.servers.map(server => buildServerCard(server, state.query)).join('');
    }

    async function fetchStatus() {
      state.isLoading = true;
      render();
      try {
        const response = await fetch('/ops-api/status', {
          method: 'GET',
          headers: { 'Accept': 'application/json' },
          cache: 'no-store'
        });
        if (!response.ok) {
          throw new Error(`HTTP ${response.status}`);
        }
        state.data = await response.json();
      } catch (error) {
        state.data = {
          generated_at: new Date().toISOString(),
          server_count: 0,
          running_total: 0,
          servers: [{
            name: '状态接口',
            host: '/ops-api/status',
            reachable: false,
            error: error.message || String(error),
            running_services: [],
            running_count: 0,
            collected_at: new Date().toISOString(),
          }]
        };
      } finally {
        state.isLoading = false;
        render();
      }
    }

    document.getElementById('refreshBtn').addEventListener('click', fetchStatus);
    document.getElementById('searchInput').addEventListener('input', (event) => {
      state.query = event.target.value || '';
      render();
    });
    document.getElementById('closePanelLink').addEventListener('click', (event) => {
      event.preventDefault();
      window.parent.postMessage({ type: 'OPS_WIDGET_CLOSE' }, window.location.origin);
    });

    window.addEventListener('message', (event) => {
      if (event.origin !== window.location.origin) return;
      if (event.data && event.data.type === 'OPS_WIDGET_REFRESH') {
        fetchStatus();
      }
    });

    fetchStatus();
    window.setInterval(fetchStatus, 30000);
  </script>
</body>
</html>

(function () {
  if (window.__OPS_STATUS_WIDGET_LOADED__) return;
  window.__OPS_STATUS_WIDGET_LOADED__ = true;

  const BUTTON_ID = 'ops-status-widget-button';
  const PANEL_ID = 'ops-status-widget-panel';
  const IFRAME_ID = 'ops-status-widget-iframe';
  const STYLE_ID = 'ops-status-widget-style';
  const PANEL_URL = '/ops-widget/index.html';

  function injectStyle() {
    if (document.getElementById(STYLE_ID)) return;

    const style = document.createElement('style');
    style.id = STYLE_ID;
    style.textContent = `
      #${BUTTON_ID} {
        position: fixed;
        right: 24px;
        bottom: 24px;
        width: 60px;
        height: 60px;
        border: 0;
        border-radius: 999px;
        background: #2563eb;
        color: #fff;
        cursor: pointer;
        font: 700 13px/1 -apple-system, BlinkMacSystemFont, "Segoe UI", sans-serif;
        box-shadow: 0 14px 30px rgba(37, 99, 235, 0.35);
        z-index: 2147483000;
      }

      #${BUTTON_ID}:hover {
        transform: translateY(-1px);
      }

      #${PANEL_ID} {
        position: fixed;
        right: 24px;
        bottom: 96px;
        width: min(860px, calc(100vw - 32px));
        height: min(680px, calc(100vh - 128px));
        border: 1px solid rgba(148, 163, 184, 0.35);
        border-radius: 18px;
        overflow: hidden;
        background: #fff;
        box-shadow: 0 22px 48px rgba(15, 23, 42, 0.22);
        z-index: 2147483000;
        display: none;
      }

      #${PANEL_ID}.open {
        display: block;
      }

      #${IFRAME_ID} {
        width: 100%;
        height: 100%;
        border: 0;
        background: #f8fafc;
      }

      @media (max-width: 768px) {
        #${BUTTON_ID} {
          right: 14px;
          bottom: 14px;
        }

        #${PANEL_ID} {
          right: 8px;
          left: 8px;
          bottom: 82px;
          width: auto;
          height: calc(100vh - 96px);
        }
      }
    `;
    document.head.appendChild(style);
  }

  function buildWidget() {
    injectStyle();

    const button = document.createElement('button');
    button.id = BUTTON_ID;
    button.type = 'button';
    button.title = '查看 Supervisor 服务状态';
    button.textContent = '状态';

    const panel = document.createElement('section');
    panel.id = PANEL_ID;
    panel.setAttribute('aria-hidden', 'true');
    panel.innerHTML = `<iframe id="${IFRAME_ID}" src="${PANEL_URL}" title="Supervisor 服务状态面板"></iframe>`;

    document.body.appendChild(button);
    document.body.appendChild(panel);

    function openPanel() {
      panel.classList.add('open');
      panel.setAttribute('aria-hidden', 'false');
      button.textContent = '收起';
    }

    function closePanel() {
      panel.classList.remove('open');
      panel.setAttribute('aria-hidden', 'true');
      button.textContent = '状态';
    }

    function togglePanel() {
      if (panel.classList.contains('open')) {
        closePanel();
      } else {
        openPanel();
      }
    }

    button.addEventListener('click', togglePanel);

    window.addEventListener('message', (event) => {
      if (event.origin !== window.location.origin) return;
      if (event.data && event.data.type === 'OPS_WIDGET_CLOSE') {
        closePanel();
      }
    });

    window.addEventListener('keydown', (event) => {
      if (event.key === 'Escape') {
        closePanel();
      }
    });
  }

  if (document.readyState === 'loading') {
    document.addEventListener('DOMContentLoaded', buildWidget);
  } else {
    buildWidget();
  }
})();

# 更新系统软件源，安装Python虚拟环境工具（系统自带Python没有venv模块）
apt update && apt install -y python3.10-venv

# 创建独立的Python虚拟环境（目录：/data/ops-status/.venv）
python3 -m venv /data/ops-status/.venv

# 激活虚拟环境
source /data/ops-status/.venv/bin/activate

# 升级pip工具
pip install -U pip

# 一键安装监控脚本所有依赖
pip install -r /data/ops-status/requirements.txt

# 用监控机的私钥，连接生产机的只读账号
ssh -i /data/ops-status/.ssh/id_ed25519 svc_supervisor_view@IP1
ssh -i /data/ops-status/.ssh/id_ed25519 svc_supervisor_view@IP2

# 1. 编写systemd服务配置文件到系统目录
vim /etc/systemd/system/ops-status.service

[Unit]
Description=ops-status gunicorn service
After=network.target

[Service]
Type=simple
User=root
Group=root
WorkingDirectory=/data/ops-status
Environment=INVENTORY_PATH=/data/ops-status/inventory.yml
Environment=PORT=18081
ExecStart=/data/ops-status/.venv/bin/gunicorn -w 2 -b 127.0.0.1:18081 app:app
Restart=always
RestartSec=3

[Install]
WantedBy=multi-user.target

# 2. 重载systemd，让系统识别新的服务配置
systemctl daemon-reload

# 3. 设置服务【开机自启】，并【立即启动】服务
systemctl enable --now ops-status

# 4. 查看服务状态（验证是否运行成功）
systemctl status ops-status

curl http://127.0.0.1:18081/health
curl http://127.0.0.1:18081/api/status

cat /etc/nginx/conf.d/ops-widget.conf 
server {
    listen 8081;
    server_name _;

    access_log  /data/ops-status/nginx/ops-widget.access.log;
    error_log   /data/ops-status/nginx/ops-widget.error.log;
    location ^~ /ops-widget/ {
        alias /data/ops-status/web/;
        try_files $uri =404;
        add_header Cache-Control "no-store" always;
    }

    location = /ops-api/status {
        proxy_pass http://127.0.0.1:18081/api/status;
        proxy_http_version 1.1;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_read_timeout 20s;
        proxy_connect_timeout 5s;
    }

    location = /ops-api/status/refresh {
        proxy_pass http://127.0.0.1:18081/api/status/refresh;
        proxy_http_version 1.1;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_read_timeout 20s;
        proxy_connect_timeout 5s;
    }

    location = /ops-api/health {
        proxy_pass http://127.0.0.1:18081/health;
        proxy_http_version 1.1;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_read_timeout 20s;
        proxy_connect_timeout 5s;
    }

	# jenkins端口
    location / {
        proxy_pass http://127.0.0.1:15107;
        proxy_http_version 1.1;
        proxy_set_header Host $host;
        proxy_set_header X-Real-IP $remote_addr;
        proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto $scheme;
        proxy_set_header Connection "";
        proxy_set_header Accept-Encoding "";
        proxy_redirect off;
        sub_filter_once on;
        sub_filter '</body>' '<script src="/ops-widget/widget.js" defer></script></body>';
    }
}

nginx -t 
nginx -s reload