#!/usr/bin/env bash
# 起 Raylight 生产服务：常驻 ComfyUI（4卡FSDP集群）+ raylight-server.py（对外契约跟旧 gateway.py
# 一致，见 notes.md）。已经在跑的会先停掉。
#
# 停端口要按端口找 PID，不要用 pkill -f——同理 minimax-h3/start-all.sh 里的教训。
set -uo pipefail
cd /data/models/raylight

COMFY_PORT=8188
SERVER_PORT="${PORT:-8100}"

for PORT in $COMFY_PORT $SERVER_PORT; do
  PID=$(ss -lptn "sport = :$PORT" | grep -o "pid=[0-9]*" | cut -d= -f2 | sort -u)
  [ -n "$PID" ] && { echo "停 $PORT (PID $PID)"; kill $PID; }
done
sleep 3

cd ComfyUI
export PYTORCH_CUDA_ALLOC_CONF=expandable_segments:True
# ponytail: 保留最近10轮日志（.1最新~.10最旧）——2026-08-23密集测试期间发现只留1份不够用，
# 连续崩溃重启会把上上轮的崩溃现场覆盖掉，排查不了；10轮之前的还是不留，不做完整logrotate
for i in 9 8 7 6 5 4 3 2 1; do
  [ -f "/data/models/raylight/comfyui-server.log.$i" ] && \
    mv -f "/data/models/raylight/comfyui-server.log.$i" "/data/models/raylight/comfyui-server.log.$((i+1))"
done
mv -f /data/models/raylight/comfyui-server.log /data/models/raylight/comfyui-server.log.1 2>/dev/null
setsid nohup .venv/bin/python main.py --listen 0.0.0.0 --port "$COMFY_PORT" \
  > /data/models/raylight/comfyui-server.log 2>&1 < /dev/null &
disown
cd ..

echo "等 ComfyUI 起来"
for i in $(seq 1 30); do
  sleep 3
  CODE=$(curl -s --max-time 3 -o /dev/null -w '%{http_code}' "http://127.0.0.1:$COMFY_PORT/system_stats")
  [ "$CODE" = "200" ] && { echo "  ComfyUI ready"; break; }
done

setsid nohup ComfyUI/.venv/bin/python -m uvicorn server:app --host 0.0.0.0 --port "$SERVER_PORT" \
  > /data/models/raylight/server.log 2>&1 < /dev/null &
disown

echo "等 raylight-server 起来"
for i in $(seq 1 20); do
  sleep 2
  CODE=$(curl -s --max-time 3 -o /dev/null -w '%{http_code}' "http://127.0.0.1:$SERVER_PORT/health")
  [ "$CODE" = "200" ] && { echo "  raylight-server ready"; break; }
done
curl -s "http://127.0.0.1:$SERVER_PORT/health"; echo