feat: 增加 systemd 定时执行与断点续跑

- backend/server.py: 信令 WebSocket 被远端关闭或视频流停滞超过阈值时自动拆除会话，由 finally 释放网易云端实例，避免会话空挂、后续 /start 一直拿到 503 - bin/maa-online-daily-run: 新增日常任务包装脚本，等待 API 就绪、启动云游戏、停滞看门狗（连续 ScreencapFailed 或 Disconnected）、失败即释放云会话并最多重试 3 次 - bin/maa-resume-tasks.py: 新增断点续跑规划器，重试时只运行未出现 Completed 的任务，全部完成则直接判定成功，不再开启新会话；StartUp 永不跳过 - systemd/: 新增 server/daily/update 服务与定时器模板，每天 04:00 起随机 0-2 小时执行 - bin/maa-online-install-systemd: 渲染 __ROOT__ 并安装、启用上述单元 - README.md, user_config/README.md: 补充自动定时执行、断点续跑与已知限制说明 - .gitignore: 忽略生成的 config/tasks/daily-resume.toml
2026-09-19 16:45:48 +08:00
parent 73250c91b6
commit 0264053b08
12 changed files with 481 additions and 3 deletions
@@ -0,0 +1,142 @@
+#!/usr/bin/env bash
+# MAA Online daily runner, designed for systemd oneshot execution.
+#
+# Flow: wait for the cloud API -> plan remaining tasks from earlier attempts ->
+# start a cloud session -> run the daily task file under a stall watchdog ->
+# on failure release the cloud session and retry (up to MAA_ONLINE_MAX_ATTEMPTS).
+#
+# Retries use resume mode: bin/maa-resume-tasks.py inspects the earlier
+# attempts' logs and only the tasks that never reported "<Type> Completed" are
+# run again, so a retry does not re-burn cloud-game minutes on finished work.
+#
+# Manual catch-up run (for example after a failed night, using the remaining
+# daily quota):
+#   MAA_ONLINE_RESUME_FROM="artifacts/maa-daily-<stamp>.log" bin/maa-online-daily-run
+set -uo pipefail
+
+ROOT="${MAA_ONLINE_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
+API="${MAA_ONLINE_API:-http://127.0.0.1:22888}"
+LOG_DIR="${MAA_ONLINE_LOG_DIR:-$ROOT/artifacts}"
+MAA_ONLINE_BIN="${MAA_ONLINE_BIN:-$ROOT/bin/maa-online}"
+PYTHON="${MAA_ONLINE_PYTHON:-$ROOT/runtime/venv/bin/python}"
+RESUME_HELPER="$ROOT/bin/maa-resume-tasks.py"
+DAILY_SOURCE="${MAA_ONLINE_DAILY_SOURCE:-$ROOT/config/tasks/daily.toml}"
+RESUME_FILE="${MAA_ONLINE_RESUME_OUT:-$ROOT/config/tasks/daily-resume.toml}"
+
+MAX_ATTEMPTS="${MAA_ONLINE_MAX_ATTEMPTS:-3}"
+STALL_FAILS="${MAA_ONLINE_STALL_FAILS:-12}"              # consecutive ScreencapFailed (~24s+)
+ATTEMPT_MAX_SECONDS="${MAA_ONLINE_ATTEMPT_MAX_SECONDS:-3000}"  # per-attempt ceiling
+
+mkdir -p "$LOG_DIR"
+
+api() { # api <METHOD> <PATH>
+    curl --noproxy '*' -fsS --max-time 180 -X "$1" "$API$2"
+}
+
+echo "[daily-run] $(date --iso-8601=seconds) waiting for cloud API at $API ..."
+ready=""
+for _ in $(seq 1 90); do
+    if api GET /info >/dev/null 2>&1; then ready=1; break; fi
+    sleep 2
+done
+if [ -z "$ready" ]; then
+    echo "[daily-run] ERROR: cloud API not ready after 180s" >&2
+    exit 2
+fi
+
+REASON=""
+run_daily_watchdog() { # arg1: log file, arg2: task file name; sets REASON on abort
+    local log="$1"
+    local task="$2"
+    REASON=""
+    "$MAA_ONLINE_BIN" run "$task" --batch -v --log-file="$log" &
+    local maa_pid=$!
+    while kill -0 "$maa_pid" 2>/dev/null; do
+        sleep 5
+        if [ $(( $(date +%s) - attempt_epoch )) -gt "$ATTEMPT_MAX_SECONDS" ]; then
+            REASON="attempt exceeded ${ATTEMPT_MAX_SECONDS}s ceiling"
+            break
+        fi
+        local streak
+        streak=$(awk '/ScreencapFailed/{c++; next} {c=0} END{print c+0}' "$log" 2>/dev/null)
+        if [ "${streak:-0}" -ge "$STALL_FAILS" ]; then
+            REASON="screencap stalled (${streak} consecutive failures; cloud session likely dead)"
+            break
+        fi
+        if grep -qE '\] Disconnected$' "$log" 2>/dev/null; then
+            REASON="MAA reported Disconnected (cloud session lost)"
+            break
+        fi
+    done
+    if [ -n "$REASON" ]; then
+        echo "[daily-run] watchdog: $REASON; aborting attempt" >&2
+        kill -INT "$maa_pid" 2>/dev/null
+        for _ in $(seq 1 6); do kill -0 "$maa_pid" 2>/dev/null || break; sleep 5; done
+        kill -0 "$maa_pid" 2>/dev/null && kill -9 "$maa_pid" 2>/dev/null
+        wait "$maa_pid" 2>/dev/null
+        return 1
+    fi
+    wait "$maa_pid" || { REASON="maa exited rc=$?"; return 1; }
+    return 0
+}
+
+attempt_logs=()
+if [ -n "${MAA_ONLINE_RESUME_FROM:-}" ]; then
+    read -r -a seeded <<< "$MAA_ONLINE_RESUME_FROM"
+    attempt_logs+=("${seeded[@]}")
+    echo "[daily-run] seeded resume planning with: ${attempt_logs[*]}"
+fi
+
+plan_tasks() { # echoes the task file to run ("daily", "daily-resume" or "NONE")
+    if [ "${#attempt_logs[@]}" -eq 0 ]; then
+        echo daily
+        return
+    fi
+    local result
+    result=$("$PYTHON" "$RESUME_HELPER" "${attempt_logs[@]}" \
+        --source "$DAILY_SOURCE" --out "$RESUME_FILE" 2>&1 | tail -n1)
+    case "$result" in
+        NONE)
+            echo NONE
+            ;;
+        RESUME_TASKS=*)
+            echo "[daily-run] resume plan: ${result#RESUME_TASKS=}" >&2
+            echo daily-resume
+            ;;
+        *)
+            echo "[daily-run] WARN: resume planning failed (${result}); using full daily" >&2
+            echo daily
+            ;;
+    esac
+}
+
+attempt=0
+while [ "$attempt" -lt "$MAX_ATTEMPTS" ]; do
+    attempt=$((attempt + 1))
+    TASK_FILE=$(plan_tasks)
+    if [ "$TASK_FILE" = "NONE" ]; then
+        echo "[daily-run] $(date --iso-8601=seconds) every daily task already completed in an earlier attempt; nothing left to do"
+        exit 0
+    fi
+    echo "[daily-run] attempt $attempt/$MAX_ATTEMPTS: task file '$TASK_FILE'; starting cloud game ..."
+    if ! api POST /start; then
+        echo "[daily-run] attempt $attempt: /start failed (cloud service or token issue)" >&2
+        sleep 20
+        continue
+    fi
+    RUN_LOG="$LOG_DIR/maa-daily-$(date +%Y%m%d-%H%M%S).log"
+    echo "[daily-run] attempt $attempt: running '$TASK_FILE' (log: $RUN_LOG) ..."
+    attempt_epoch=$(date +%s)
+    if run_daily_watchdog "$RUN_LOG" "$TASK_FILE"; then
+        echo "[daily-run] $(date --iso-8601=seconds) daily completed successfully (attempt $attempt)"
+        exit 0
+    fi
+    attempt_logs+=("$RUN_LOG")
+    echo "[daily-run] attempt $attempt failed: ${REASON:-unknown}; releasing cloud session" >&2
+    api POST /exit >/dev/null 2>&1 || true
+    sleep 30
+done
+
+echo "[daily-run] ERROR: daily failed after $MAX_ATTEMPTS attempts (last: ${REASON:-unknown})" >&2
+echo "[daily-run] if the failure mentions login/token, re-run: runtime/venv/bin/python backend/login.py request <phone>" >&2
+exit 1