feat: 增加 systemd 定时执行与断点续跑

- backend/server.py: 信令 WebSocket 被远端关闭或视频流停滞超过阈值时自动拆除会话，由 finally 释放网易云端实例，避免会话空挂、后续 /start 一直拿到 503 - bin/maa-online-daily-run: 新增日常任务包装脚本，等待 API 就绪、启动云游戏、停滞看门狗（连续 ScreencapFailed 或 Disconnected）、失败即释放云会话并最多重试 3 次 - bin/maa-resume-tasks.py: 新增断点续跑规划器，重试时只运行未出现 Completed 的任务，全部完成则直接判定成功，不再开启新会话；StartUp 永不跳过 - systemd/: 新增 server/daily/update 服务与定时器模板，每天 04:00 起随机 0-2 小时执行 - bin/maa-online-install-systemd: 渲染 __ROOT__ 并安装、启用上述单元 - README.md, user_config/README.md: 补充自动定时执行、断点续跑与已知限制说明 - .gitignore: 忽略生成的 config/tasks/daily-resume.toml
2026-09-19 16:45:48 +08:00
parent 73250c91b6
commit 0264053b08
12 changed files with 481 additions and 3 deletions
@@ -0,0 +1,142 @@
+#!/usr/bin/env bash
+# MAA Online daily runner, designed for systemd oneshot execution.
+#
+# Flow: wait for the cloud API -> plan remaining tasks from earlier attempts ->
+# start a cloud session -> run the daily task file under a stall watchdog ->
+# on failure release the cloud session and retry (up to MAA_ONLINE_MAX_ATTEMPTS).
+#
+# Retries use resume mode: bin/maa-resume-tasks.py inspects the earlier
+# attempts' logs and only the tasks that never reported "<Type> Completed" are
+# run again, so a retry does not re-burn cloud-game minutes on finished work.
+#
+# Manual catch-up run (for example after a failed night, using the remaining
+# daily quota):
+#   MAA_ONLINE_RESUME_FROM="artifacts/maa-daily-<stamp>.log" bin/maa-online-daily-run
+set -uo pipefail
+
+ROOT="${MAA_ONLINE_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
+API="${MAA_ONLINE_API:-http://127.0.0.1:22888}"
+LOG_DIR="${MAA_ONLINE_LOG_DIR:-$ROOT/artifacts}"
+MAA_ONLINE_BIN="${MAA_ONLINE_BIN:-$ROOT/bin/maa-online}"
+PYTHON="${MAA_ONLINE_PYTHON:-$ROOT/runtime/venv/bin/python}"
+RESUME_HELPER="$ROOT/bin/maa-resume-tasks.py"
+DAILY_SOURCE="${MAA_ONLINE_DAILY_SOURCE:-$ROOT/config/tasks/daily.toml}"
+RESUME_FILE="${MAA_ONLINE_RESUME_OUT:-$ROOT/config/tasks/daily-resume.toml}"
+
+MAX_ATTEMPTS="${MAA_ONLINE_MAX_ATTEMPTS:-3}"
+STALL_FAILS="${MAA_ONLINE_STALL_FAILS:-12}"              # consecutive ScreencapFailed (~24s+)
+ATTEMPT_MAX_SECONDS="${MAA_ONLINE_ATTEMPT_MAX_SECONDS:-3000}"  # per-attempt ceiling
+
+mkdir -p "$LOG_DIR"
+
+api() { # api <METHOD> <PATH>
+    curl --noproxy '*' -fsS --max-time 180 -X "$1" "$API$2"
+}
+
+echo "[daily-run] $(date --iso-8601=seconds) waiting for cloud API at $API ..."
+ready=""
+for _ in $(seq 1 90); do
+    if api GET /info >/dev/null 2>&1; then ready=1; break; fi
+    sleep 2
+done
+if [ -z "$ready" ]; then
+    echo "[daily-run] ERROR: cloud API not ready after 180s" >&2
+    exit 2
+fi
+
+REASON=""
+run_daily_watchdog() { # arg1: log file, arg2: task file name; sets REASON on abort
+    local log="$1"
+    local task="$2"
+    REASON=""
+    "$MAA_ONLINE_BIN" run "$task" --batch -v --log-file="$log" &
+    local maa_pid=$!
+    while kill -0 "$maa_pid" 2>/dev/null; do
+        sleep 5
+        if [ $(( $(date +%s) - attempt_epoch )) -gt "$ATTEMPT_MAX_SECONDS" ]; then
+            REASON="attempt exceeded ${ATTEMPT_MAX_SECONDS}s ceiling"
+            break
+        fi
+        local streak
+        streak=$(awk '/ScreencapFailed/{c++; next} {c=0} END{print c+0}' "$log" 2>/dev/null)
+        if [ "${streak:-0}" -ge "$STALL_FAILS" ]; then
+            REASON="screencap stalled (${streak} consecutive failures; cloud session likely dead)"
+            break
+        fi
+        if grep -qE '\] Disconnected$' "$log" 2>/dev/null; then
+            REASON="MAA reported Disconnected (cloud session lost)"
+            break
+        fi
+    done
+    if [ -n "$REASON" ]; then
+        echo "[daily-run] watchdog: $REASON; aborting attempt" >&2
+        kill -INT "$maa_pid" 2>/dev/null
+        for _ in $(seq 1 6); do kill -0 "$maa_pid" 2>/dev/null || break; sleep 5; done
+        kill -0 "$maa_pid" 2>/dev/null && kill -9 "$maa_pid" 2>/dev/null
+        wait "$maa_pid" 2>/dev/null
+        return 1
+    fi
+    wait "$maa_pid" || { REASON="maa exited rc=$?"; return 1; }
+    return 0
+}
+
+attempt_logs=()
+if [ -n "${MAA_ONLINE_RESUME_FROM:-}" ]; then
+    read -r -a seeded <<< "$MAA_ONLINE_RESUME_FROM"
+    attempt_logs+=("${seeded[@]}")
+    echo "[daily-run] seeded resume planning with: ${attempt_logs[*]}"
+fi
+
+plan_tasks() { # echoes the task file to run ("daily", "daily-resume" or "NONE")
+    if [ "${#attempt_logs[@]}" -eq 0 ]; then
+        echo daily
+        return
+    fi
+    local result
+    result=$("$PYTHON" "$RESUME_HELPER" "${attempt_logs[@]}" \
+        --source "$DAILY_SOURCE" --out "$RESUME_FILE" 2>&1 | tail -n1)
+    case "$result" in
+        NONE)
+            echo NONE
+            ;;
+        RESUME_TASKS=*)
+            echo "[daily-run] resume plan: ${result#RESUME_TASKS=}" >&2
+            echo daily-resume
+            ;;
+        *)
+            echo "[daily-run] WARN: resume planning failed (${result}); using full daily" >&2
+            echo daily
+            ;;
+    esac
+}
+
+attempt=0
+while [ "$attempt" -lt "$MAX_ATTEMPTS" ]; do
+    attempt=$((attempt + 1))
+    TASK_FILE=$(plan_tasks)
+    if [ "$TASK_FILE" = "NONE" ]; then
+        echo "[daily-run] $(date --iso-8601=seconds) every daily task already completed in an earlier attempt; nothing left to do"
+        exit 0
+    fi
+    echo "[daily-run] attempt $attempt/$MAX_ATTEMPTS: task file '$TASK_FILE'; starting cloud game ..."
+    if ! api POST /start; then
+        echo "[daily-run] attempt $attempt: /start failed (cloud service or token issue)" >&2
+        sleep 20
+        continue
+    fi
+    RUN_LOG="$LOG_DIR/maa-daily-$(date +%Y%m%d-%H%M%S).log"
+    echo "[daily-run] attempt $attempt: running '$TASK_FILE' (log: $RUN_LOG) ..."
+    attempt_epoch=$(date +%s)
+    if run_daily_watchdog "$RUN_LOG" "$TASK_FILE"; then
+        echo "[daily-run] $(date --iso-8601=seconds) daily completed successfully (attempt $attempt)"
+        exit 0
+    fi
+    attempt_logs+=("$RUN_LOG")
+    echo "[daily-run] attempt $attempt failed: ${REASON:-unknown}; releasing cloud session" >&2
+    api POST /exit >/dev/null 2>&1 || true
+    sleep 30
+done
+
+echo "[daily-run] ERROR: daily failed after $MAX_ATTEMPTS attempts (last: ${REASON:-unknown})" >&2
+echo "[daily-run] if the failure mentions login/token, re-run: runtime/venv/bin/python backend/login.py request <phone>" >&2
+exit 1
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+# Install (or refresh) the systemd units that schedule the MAA Online daily
+# task. Run as root. Units are rendered from systemd/*.tmpl-style sources in
+# this repository with __ROOT__ replaced by the checkout path.
+set -euo pipefail
+
+ROOT="${MAA_ONLINE_ROOT:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)}"
+SRC="$ROOT/systemd"
+DST="${MAA_ONLINE_SYSTEMD_DIR:-/etc/systemd/system}"
+UNITS=(maa-online-server.service maa-daily.service maa-daily.timer maa-update.service maa-update.timer)
+
+if [ "$(id -u)" != "0" ]; then
+    echo "error: must run as root (writes to $DST)" >&2
+    exit 1
+fi
+if [ ! -d "$SRC" ]; then
+    echo "error: $SRC not found" >&2
+    exit 1
+fi
+
+for unit in "${UNITS[@]}"; do
+    sed "s|__ROOT__|$ROOT|g" "$SRC/$unit" > "$DST/$unit"
+    echo "installed $DST/$unit"
+done
+
+systemctl daemon-reload
+systemctl enable --now maa-online-server.service
+systemctl enable maa-daily.timer maa-update.timer
+systemctl start maa-daily.timer maa-update.timer
+
+echo
+systemctl --no-pager list-timers maa-daily.timer maa-update.timer
+cat <<EOF
+
+MAA Online systemd units are active.
+  server log:  journalctl -u maa-online-server -f
+  daily log:   journalctl -u maa-daily -n 50
+  next runs:   systemctl list-timers maa-daily.timer maa-update.timer
+EOF
@@ -0,0 +1,145 @@
+#!/usr/bin/env python3
+"""Build a resume task file from partially finished MAA daily attempts.
+
+A failed daily attempt still completes some tasks (their progress lives on the
+game server), so re-running the whole daily file on a retry wastes cloud-game
+minutes. This helper reads the previous attempts' MAA logs, finds which task
+types already reported "<Type> Completed", and writes a task file containing
+only the remaining tasks. The daily runner (bin/maa-online-daily-run) calls it
+automatically before every retry.
+
+Usage:
+    maa-resume-tasks.py <log> [<log> ...] [--source <daily.toml>] [--out <file>]
+
+Output (stdout, last line):
+    RESUME_TASKS=<name1>,<name2>   remaining tasks written to --out
+    NONE                           nothing left to do
+    ERROR:<reason>                 could not build a resume file (caller should
+                                   fall back to the full daily run)
+
+StartUp is intentionally never skipped: every /start yields a fresh cloud
+instance, and the global `startup = true` flag guarantees the game is launched
+and sitting on the main screen before the remaining tasks run.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import sys
+from pathlib import Path
+
+ROOT = Path(os.environ.get("MAA_ONLINE_ROOT") or Path(__file__).resolve().parent.parent)
+DEFAULT_SOURCE = Path(os.environ.get("MAA_ONLINE_DAILY_SOURCE", ROOT / "config/tasks/daily.toml"))
+DEFAULT_OUT = Path(os.environ.get("MAA_ONLINE_RESUME_OUT", ROOT / "config/tasks/daily-resume.toml"))
+
+TASK_HEADER = re.compile(r"^\[\[tasks\]\]\s*$")
+TYPE_RE = re.compile(r'^type\s*=\s*"([^"]+)"\s*$', re.MULTILINE)
+NAME_RE = re.compile(r'^name\s*=\s*"([^"]+)"\s*$', re.MULTILINE)
+COMPLETED_RE = re.compile(r"^.*\b(\w+) Completed\s*$", re.MULTILINE)
+
+# Task types that are never skipped even when a completion marker is present:
+# a retry always reconnects to a fresh cloud instance and must relaunch/verify
+# the game before the remaining tasks can run.
+NEVER_SKIP = {"StartUp"}
+
+
+def split_source(text: str) -> tuple[str, list[str]]:
+    """Return (prelude, task blocks).
+
+    A block owns the contiguous comment lines directly above its [[tasks]]
+    header, so a "# 1. xxx" section comment travels with its task instead of
+    being left behind in the prelude. Blocks keep their exact formatting.
+    """
+    lines = text.splitlines(keepends=True)
+    header_indexes = [i for i, line in enumerate(lines) if TASK_HEADER.match(line)]
+    if not header_indexes:
+        raise ValueError("no [[tasks]] blocks found in source file")
+
+    starts = []
+    for index in header_indexes:
+        start = index
+        while start - 1 >= 0 and lines[start - 1].lstrip().startswith("#"):
+            start -= 1
+        starts.append(start)
+
+    prelude = "".join(lines[: starts[0]])
+    blocks = []
+    for position, start in enumerate(starts):
+        end = starts[position + 1] if position + 1 < len(starts) else len(lines)
+        blocks.append("".join(lines[start:end]).strip("\n") + "\n")
+    return prelude, blocks
+
+
+def completed_types(log_text: str) -> set[str]:
+    return set(COMPLETED_RE.findall(log_text))
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("logs", nargs="+", help="previous attempts' MAA log files")
+    parser.add_argument("--source", default=str(DEFAULT_SOURCE))
+    parser.add_argument("--out", default=str(DEFAULT_OUT))
+    args = parser.parse_args()
+
+    source = Path(args.source)
+    if not source.is_file():
+        print(f"ERROR:source task file not found: {source}")
+        return 0
+
+    try:
+        prelude, blocks = split_source(source.read_text(encoding="utf-8"))
+    except (OSError, ValueError) as exc:
+        print(f"ERROR:{exc}")
+        return 0
+
+    done: set[str] = set()
+    for log in args.logs:
+        log_path = Path(log)
+        if not log_path.is_file():
+            continue
+        try:
+            done |= completed_types(log_path.read_text(encoding="utf-8", errors="replace"))
+        except OSError as exc:
+            print(f"ERROR:cannot read previous log {log}: {exc}")
+            return 0
+
+    kept: list[str] = []
+    kept_names: list[str] = []
+    for block in blocks:
+        match = TYPE_RE.search(block)
+        if not match:
+            continue
+        task_type = match.group(1)
+        if task_type in done and task_type not in NEVER_SKIP:
+            continue
+        kept.append(block)
+        name_match = NAME_RE.search(block)
+        kept_names.append(name_match.group(1) if name_match else task_type)
+
+    if not kept:
+        print("NONE")
+        return 0
+
+    header = (
+        "# AUTO-GENERATED by bin/maa-resume-tasks.py -- do not edit.\n"
+        f"# Source: {source}\n"
+        f"# Already completed (skipped): {', '.join(sorted(done)) or 'none'}\n\n"
+    )
+    out = Path(args.out)
+    try:
+        out.parent.mkdir(parents=True, exist_ok=True)
+        out.write_text(
+            header + prelude.rstrip("\n") + "\n\n" + "\n".join(kept), encoding="utf-8"
+        )
+    except OSError as exc:
+        print(f"ERROR:cannot write resume file: {exc}")
+        return 0
+
+    print(f"RESUME_TASKS={','.join(kept_names)}")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())