#!/usr/bin/env python3 """Continuously record MAA Online memory/swap behaviour into a log. Run once per minute by maa-online-memlog.timer. Each run appends one row to /logs/memlog.csv (auto-rotated at 5 MB, one generation kept) and prints a compact line to the journal. The numbers tracked here are the ones that matter for the glibc malloc arena behaviour: the service cgroup's anonymous memory, how much of it has been pushed into swap, and how many large (>= 32 MB) anonymous mappings the process holds. An arena-bloat regression shows up as the big-region count climbing while the service is streaming. """ from __future__ import annotations import os import re import subprocess import sys import time import urllib.error import urllib.request from pathlib import Path ROOT = Path(os.environ.get("MAA_ONLINE_ROOT") or Path(__file__).resolve().parent.parent) LOG_DIR = Path(os.environ.get("MAA_ONLINE_LOG_DIR", ROOT / "logs")) LOG = Path(os.environ.get("MAA_ONLINE_MEMLOG", LOG_DIR / "memlog.csv")) MAX_BYTES = int(os.environ.get("MAA_ONLINE_MEMLOG_MAX_BYTES", 5 * 1024 * 1024)) CG = Path("/sys/fs/cgroup/system.slice/maa-online-server.service") API = os.environ.get("MAA_ONLINE_API", "http://127.0.0.1:22888") FIELDS = [ "timestamp", "cloud_state", "pid", "proc_uptime_s", "cg_current_mb", "cg_peak_mb", "cg_swap_mb", "cg_swap_peak_mb", "rss_mb", "anon_regions", "big_anon_regions", "anon_total_mb", "sys_mem_avail_mb", "sys_swap_used_mb", "sys_swap_free_mb", ] MIB = 1048576 def read_text(path: Path, default: str = "") -> str: try: return path.read_text(encoding="utf-8").strip() except OSError: return default def cg_value(name: str) -> int: raw = read_text(CG / name, "0") try: return int(raw) except ValueError: return 0 def mb(value: int) -> float: return round(value / MIB, 1) def main_pid() -> int: try: out = subprocess.run( ["systemctl", "show", "-p", "MainPID", "--value", "maa-online-server.service"], capture_output=True, text=True, timeout=10, check=False, ).stdout.strip() return int(out or 0) except (OSError, ValueError, subprocess.SubprocessError): return 0 def proc_uptime(pid: int) -> int: stat = read_text(Path(f"/proc/{pid}/stat")) if not stat: return 0 try: start_ticks = int(stat.rsplit(")", 1)[1].split()[19]) except (IndexError, ValueError): return 0 try: with open("/proc/uptime") as handle: uptime = float(handle.read().split()[0]) except (OSError, ValueError): return 0 return int(uptime - start_ticks / os.sysconf("SC_CLK_TCK")) def proc_memory(pid: int) -> tuple[float, int, int, float]: """Return (rss_mb, anon_regions, big_anon_regions, anon_total_mb).""" rss = anon_regions = big_anon_regions = 0 anon_total = 0 try: with open(f"/proc/{pid}/smaps") as handle: for line in handle: if line[0] in "0123456789abcdef" and "-" in line.split(" ", 1)[0]: parts = line.split() start, end = parts[0].split("-") size = (int(end, 16) - int(start, 16)) // 1024 path = parts[5] if len(parts) > 5 else "[anon]" if path == "[anon]": anon_regions += 1 anon_total += size if size >= 32 * 1024: big_anon_regions += 1 elif line.startswith("Rss:"): rss += int(line.split()[1]) except OSError: pass return round(rss / 1024, 1), anon_regions, big_anon_regions, round(anon_total / 1024, 1) def system_memory() -> tuple[float, float, float]: values: dict[str, int] = {} try: with open("/proc/meminfo") as handle: for line in handle: key, _, rest = line.partition(":") values[key] = int(rest.split()[0]) except (OSError, ValueError, IndexError): return 0.0, 0.0, 0.0 return ( mb(values.get("MemAvailable", 0) * 1024), mb((values.get("SwapTotal", 0) - values.get("SwapFree", 0)) * 1024), mb(values.get("SwapFree", 0) * 1024), ) def cloud_state() -> str: try: with urllib.request.urlopen(f"{API}/info", timeout=5) as response: body = response.read().decode("utf-8", "replace") except (urllib.error.URLError, OSError, ValueError): return "unreachable" match = re.search(r'"status"\s*:\s*"([^"]+)"', body) return match.group(1) if match else "unknown" def rotate_if_needed() -> None: try: if LOG.exists() and LOG.stat().st_size > MAX_BYTES: backup = LOG.with_name(LOG.name + ".1") backup.unlink(missing_ok=True) LOG.rename(backup) except OSError: pass def main() -> int: LOG_DIR.mkdir(parents=True, exist_ok=True) rotate_if_needed() pid = main_pid() state = cloud_state() if pid: rss, anon_regions, big_regions, anon_total = proc_memory(pid) uptime = proc_uptime(pid) else: rss = anon_total = 0.0 anon_regions = big_regions = uptime = 0 mem_avail, swap_used, swap_free = system_memory() row = [ time.strftime("%Y-%m-%dT%H:%M:%S%z"), state, pid, uptime, mb(cg_value("memory.current")), mb(cg_value("memory.peak")), mb(cg_value("memory.swap.current")), mb(cg_value("memory.swap.peak")), rss, anon_regions, big_regions, anon_total, mem_avail, swap_used, swap_free, ] fresh = not LOG.exists() or LOG.stat().st_size == 0 with LOG.open("a", encoding="utf-8") as handle: if fresh: handle.write(",".join(FIELDS) + "\n") handle.write(",".join(str(value) for value in row) + "\n") print( f"{row[0]} state={state} pid={pid} " f"cg={row[4]}MB peak={row[5]}MB cg_swap={row[6]}MB " f"rss={rss}MB anon_regions={anon_regions} big={big_regions} anon={anon_total}MB " f"mem_avail={mem_avail}MB swap_used={swap_used}MB" ) return 0 if __name__ == "__main__": sys.exit(main())