diff --git a/bin/maa-online-memlog b/bin/maa-online-memlog index c85192b..d935bd6 100755 --- a/bin/maa-online-memlog +++ b/bin/maa-online-memlog @@ -5,11 +5,17 @@ Run once per minute by maa-online-memlog.timer. Each run appends one row to /logs/memlog.csv (auto-rotated at 5 MB, one generation kept) and prints a compact line to the journal. -The numbers tracked here are the ones that matter for the glibc malloc arena -behaviour: the service cgroup's anonymous memory, how much of it has been -pushed into swap, and how many large (>= 32 MB) anonymous mappings the process -holds. An arena-bloat regression shows up as the big-region count climbing -while the service is streaming. +Why this is instrumented the way it is: a streaming session makes the API +server's resident memory climb by tens of MB per minute and never give it back. +The interesting question each time is *where* it lands, so this sampler records + + * the service cgroup's own breakdown (anon / file / kernel / shmem), + * the process RSS split into anonymous ([heap], [anon...], unnamed), named + file-backed mappings, and everything else, + * the largest named mappings, so an unexpected consumer names itself. + +Anonymous memory that reaches swap is the failure mode we care about, hence the +explicit cg_swap_* and system swap columns. """ from __future__ import annotations @@ -47,9 +53,19 @@ FIELDS = [ "sys_mem_avail_mb", "sys_swap_used_mb", "sys_swap_free_mb", + "cg_anon_mb", + "cg_file_mb", + "cg_kernel_mb", + "cg_shmem_mb", + "heap_rss_mb", + "anon_rss_mb", + "file_rss_mb", + "other_rss_mb", + "top_named", ] MIB = 1048576 +ANON_NAME = re.compile(r"^\[anon") def read_text(path: Path, default: str = "") -> str: @@ -60,13 +76,23 @@ def read_text(path: Path, default: str = "") -> str: def cg_value(name: str) -> int: - raw = read_text(CG / name, "0") try: - return int(raw) + return int(read_text(CG / name, "0")) except ValueError: return 0 +def cg_stat(name: str) -> int: + for line in read_text(CG / "memory.stat").splitlines(): + key, _, value = line.partition(" ") + if key == name: + try: + return int(value) + except ValueError: + return 0 + return 0 + + def mb(value: int) -> float: return round(value / MIB, 1) @@ -98,10 +124,14 @@ def proc_uptime(pid: int) -> int: return int(uptime - start_ticks / os.sysconf("SC_CLK_TCK")) -def proc_memory(pid: int) -> tuple[float, int, int, float]: - """Return (rss_mb, anon_regions, big_anon_regions, anon_total_mb).""" - rss = anon_regions = big_anon_regions = 0 +def proc_memory(pid: int) -> dict: + """Split the process RSS by mapping kind, and name the biggest file mappings.""" + rss = anon_rss = file_rss = heap_rss = 0 + anon_regions = big_anon_regions = 0 anon_total = 0 + named: dict[str, float] = {} + name = "" + is_anon = is_file = False try: with open(f"/proc/{pid}/smaps") as handle: for line in handle: @@ -109,17 +139,43 @@ def proc_memory(pid: int) -> tuple[float, int, int, float]: parts = line.split() start, end = parts[0].split("-") size = (int(end, 16) - int(start, 16)) // 1024 - path = parts[5] if len(parts) > 5 else "[anon]" - if path == "[anon]": + name = parts[5] if len(parts) > 5 else "" + is_anon = name == "" or name == "[heap]" or bool(ANON_NAME.match(name)) + is_file = name.startswith("/") + if is_anon: anon_regions += 1 anon_total += size if size >= 32 * 1024: big_anon_regions += 1 elif line.startswith("Rss:"): - rss += int(line.split()[1]) + value = int(line.split()[1]) + rss += value + if is_anon: + anon_rss += value + if name == "[heap]": + heap_rss += value + elif is_file and name: + file_rss += value + named[name] = named.get(name, 0.0) + value / 1024 except OSError: pass - return round(rss / 1024, 1), anon_regions, big_anon_regions, round(anon_total / 1024, 1) + + top = ", ".join( + f"{Path(n).name}={v:.0f}MB" + for n, v in sorted(named.items(), key=lambda kv: -kv[1])[:3] + if v > 0 + ) + return { + "rss_mb": round(rss / 1024, 1), + "anon_regions": anon_regions, + "big_anon_regions": big_anon_regions, + "anon_total_mb": round(anon_total / 1024, 1), + "heap_rss_mb": round(heap_rss / 1024, 1), + "anon_rss_mb": round(anon_rss / 1024, 1), + "file_rss_mb": round(file_rss / 1024, 1), + "other_rss_mb": round((rss - anon_rss - file_rss) / 1024, 1), + "top_named": top.replace(",", ";") or "-", + } def system_memory() -> tuple[float, float, float]: @@ -165,11 +221,15 @@ def main() -> int: pid = main_pid() state = cloud_state() if pid: - rss, anon_regions, big_regions, anon_total = proc_memory(pid) + mem = proc_memory(pid) uptime = proc_uptime(pid) else: - rss = anon_total = 0.0 - anon_regions = big_regions = uptime = 0 + mem = { + "rss_mb": 0, "anon_regions": 0, "big_anon_regions": 0, "anon_total_mb": 0, + "heap_rss_mb": 0, "anon_rss_mb": 0, "file_rss_mb": 0, "other_rss_mb": 0, + "top_named": "-", + } + uptime = 0 mem_avail, swap_used, swap_free = system_memory() row = [ @@ -181,13 +241,22 @@ def main() -> int: mb(cg_value("memory.peak")), mb(cg_value("memory.swap.current")), mb(cg_value("memory.swap.peak")), - rss, - anon_regions, - big_regions, - anon_total, + mem["rss_mb"], + mem["anon_regions"], + mem["big_anon_regions"], + mem["anon_total_mb"], mem_avail, swap_used, swap_free, + mb(cg_stat("anon")), + mb(cg_stat("file")), + mb(cg_stat("kernel")), + mb(cg_stat("shmem")), + mem["heap_rss_mb"], + mem["anon_rss_mb"], + mem["file_rss_mb"], + mem["other_rss_mb"], + mem["top_named"], ] fresh = not LOG.exists() or LOG.stat().st_size == 0 @@ -197,10 +266,11 @@ def main() -> int: handle.write(",".join(str(value) for value in row) + "\n") print( - f"{row[0]} state={state} pid={pid} " - f"cg={row[4]}MB peak={row[5]}MB cg_swap={row[6]}MB " - f"rss={rss}MB anon_regions={anon_regions} big={big_regions} anon={anon_total}MB " - f"mem_avail={mem_avail}MB swap_used={swap_used}MB" + f"{row[0]} state={state} pid={pid} cg={row[4]}MB peak={row[5]}MB cg_swap={row[6]}MB " + f"rss={mem['rss_mb']}MB (heap={mem['heap_rss_mb']} anon={mem['anon_rss_mb']} " + f"file={mem['file_rss_mb']} other={mem['other_rss_mb']}) " + f"cg_anon={row[15]}MB cg_file={row[16]}MB regions={mem['anon_regions']} " + f"big={mem['big_anon_regions']} top=[{mem['top_named']}]" ) return 0