fix(memlog): 按映射类型拆分 RSS,并记录 cgroup 内存构成

原分类把 [heap] 当成非匿名区域,导致会话中主 arena(brk 堆)的增长被漏报:
实测 9/21 会话 RSS 涨到 2474 MB 而"匿名区域"仅 193 MB,误判为已无增长。

- 把 [heap]、[anon...]、无名映射统一计入匿名 RSS,并单独记录 heap_rss_mb
- 新增 cg_anon_mb / cg_file_mb / cg_kernel_mb / cg_shmem_mb(memory.stat)
- 新增 top_named:按 RSS 排序的前三个文件映射,便于识别异常消费者
This commit is contained in:
RRRosmontis
2026-09-21 08:12:49 +08:00
parent 636ecbd974
commit c7f3f4496a
+95 -25
View File
@@ -5,11 +5,17 @@ Run once per minute by maa-online-memlog.timer. Each run appends one row to
<ROOT>/logs/memlog.csv (auto-rotated at 5 MB, one generation kept) and prints a <ROOT>/logs/memlog.csv (auto-rotated at 5 MB, one generation kept) and prints a
compact line to the journal. compact line to the journal.
The numbers tracked here are the ones that matter for the glibc malloc arena Why this is instrumented the way it is: a streaming session makes the API
behaviour: the service cgroup's anonymous memory, how much of it has been server's resident memory climb by tens of MB per minute and never give it back.
pushed into swap, and how many large (>= 32 MB) anonymous mappings the process The interesting question each time is *where* it lands, so this sampler records
holds. An arena-bloat regression shows up as the big-region count climbing
while the service is streaming. * the service cgroup's own breakdown (anon / file / kernel / shmem),
* the process RSS split into anonymous ([heap], [anon...], unnamed), named
file-backed mappings, and everything else,
* the largest named mappings, so an unexpected consumer names itself.
Anonymous memory that reaches swap is the failure mode we care about, hence the
explicit cg_swap_* and system swap columns.
""" """
from __future__ import annotations from __future__ import annotations
@@ -47,9 +53,19 @@ FIELDS = [
"sys_mem_avail_mb", "sys_mem_avail_mb",
"sys_swap_used_mb", "sys_swap_used_mb",
"sys_swap_free_mb", "sys_swap_free_mb",
"cg_anon_mb",
"cg_file_mb",
"cg_kernel_mb",
"cg_shmem_mb",
"heap_rss_mb",
"anon_rss_mb",
"file_rss_mb",
"other_rss_mb",
"top_named",
] ]
MIB = 1048576 MIB = 1048576
ANON_NAME = re.compile(r"^\[anon")
def read_text(path: Path, default: str = "") -> str: def read_text(path: Path, default: str = "") -> str:
@@ -60,13 +76,23 @@ def read_text(path: Path, default: str = "") -> str:
def cg_value(name: str) -> int: def cg_value(name: str) -> int:
raw = read_text(CG / name, "0")
try: try:
return int(raw) return int(read_text(CG / name, "0"))
except ValueError: except ValueError:
return 0 return 0
def cg_stat(name: str) -> int:
for line in read_text(CG / "memory.stat").splitlines():
key, _, value = line.partition(" ")
if key == name:
try:
return int(value)
except ValueError:
return 0
return 0
def mb(value: int) -> float: def mb(value: int) -> float:
return round(value / MIB, 1) return round(value / MIB, 1)
@@ -98,10 +124,14 @@ def proc_uptime(pid: int) -> int:
return int(uptime - start_ticks / os.sysconf("SC_CLK_TCK")) return int(uptime - start_ticks / os.sysconf("SC_CLK_TCK"))
def proc_memory(pid: int) -> tuple[float, int, int, float]: def proc_memory(pid: int) -> dict:
"""Return (rss_mb, anon_regions, big_anon_regions, anon_total_mb).""" """Split the process RSS by mapping kind, and name the biggest file mappings."""
rss = anon_regions = big_anon_regions = 0 rss = anon_rss = file_rss = heap_rss = 0
anon_regions = big_anon_regions = 0
anon_total = 0 anon_total = 0
named: dict[str, float] = {}
name = ""
is_anon = is_file = False
try: try:
with open(f"/proc/{pid}/smaps") as handle: with open(f"/proc/{pid}/smaps") as handle:
for line in handle: for line in handle:
@@ -109,17 +139,43 @@ def proc_memory(pid: int) -> tuple[float, int, int, float]:
parts = line.split() parts = line.split()
start, end = parts[0].split("-") start, end = parts[0].split("-")
size = (int(end, 16) - int(start, 16)) // 1024 size = (int(end, 16) - int(start, 16)) // 1024
path = parts[5] if len(parts) > 5 else "[anon]" name = parts[5] if len(parts) > 5 else ""
if path == "[anon]": is_anon = name == "" or name == "[heap]" or bool(ANON_NAME.match(name))
is_file = name.startswith("/")
if is_anon:
anon_regions += 1 anon_regions += 1
anon_total += size anon_total += size
if size >= 32 * 1024: if size >= 32 * 1024:
big_anon_regions += 1 big_anon_regions += 1
elif line.startswith("Rss:"): elif line.startswith("Rss:"):
rss += int(line.split()[1]) value = int(line.split()[1])
rss += value
if is_anon:
anon_rss += value
if name == "[heap]":
heap_rss += value
elif is_file and name:
file_rss += value
named[name] = named.get(name, 0.0) + value / 1024
except OSError: except OSError:
pass pass
return round(rss / 1024, 1), anon_regions, big_anon_regions, round(anon_total / 1024, 1)
top = ", ".join(
f"{Path(n).name}={v:.0f}MB"
for n, v in sorted(named.items(), key=lambda kv: -kv[1])[:3]
if v > 0
)
return {
"rss_mb": round(rss / 1024, 1),
"anon_regions": anon_regions,
"big_anon_regions": big_anon_regions,
"anon_total_mb": round(anon_total / 1024, 1),
"heap_rss_mb": round(heap_rss / 1024, 1),
"anon_rss_mb": round(anon_rss / 1024, 1),
"file_rss_mb": round(file_rss / 1024, 1),
"other_rss_mb": round((rss - anon_rss - file_rss) / 1024, 1),
"top_named": top.replace(",", ";") or "-",
}
def system_memory() -> tuple[float, float, float]: def system_memory() -> tuple[float, float, float]:
@@ -165,11 +221,15 @@ def main() -> int:
pid = main_pid() pid = main_pid()
state = cloud_state() state = cloud_state()
if pid: if pid:
rss, anon_regions, big_regions, anon_total = proc_memory(pid) mem = proc_memory(pid)
uptime = proc_uptime(pid) uptime = proc_uptime(pid)
else: else:
rss = anon_total = 0.0 mem = {
anon_regions = big_regions = uptime = 0 "rss_mb": 0, "anon_regions": 0, "big_anon_regions": 0, "anon_total_mb": 0,
"heap_rss_mb": 0, "anon_rss_mb": 0, "file_rss_mb": 0, "other_rss_mb": 0,
"top_named": "-",
}
uptime = 0
mem_avail, swap_used, swap_free = system_memory() mem_avail, swap_used, swap_free = system_memory()
row = [ row = [
@@ -181,13 +241,22 @@ def main() -> int:
mb(cg_value("memory.peak")), mb(cg_value("memory.peak")),
mb(cg_value("memory.swap.current")), mb(cg_value("memory.swap.current")),
mb(cg_value("memory.swap.peak")), mb(cg_value("memory.swap.peak")),
rss, mem["rss_mb"],
anon_regions, mem["anon_regions"],
big_regions, mem["big_anon_regions"],
anon_total, mem["anon_total_mb"],
mem_avail, mem_avail,
swap_used, swap_used,
swap_free, swap_free,
mb(cg_stat("anon")),
mb(cg_stat("file")),
mb(cg_stat("kernel")),
mb(cg_stat("shmem")),
mem["heap_rss_mb"],
mem["anon_rss_mb"],
mem["file_rss_mb"],
mem["other_rss_mb"],
mem["top_named"],
] ]
fresh = not LOG.exists() or LOG.stat().st_size == 0 fresh = not LOG.exists() or LOG.stat().st_size == 0
@@ -197,10 +266,11 @@ def main() -> int:
handle.write(",".join(str(value) for value in row) + "\n") handle.write(",".join(str(value) for value in row) + "\n")
print( print(
f"{row[0]} state={state} pid={pid} " f"{row[0]} state={state} pid={pid} cg={row[4]}MB peak={row[5]}MB cg_swap={row[6]}MB "
f"cg={row[4]}MB peak={row[5]}MB cg_swap={row[6]}MB " f"rss={mem['rss_mb']}MB (heap={mem['heap_rss_mb']} anon={mem['anon_rss_mb']} "
f"rss={rss}MB anon_regions={anon_regions} big={big_regions} anon={anon_total}MB " f"file={mem['file_rss_mb']} other={mem['other_rss_mb']}) "
f"mem_avail={mem_avail}MB swap_used={swap_used}MB" f"cg_anon={row[15]}MB cg_file={row[16]}MB regions={mem['anon_regions']} "
f"big={mem['big_anon_regions']} top=[{mem['top_named']}]"
) )
return 0 return 0