fix(memlog): 按映射类型拆分 RSS，并记录 cgroup 内存构成
原分类把 [heap] 当成非匿名区域，导致会话中主 arena（brk 堆）的增长被漏报： 实测 9/21 会话 RSS 涨到 2474 MB 而"匿名区域"仅 193 MB，误判为已无增长。 - 把 [heap]、[anon...]、无名映射统一计入匿名 RSS，并单独记录 heap_rss_mb - 新增 cg_anon_mb / cg_file_mb / cg_kernel_mb / cg_shmem_mb（memory.stat） - 新增 top_named：按 RSS 排序的前三个文件映射，便于识别异常消费者
This commit is contained in:
RRRosmontis
+95
-25
@@ -5,11 +5,17 @@ Run once per minute by maa-online-memlog.timer. Each run appends one row to
|
||||
<ROOT>/logs/memlog.csv (auto-rotated at 5 MB, one generation kept) and prints a
|
||||
compact line to the journal.
|
||||
|
||||
The numbers tracked here are the ones that matter for the glibc malloc arena
|
||||
behaviour: the service cgroup's anonymous memory, how much of it has been
|
||||
pushed into swap, and how many large (>= 32 MB) anonymous mappings the process
|
||||
holds. An arena-bloat regression shows up as the big-region count climbing
|
||||
while the service is streaming.
|
||||
Why this is instrumented the way it is: a streaming session makes the API
|
||||
server's resident memory climb by tens of MB per minute and never give it back.
|
||||
The interesting question each time is *where* it lands, so this sampler records
|
||||
|
||||
* the service cgroup's own breakdown (anon / file / kernel / shmem),
|
||||
* the process RSS split into anonymous ([heap], [anon...], unnamed), named
|
||||
file-backed mappings, and everything else,
|
||||
* the largest named mappings, so an unexpected consumer names itself.
|
||||
|
||||
Anonymous memory that reaches swap is the failure mode we care about, hence the
|
||||
explicit cg_swap_* and system swap columns.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -47,9 +53,19 @@ FIELDS = [
|
||||
"sys_mem_avail_mb",
|
||||
"sys_swap_used_mb",
|
||||
"sys_swap_free_mb",
|
||||
"cg_anon_mb",
|
||||
"cg_file_mb",
|
||||
"cg_kernel_mb",
|
||||
"cg_shmem_mb",
|
||||
"heap_rss_mb",
|
||||
"anon_rss_mb",
|
||||
"file_rss_mb",
|
||||
"other_rss_mb",
|
||||
"top_named",
|
||||
]
|
||||
|
||||
MIB = 1048576
|
||||
ANON_NAME = re.compile(r"^\[anon")
|
||||
|
||||
|
||||
def read_text(path: Path, default: str = "") -> str:
|
||||
@@ -60,13 +76,23 @@ def read_text(path: Path, default: str = "") -> str:
|
||||
|
||||
|
||||
def cg_value(name: str) -> int:
|
||||
raw = read_text(CG / name, "0")
|
||||
try:
|
||||
return int(raw)
|
||||
return int(read_text(CG / name, "0"))
|
||||
except ValueError:
|
||||
return 0
|
||||
|
||||
|
||||
def cg_stat(name: str) -> int:
|
||||
for line in read_text(CG / "memory.stat").splitlines():
|
||||
key, _, value = line.partition(" ")
|
||||
if key == name:
|
||||
try:
|
||||
return int(value)
|
||||
except ValueError:
|
||||
return 0
|
||||
return 0
|
||||
|
||||
|
||||
def mb(value: int) -> float:
|
||||
return round(value / MIB, 1)
|
||||
|
||||
@@ -98,10 +124,14 @@ def proc_uptime(pid: int) -> int:
|
||||
return int(uptime - start_ticks / os.sysconf("SC_CLK_TCK"))
|
||||
|
||||
|
||||
def proc_memory(pid: int) -> tuple[float, int, int, float]:
|
||||
"""Return (rss_mb, anon_regions, big_anon_regions, anon_total_mb)."""
|
||||
rss = anon_regions = big_anon_regions = 0
|
||||
def proc_memory(pid: int) -> dict:
|
||||
"""Split the process RSS by mapping kind, and name the biggest file mappings."""
|
||||
rss = anon_rss = file_rss = heap_rss = 0
|
||||
anon_regions = big_anon_regions = 0
|
||||
anon_total = 0
|
||||
named: dict[str, float] = {}
|
||||
name = ""
|
||||
is_anon = is_file = False
|
||||
try:
|
||||
with open(f"/proc/{pid}/smaps") as handle:
|
||||
for line in handle:
|
||||
@@ -109,17 +139,43 @@ def proc_memory(pid: int) -> tuple[float, int, int, float]:
|
||||
parts = line.split()
|
||||
start, end = parts[0].split("-")
|
||||
size = (int(end, 16) - int(start, 16)) // 1024
|
||||
path = parts[5] if len(parts) > 5 else "[anon]"
|
||||
if path == "[anon]":
|
||||
name = parts[5] if len(parts) > 5 else ""
|
||||
is_anon = name == "" or name == "[heap]" or bool(ANON_NAME.match(name))
|
||||
is_file = name.startswith("/")
|
||||
if is_anon:
|
||||
anon_regions += 1
|
||||
anon_total += size
|
||||
if size >= 32 * 1024:
|
||||
big_anon_regions += 1
|
||||
elif line.startswith("Rss:"):
|
||||
rss += int(line.split()[1])
|
||||
value = int(line.split()[1])
|
||||
rss += value
|
||||
if is_anon:
|
||||
anon_rss += value
|
||||
if name == "[heap]":
|
||||
heap_rss += value
|
||||
elif is_file and name:
|
||||
file_rss += value
|
||||
named[name] = named.get(name, 0.0) + value / 1024
|
||||
except OSError:
|
||||
pass
|
||||
return round(rss / 1024, 1), anon_regions, big_anon_regions, round(anon_total / 1024, 1)
|
||||
|
||||
top = ", ".join(
|
||||
f"{Path(n).name}={v:.0f}MB"
|
||||
for n, v in sorted(named.items(), key=lambda kv: -kv[1])[:3]
|
||||
if v > 0
|
||||
)
|
||||
return {
|
||||
"rss_mb": round(rss / 1024, 1),
|
||||
"anon_regions": anon_regions,
|
||||
"big_anon_regions": big_anon_regions,
|
||||
"anon_total_mb": round(anon_total / 1024, 1),
|
||||
"heap_rss_mb": round(heap_rss / 1024, 1),
|
||||
"anon_rss_mb": round(anon_rss / 1024, 1),
|
||||
"file_rss_mb": round(file_rss / 1024, 1),
|
||||
"other_rss_mb": round((rss - anon_rss - file_rss) / 1024, 1),
|
||||
"top_named": top.replace(",", ";") or "-",
|
||||
}
|
||||
|
||||
|
||||
def system_memory() -> tuple[float, float, float]:
|
||||
@@ -165,11 +221,15 @@ def main() -> int:
|
||||
pid = main_pid()
|
||||
state = cloud_state()
|
||||
if pid:
|
||||
rss, anon_regions, big_regions, anon_total = proc_memory(pid)
|
||||
mem = proc_memory(pid)
|
||||
uptime = proc_uptime(pid)
|
||||
else:
|
||||
rss = anon_total = 0.0
|
||||
anon_regions = big_regions = uptime = 0
|
||||
mem = {
|
||||
"rss_mb": 0, "anon_regions": 0, "big_anon_regions": 0, "anon_total_mb": 0,
|
||||
"heap_rss_mb": 0, "anon_rss_mb": 0, "file_rss_mb": 0, "other_rss_mb": 0,
|
||||
"top_named": "-",
|
||||
}
|
||||
uptime = 0
|
||||
|
||||
mem_avail, swap_used, swap_free = system_memory()
|
||||
row = [
|
||||
@@ -181,13 +241,22 @@ def main() -> int:
|
||||
mb(cg_value("memory.peak")),
|
||||
mb(cg_value("memory.swap.current")),
|
||||
mb(cg_value("memory.swap.peak")),
|
||||
rss,
|
||||
anon_regions,
|
||||
big_regions,
|
||||
anon_total,
|
||||
mem["rss_mb"],
|
||||
mem["anon_regions"],
|
||||
mem["big_anon_regions"],
|
||||
mem["anon_total_mb"],
|
||||
mem_avail,
|
||||
swap_used,
|
||||
swap_free,
|
||||
mb(cg_stat("anon")),
|
||||
mb(cg_stat("file")),
|
||||
mb(cg_stat("kernel")),
|
||||
mb(cg_stat("shmem")),
|
||||
mem["heap_rss_mb"],
|
||||
mem["anon_rss_mb"],
|
||||
mem["file_rss_mb"],
|
||||
mem["other_rss_mb"],
|
||||
mem["top_named"],
|
||||
]
|
||||
|
||||
fresh = not LOG.exists() or LOG.stat().st_size == 0
|
||||
@@ -197,10 +266,11 @@ def main() -> int:
|
||||
handle.write(",".join(str(value) for value in row) + "\n")
|
||||
|
||||
print(
|
||||
f"{row[0]} state={state} pid={pid} "
|
||||
f"cg={row[4]}MB peak={row[5]}MB cg_swap={row[6]}MB "
|
||||
f"rss={rss}MB anon_regions={anon_regions} big={big_regions} anon={anon_total}MB "
|
||||
f"mem_avail={mem_avail}MB swap_used={swap_used}MB"
|
||||
f"{row[0]} state={state} pid={pid} cg={row[4]}MB peak={row[5]}MB cg_swap={row[6]}MB "
|
||||
f"rss={mem['rss_mb']}MB (heap={mem['heap_rss_mb']} anon={mem['anon_rss_mb']} "
|
||||
f"file={mem['file_rss_mb']} other={mem['other_rss_mb']}) "
|
||||
f"cg_anon={row[15]}MB cg_file={row[16]}MB regions={mem['anon_regions']} "
|
||||
f"big={mem['big_anon_regions']} top=[{mem['top_named']}]"
|
||||
)
|
||||
return 0
|
||||
|
||||
|
||||
Reference in New Issue
Block a user