fix(memlog): 按映射类型拆分 RSS,并记录 cgroup 内存构成
原分类把 [heap] 当成非匿名区域,导致会话中主 arena(brk 堆)的增长被漏报: 实测 9/21 会话 RSS 涨到 2474 MB 而"匿名区域"仅 193 MB,误判为已无增长。 - 把 [heap]、[anon...]、无名映射统一计入匿名 RSS,并单独记录 heap_rss_mb - 新增 cg_anon_mb / cg_file_mb / cg_kernel_mb / cg_shmem_mb(memory.stat) - 新增 top_named:按 RSS 排序的前三个文件映射,便于识别异常消费者
This commit is contained in:
+95
-25
@@ -5,11 +5,17 @@ Run once per minute by maa-online-memlog.timer. Each run appends one row to
|
|||||||
<ROOT>/logs/memlog.csv (auto-rotated at 5 MB, one generation kept) and prints a
|
<ROOT>/logs/memlog.csv (auto-rotated at 5 MB, one generation kept) and prints a
|
||||||
compact line to the journal.
|
compact line to the journal.
|
||||||
|
|
||||||
The numbers tracked here are the ones that matter for the glibc malloc arena
|
Why this is instrumented the way it is: a streaming session makes the API
|
||||||
behaviour: the service cgroup's anonymous memory, how much of it has been
|
server's resident memory climb by tens of MB per minute and never give it back.
|
||||||
pushed into swap, and how many large (>= 32 MB) anonymous mappings the process
|
The interesting question each time is *where* it lands, so this sampler records
|
||||||
holds. An arena-bloat regression shows up as the big-region count climbing
|
|
||||||
while the service is streaming.
|
* the service cgroup's own breakdown (anon / file / kernel / shmem),
|
||||||
|
* the process RSS split into anonymous ([heap], [anon...], unnamed), named
|
||||||
|
file-backed mappings, and everything else,
|
||||||
|
* the largest named mappings, so an unexpected consumer names itself.
|
||||||
|
|
||||||
|
Anonymous memory that reaches swap is the failure mode we care about, hence the
|
||||||
|
explicit cg_swap_* and system swap columns.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
@@ -47,9 +53,19 @@ FIELDS = [
|
|||||||
"sys_mem_avail_mb",
|
"sys_mem_avail_mb",
|
||||||
"sys_swap_used_mb",
|
"sys_swap_used_mb",
|
||||||
"sys_swap_free_mb",
|
"sys_swap_free_mb",
|
||||||
|
"cg_anon_mb",
|
||||||
|
"cg_file_mb",
|
||||||
|
"cg_kernel_mb",
|
||||||
|
"cg_shmem_mb",
|
||||||
|
"heap_rss_mb",
|
||||||
|
"anon_rss_mb",
|
||||||
|
"file_rss_mb",
|
||||||
|
"other_rss_mb",
|
||||||
|
"top_named",
|
||||||
]
|
]
|
||||||
|
|
||||||
MIB = 1048576
|
MIB = 1048576
|
||||||
|
ANON_NAME = re.compile(r"^\[anon")
|
||||||
|
|
||||||
|
|
||||||
def read_text(path: Path, default: str = "") -> str:
|
def read_text(path: Path, default: str = "") -> str:
|
||||||
@@ -60,13 +76,23 @@ def read_text(path: Path, default: str = "") -> str:
|
|||||||
|
|
||||||
|
|
||||||
def cg_value(name: str) -> int:
|
def cg_value(name: str) -> int:
|
||||||
raw = read_text(CG / name, "0")
|
|
||||||
try:
|
try:
|
||||||
return int(raw)
|
return int(read_text(CG / name, "0"))
|
||||||
except ValueError:
|
except ValueError:
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|
||||||
|
def cg_stat(name: str) -> int:
|
||||||
|
for line in read_text(CG / "memory.stat").splitlines():
|
||||||
|
key, _, value = line.partition(" ")
|
||||||
|
if key == name:
|
||||||
|
try:
|
||||||
|
return int(value)
|
||||||
|
except ValueError:
|
||||||
|
return 0
|
||||||
|
return 0
|
||||||
|
|
||||||
|
|
||||||
def mb(value: int) -> float:
|
def mb(value: int) -> float:
|
||||||
return round(value / MIB, 1)
|
return round(value / MIB, 1)
|
||||||
|
|
||||||
@@ -98,10 +124,14 @@ def proc_uptime(pid: int) -> int:
|
|||||||
return int(uptime - start_ticks / os.sysconf("SC_CLK_TCK"))
|
return int(uptime - start_ticks / os.sysconf("SC_CLK_TCK"))
|
||||||
|
|
||||||
|
|
||||||
def proc_memory(pid: int) -> tuple[float, int, int, float]:
|
def proc_memory(pid: int) -> dict:
|
||||||
"""Return (rss_mb, anon_regions, big_anon_regions, anon_total_mb)."""
|
"""Split the process RSS by mapping kind, and name the biggest file mappings."""
|
||||||
rss = anon_regions = big_anon_regions = 0
|
rss = anon_rss = file_rss = heap_rss = 0
|
||||||
|
anon_regions = big_anon_regions = 0
|
||||||
anon_total = 0
|
anon_total = 0
|
||||||
|
named: dict[str, float] = {}
|
||||||
|
name = ""
|
||||||
|
is_anon = is_file = False
|
||||||
try:
|
try:
|
||||||
with open(f"/proc/{pid}/smaps") as handle:
|
with open(f"/proc/{pid}/smaps") as handle:
|
||||||
for line in handle:
|
for line in handle:
|
||||||
@@ -109,17 +139,43 @@ def proc_memory(pid: int) -> tuple[float, int, int, float]:
|
|||||||
parts = line.split()
|
parts = line.split()
|
||||||
start, end = parts[0].split("-")
|
start, end = parts[0].split("-")
|
||||||
size = (int(end, 16) - int(start, 16)) // 1024
|
size = (int(end, 16) - int(start, 16)) // 1024
|
||||||
path = parts[5] if len(parts) > 5 else "[anon]"
|
name = parts[5] if len(parts) > 5 else ""
|
||||||
if path == "[anon]":
|
is_anon = name == "" or name == "[heap]" or bool(ANON_NAME.match(name))
|
||||||
|
is_file = name.startswith("/")
|
||||||
|
if is_anon:
|
||||||
anon_regions += 1
|
anon_regions += 1
|
||||||
anon_total += size
|
anon_total += size
|
||||||
if size >= 32 * 1024:
|
if size >= 32 * 1024:
|
||||||
big_anon_regions += 1
|
big_anon_regions += 1
|
||||||
elif line.startswith("Rss:"):
|
elif line.startswith("Rss:"):
|
||||||
rss += int(line.split()[1])
|
value = int(line.split()[1])
|
||||||
|
rss += value
|
||||||
|
if is_anon:
|
||||||
|
anon_rss += value
|
||||||
|
if name == "[heap]":
|
||||||
|
heap_rss += value
|
||||||
|
elif is_file and name:
|
||||||
|
file_rss += value
|
||||||
|
named[name] = named.get(name, 0.0) + value / 1024
|
||||||
except OSError:
|
except OSError:
|
||||||
pass
|
pass
|
||||||
return round(rss / 1024, 1), anon_regions, big_anon_regions, round(anon_total / 1024, 1)
|
|
||||||
|
top = ", ".join(
|
||||||
|
f"{Path(n).name}={v:.0f}MB"
|
||||||
|
for n, v in sorted(named.items(), key=lambda kv: -kv[1])[:3]
|
||||||
|
if v > 0
|
||||||
|
)
|
||||||
|
return {
|
||||||
|
"rss_mb": round(rss / 1024, 1),
|
||||||
|
"anon_regions": anon_regions,
|
||||||
|
"big_anon_regions": big_anon_regions,
|
||||||
|
"anon_total_mb": round(anon_total / 1024, 1),
|
||||||
|
"heap_rss_mb": round(heap_rss / 1024, 1),
|
||||||
|
"anon_rss_mb": round(anon_rss / 1024, 1),
|
||||||
|
"file_rss_mb": round(file_rss / 1024, 1),
|
||||||
|
"other_rss_mb": round((rss - anon_rss - file_rss) / 1024, 1),
|
||||||
|
"top_named": top.replace(",", ";") or "-",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def system_memory() -> tuple[float, float, float]:
|
def system_memory() -> tuple[float, float, float]:
|
||||||
@@ -165,11 +221,15 @@ def main() -> int:
|
|||||||
pid = main_pid()
|
pid = main_pid()
|
||||||
state = cloud_state()
|
state = cloud_state()
|
||||||
if pid:
|
if pid:
|
||||||
rss, anon_regions, big_regions, anon_total = proc_memory(pid)
|
mem = proc_memory(pid)
|
||||||
uptime = proc_uptime(pid)
|
uptime = proc_uptime(pid)
|
||||||
else:
|
else:
|
||||||
rss = anon_total = 0.0
|
mem = {
|
||||||
anon_regions = big_regions = uptime = 0
|
"rss_mb": 0, "anon_regions": 0, "big_anon_regions": 0, "anon_total_mb": 0,
|
||||||
|
"heap_rss_mb": 0, "anon_rss_mb": 0, "file_rss_mb": 0, "other_rss_mb": 0,
|
||||||
|
"top_named": "-",
|
||||||
|
}
|
||||||
|
uptime = 0
|
||||||
|
|
||||||
mem_avail, swap_used, swap_free = system_memory()
|
mem_avail, swap_used, swap_free = system_memory()
|
||||||
row = [
|
row = [
|
||||||
@@ -181,13 +241,22 @@ def main() -> int:
|
|||||||
mb(cg_value("memory.peak")),
|
mb(cg_value("memory.peak")),
|
||||||
mb(cg_value("memory.swap.current")),
|
mb(cg_value("memory.swap.current")),
|
||||||
mb(cg_value("memory.swap.peak")),
|
mb(cg_value("memory.swap.peak")),
|
||||||
rss,
|
mem["rss_mb"],
|
||||||
anon_regions,
|
mem["anon_regions"],
|
||||||
big_regions,
|
mem["big_anon_regions"],
|
||||||
anon_total,
|
mem["anon_total_mb"],
|
||||||
mem_avail,
|
mem_avail,
|
||||||
swap_used,
|
swap_used,
|
||||||
swap_free,
|
swap_free,
|
||||||
|
mb(cg_stat("anon")),
|
||||||
|
mb(cg_stat("file")),
|
||||||
|
mb(cg_stat("kernel")),
|
||||||
|
mb(cg_stat("shmem")),
|
||||||
|
mem["heap_rss_mb"],
|
||||||
|
mem["anon_rss_mb"],
|
||||||
|
mem["file_rss_mb"],
|
||||||
|
mem["other_rss_mb"],
|
||||||
|
mem["top_named"],
|
||||||
]
|
]
|
||||||
|
|
||||||
fresh = not LOG.exists() or LOG.stat().st_size == 0
|
fresh = not LOG.exists() or LOG.stat().st_size == 0
|
||||||
@@ -197,10 +266,11 @@ def main() -> int:
|
|||||||
handle.write(",".join(str(value) for value in row) + "\n")
|
handle.write(",".join(str(value) for value in row) + "\n")
|
||||||
|
|
||||||
print(
|
print(
|
||||||
f"{row[0]} state={state} pid={pid} "
|
f"{row[0]} state={state} pid={pid} cg={row[4]}MB peak={row[5]}MB cg_swap={row[6]}MB "
|
||||||
f"cg={row[4]}MB peak={row[5]}MB cg_swap={row[6]}MB "
|
f"rss={mem['rss_mb']}MB (heap={mem['heap_rss_mb']} anon={mem['anon_rss_mb']} "
|
||||||
f"rss={rss}MB anon_regions={anon_regions} big={big_regions} anon={anon_total}MB "
|
f"file={mem['file_rss_mb']} other={mem['other_rss_mb']}) "
|
||||||
f"mem_avail={mem_avail}MB swap_used={swap_used}MB"
|
f"cg_anon={row[15]}MB cg_file={row[16]}MB regions={mem['anon_regions']} "
|
||||||
|
f"big={mem['big_anon_regions']} top=[{mem['top_named']}]"
|
||||||
)
|
)
|
||||||
return 0
|
return 0
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user