feat(bilibili): 新增 BV 本地缓存,修复视频暂存目录泄漏

- 新增 CACHE_DIR(../cache/bilibili),以 BV 号为键缓存 {BV}.mp4,命中即 FileResponse、不再调用 yt-dlp
- 未命中时下载到缓存目录内的隐藏暂存目录 .dl-XXXX,成功后 os.replace 原子落盘
- finally 中 shutil.rmtree(tmpdir):原实现只清理 cookie 临时文件,导致每次取视频都把完整视频
  永久留在 /tmp(无上限磁盘泄漏)
- 淘汰策略:TTL 7 天未访问过期 + 总量超过 2 GiB 按 mtime 从最旧开始删除
- 新增运维接口 GET /bilibili/cache_info(缓存目录/上限/文件数/总量/剩余磁盘)
- 缓存 HIT/MISS/STORE 日志加 flush=True,实时可见
This commit is contained in:
RRRosmontis
2026-09-13 02:26:34 +08:00
parent 99543c9935
commit 5400dc0a86
+123 -24
View File
@@ -3,7 +3,9 @@ import sys
import base64
import os
import re
import shutil
import tempfile
import time
from pathlib import Path
import httpx
@@ -12,6 +14,19 @@ from aiohttp import web
COOKIE_FILE = str(Path(__file__).parent / "cookies.txt")
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
# ---------------- 本地缓存(与 pixiv 同级:source/cache/bilibili) ----------------
CACHE_DIR = (Path(__file__).resolve().parent / "../cache/bilibili").resolve()
CACHE_DIR.mkdir(parents=True, exist_ok=True)
CACHE_MAX_BYTES = 2 * 1024 ** 3 # 缓存总量上限 2 GiB
CACHE_TTL_SECONDS = 7 * 24 * 3600 # 7 天未访问即淘汰
VIDEO_EXTS = (".mp4", ".flv", ".mkv", ".webm")
CONTENT_TYPES = {
".mp4": "video/mp4",
".flv": "video/x-flv",
".mkv": "video/x-matroska",
".webm": "video/webm",
}
def _prepare_cookie_file():
"""Convert simple name=value cookies.txt to Netscape format for yt-dlp."""
@@ -36,6 +51,65 @@ def _prepare_cookie_file():
return tmp.name
def _cache_file(bvid):
"""缓存命中则返回文件路径(同时刷新 mtime 作为 LRU 依据),否则 None。"""
for f in CACHE_DIR.glob(f"{bvid}.*"):
if f.is_file() and f.suffix.lower() in VIDEO_EXTS:
try:
os.utime(f, None)
except OSError:
pass
return f
return None
def _cache_files():
out = []
for f in CACHE_DIR.iterdir():
if f.is_file() and f.suffix.lower() in VIDEO_EXTS:
out.append(f)
return out
def _cache_sweep():
"""先按 TTL 淘汰过期文件,再按总量上限从最旧(mtime 最小)开始淘汰。"""
now = time.time()
for f in _cache_files():
try:
if now - f.stat().st_mtime > CACHE_TTL_SECONDS:
f.unlink()
except OSError:
pass
files = _cache_files()
total = 0
for f in files:
try:
total += f.stat().st_size
except OSError:
pass
if total > CACHE_MAX_BYTES:
for f in sorted(files, key=lambda p: p.stat().st_mtime):
if total <= CACHE_MAX_BYTES:
break
try:
size = f.stat().st_size
f.unlink()
total -= size
except OSError:
pass
def _file_response(path, bvid):
return web.FileResponse(
path=path,
headers={
"Content-Type": CONTENT_TYPES.get(path.suffix.lower(), "application/octet-stream"),
"Content-Disposition": f'attachment; filename="{path.name}"',
},
)
async def resolve_bvid(link):
# 直接是 BV 号
m = re.search(r"(BV[a-zA-Z0-9]{10})", link)
@@ -90,11 +164,19 @@ async def get_video(url_b64=None, bv=None):
except ValueError as e:
return web.json_response({"error": str(e)}, status=400)
# 1) 缓存命中:直接回文件,不再调用 yt-dlp
cached = _cache_file(bvid)
if cached:
print(f"[bilibili] cache HIT {bvid} -> {cached.name} ({cached.stat().st_size} B)", flush=True)
return _file_response(cached, bvid)
cookie_tmp = _prepare_cookie_file()
if not cookie_tmp:
return web.json_response({"error": "missing bilibili cookie"}, status=500)
tmpdir = tempfile.mkdtemp(prefix="bilibili_")
print(f"[bilibili] cache MISS {bvid}, downloading...", flush=True)
# 2) 下载到缓存目录内的隐藏临时目录(同盘,才能原子 rename 落盘)
tmpdir = tempfile.mkdtemp(prefix=".dl-", dir=CACHE_DIR)
output_template = os.path.join(tmpdir, "video.%(ext)s")
cmd = [
os.path.join(os.path.dirname(sys.executable), "yt-dlp"),
@@ -113,16 +195,11 @@ async def get_video(url_b64=None, bv=None):
stderr=asyncio.subprocess.PIPE,
)
try:
stdout, stderr = await asyncio.wait_for(
proc.communicate(), timeout=300
)
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=300)
except asyncio.TimeoutError:
proc.kill()
await proc.wait()
return web.json_response({"error": "download timeout"}, status=504)
finally:
if cookie_tmp and os.path.exists(cookie_tmp):
os.unlink(cookie_tmp)
video_files = [p for p in Path(tmpdir).glob("*") if p.is_file()]
if proc.returncode != 0 and not video_files:
@@ -136,26 +213,23 @@ async def get_video(url_b64=None, bv=None):
return web.json_response({"error": "no output file"}, status=502)
video_path = max(video_files, key=lambda p: p.stat().st_size)
ext = video_path.suffix.lower()
content_types = {
".mp4": "video/mp4",
".flv": "video/x-flv",
".mkv": "video/x-matroska",
".webm": "video/webm",
}
content_type = content_types.get(ext, "application/octet-stream")
return web.FileResponse(
path=video_path,
headers={
"Content-Type": content_type,
"Content-Disposition": f'attachment; filename="{video_path.name}"',
},
)
final = CACHE_DIR / f"{bvid}{ext}"
try:
os.replace(video_path, final) # 同盘原子落盘
except OSError:
shutil.move(str(video_path), str(final))
_cache_sweep()
print(f"[bilibili] cache STORE {bvid} -> {final.name} ({final.stat().st_size} B)", flush=True)
return _file_response(final, bvid)
except Exception as e:
print(f"[bilibili] Error: {e}")
return web.json_response({"error": str(e)}, status=500)
finally:
# 关键修复:无论成功/失败/超时都回收临时目录(原实现会把整个视频永久留在 /tmp)
shutil.rmtree(tmpdir, ignore_errors=True)
if cookie_tmp and os.path.exists(cookie_tmp):
os.unlink(cookie_tmp)
async def handle_bilibili(request):
@@ -168,8 +242,33 @@ async def handle_bilibili(request):
return await get_video(url_b64=url_b64)
async def handle_cache_info(request):
files = []
total = 0
for f in _cache_files():
try:
st = f.stat()
except OSError:
continue
files.append({"name": f.name, "size": st.st_size, "mtime": int(st.st_mtime)})
total += st.st_size
return web.json_response(
{
"cache_dir": str(CACHE_DIR),
"cache_max_bytes": CACHE_MAX_BYTES,
"cache_ttl_seconds": CACHE_TTL_SECONDS,
"count": len(files),
"total_bytes": total,
"free_bytes": shutil.disk_usage(CACHE_DIR).free,
"files": sorted(files, key=lambda x: -x["size"]),
}
)
app = web.Application()
app.router.add_get("/bilibili/get_video", handle_bilibili)
app.router.add_get("/bilibili/cache_info", handle_cache_info)
if __name__ == "__main__":
web.run_app(app, host="127.0.0.1", port=19990)
print(f"Cache directory: {CACHE_DIR}")
web.run_app(app, host="127.0.0.1", port=9006)