feat(bilibili): 新增 BV 本地缓存,修复视频暂存目录泄漏

- 新增 CACHE_DIR(../cache/bilibili),以 BV 号为键缓存 {BV}.mp4,命中即 FileResponse、不再调用 yt-dlp
- 未命中时下载到缓存目录内的隐藏暂存目录 .dl-XXXX,成功后 os.replace 原子落盘
- finally 中 shutil.rmtree(tmpdir):原实现只清理 cookie 临时文件,导致每次取视频都把完整视频
  永久留在 /tmp(无上限磁盘泄漏)
- 淘汰策略:TTL 7 天未访问过期 + 总量超过 2 GiB 按 mtime 从最旧开始删除
- 新增运维接口 GET /bilibili/cache_info(缓存目录/上限/文件数/总量/剩余磁盘)
- 缓存 HIT/MISS/STORE 日志加 flush=True,实时可见
This commit is contained in:
RRRosmontis
2026-09-13 02:26:34 +08:00
parent 99543c9935
commit 5400dc0a86
+123 -24
View File
@@ -3,7 +3,9 @@ import sys
import base64 import base64
import os import os
import re import re
import shutil
import tempfile import tempfile
import time
from pathlib import Path from pathlib import Path
import httpx import httpx
@@ -12,6 +14,19 @@ from aiohttp import web
COOKIE_FILE = str(Path(__file__).parent / "cookies.txt") COOKIE_FILE = str(Path(__file__).parent / "cookies.txt")
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36" UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
# ---------------- 本地缓存(与 pixiv 同级:source/cache/bilibili) ----------------
CACHE_DIR = (Path(__file__).resolve().parent / "../cache/bilibili").resolve()
CACHE_DIR.mkdir(parents=True, exist_ok=True)
CACHE_MAX_BYTES = 2 * 1024 ** 3 # 缓存总量上限 2 GiB
CACHE_TTL_SECONDS = 7 * 24 * 3600 # 7 天未访问即淘汰
VIDEO_EXTS = (".mp4", ".flv", ".mkv", ".webm")
CONTENT_TYPES = {
".mp4": "video/mp4",
".flv": "video/x-flv",
".mkv": "video/x-matroska",
".webm": "video/webm",
}
def _prepare_cookie_file(): def _prepare_cookie_file():
"""Convert simple name=value cookies.txt to Netscape format for yt-dlp.""" """Convert simple name=value cookies.txt to Netscape format for yt-dlp."""
@@ -36,6 +51,65 @@ def _prepare_cookie_file():
return tmp.name return tmp.name
def _cache_file(bvid):
"""缓存命中则返回文件路径(同时刷新 mtime 作为 LRU 依据),否则 None。"""
for f in CACHE_DIR.glob(f"{bvid}.*"):
if f.is_file() and f.suffix.lower() in VIDEO_EXTS:
try:
os.utime(f, None)
except OSError:
pass
return f
return None
def _cache_files():
out = []
for f in CACHE_DIR.iterdir():
if f.is_file() and f.suffix.lower() in VIDEO_EXTS:
out.append(f)
return out
def _cache_sweep():
"""先按 TTL 淘汰过期文件,再按总量上限从最旧(mtime 最小)开始淘汰。"""
now = time.time()
for f in _cache_files():
try:
if now - f.stat().st_mtime > CACHE_TTL_SECONDS:
f.unlink()
except OSError:
pass
files = _cache_files()
total = 0
for f in files:
try:
total += f.stat().st_size
except OSError:
pass
if total > CACHE_MAX_BYTES:
for f in sorted(files, key=lambda p: p.stat().st_mtime):
if total <= CACHE_MAX_BYTES:
break
try:
size = f.stat().st_size
f.unlink()
total -= size
except OSError:
pass
def _file_response(path, bvid):
return web.FileResponse(
path=path,
headers={
"Content-Type": CONTENT_TYPES.get(path.suffix.lower(), "application/octet-stream"),
"Content-Disposition": f'attachment; filename="{path.name}"',
},
)
async def resolve_bvid(link): async def resolve_bvid(link):
# 直接是 BV 号 # 直接是 BV 号
m = re.search(r"(BV[a-zA-Z0-9]{10})", link) m = re.search(r"(BV[a-zA-Z0-9]{10})", link)
@@ -90,11 +164,19 @@ async def get_video(url_b64=None, bv=None):
except ValueError as e: except ValueError as e:
return web.json_response({"error": str(e)}, status=400) return web.json_response({"error": str(e)}, status=400)
# 1) 缓存命中:直接回文件,不再调用 yt-dlp
cached = _cache_file(bvid)
if cached:
print(f"[bilibili] cache HIT {bvid} -> {cached.name} ({cached.stat().st_size} B)", flush=True)
return _file_response(cached, bvid)
cookie_tmp = _prepare_cookie_file() cookie_tmp = _prepare_cookie_file()
if not cookie_tmp: if not cookie_tmp:
return web.json_response({"error": "missing bilibili cookie"}, status=500) return web.json_response({"error": "missing bilibili cookie"}, status=500)
tmpdir = tempfile.mkdtemp(prefix="bilibili_") print(f"[bilibili] cache MISS {bvid}, downloading...", flush=True)
# 2) 下载到缓存目录内的隐藏临时目录(同盘,才能原子 rename 落盘)
tmpdir = tempfile.mkdtemp(prefix=".dl-", dir=CACHE_DIR)
output_template = os.path.join(tmpdir, "video.%(ext)s") output_template = os.path.join(tmpdir, "video.%(ext)s")
cmd = [ cmd = [
os.path.join(os.path.dirname(sys.executable), "yt-dlp"), os.path.join(os.path.dirname(sys.executable), "yt-dlp"),
@@ -113,16 +195,11 @@ async def get_video(url_b64=None, bv=None):
stderr=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
) )
try: try:
stdout, stderr = await asyncio.wait_for( stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=300)
proc.communicate(), timeout=300
)
except asyncio.TimeoutError: except asyncio.TimeoutError:
proc.kill() proc.kill()
await proc.wait() await proc.wait()
return web.json_response({"error": "download timeout"}, status=504) return web.json_response({"error": "download timeout"}, status=504)
finally:
if cookie_tmp and os.path.exists(cookie_tmp):
os.unlink(cookie_tmp)
video_files = [p for p in Path(tmpdir).glob("*") if p.is_file()] video_files = [p for p in Path(tmpdir).glob("*") if p.is_file()]
if proc.returncode != 0 and not video_files: if proc.returncode != 0 and not video_files:
@@ -136,26 +213,23 @@ async def get_video(url_b64=None, bv=None):
return web.json_response({"error": "no output file"}, status=502) return web.json_response({"error": "no output file"}, status=502)
video_path = max(video_files, key=lambda p: p.stat().st_size) video_path = max(video_files, key=lambda p: p.stat().st_size)
ext = video_path.suffix.lower() ext = video_path.suffix.lower()
content_types = { final = CACHE_DIR / f"{bvid}{ext}"
".mp4": "video/mp4", try:
".flv": "video/x-flv", os.replace(video_path, final) # 同盘原子落盘
".mkv": "video/x-matroska", except OSError:
".webm": "video/webm", shutil.move(str(video_path), str(final))
} _cache_sweep()
content_type = content_types.get(ext, "application/octet-stream") print(f"[bilibili] cache STORE {bvid} -> {final.name} ({final.stat().st_size} B)", flush=True)
return _file_response(final, bvid)
return web.FileResponse(
path=video_path,
headers={
"Content-Type": content_type,
"Content-Disposition": f'attachment; filename="{video_path.name}"',
},
)
except Exception as e: except Exception as e:
print(f"[bilibili] Error: {e}") print(f"[bilibili] Error: {e}")
return web.json_response({"error": str(e)}, status=500) return web.json_response({"error": str(e)}, status=500)
finally:
# 关键修复:无论成功/失败/超时都回收临时目录(原实现会把整个视频永久留在 /tmp)
shutil.rmtree(tmpdir, ignore_errors=True)
if cookie_tmp and os.path.exists(cookie_tmp):
os.unlink(cookie_tmp)
async def handle_bilibili(request): async def handle_bilibili(request):
@@ -168,8 +242,33 @@ async def handle_bilibili(request):
return await get_video(url_b64=url_b64) return await get_video(url_b64=url_b64)
async def handle_cache_info(request):
files = []
total = 0
for f in _cache_files():
try:
st = f.stat()
except OSError:
continue
files.append({"name": f.name, "size": st.st_size, "mtime": int(st.st_mtime)})
total += st.st_size
return web.json_response(
{
"cache_dir": str(CACHE_DIR),
"cache_max_bytes": CACHE_MAX_BYTES,
"cache_ttl_seconds": CACHE_TTL_SECONDS,
"count": len(files),
"total_bytes": total,
"free_bytes": shutil.disk_usage(CACHE_DIR).free,
"files": sorted(files, key=lambda x: -x["size"]),
}
)
app = web.Application() app = web.Application()
app.router.add_get("/bilibili/get_video", handle_bilibili) app.router.add_get("/bilibili/get_video", handle_bilibili)
app.router.add_get("/bilibili/cache_info", handle_cache_info)
if __name__ == "__main__": if __name__ == "__main__":
web.run_app(app, host="127.0.0.1", port=19990) print(f"Cache directory: {CACHE_DIR}")
web.run_app(app, host="127.0.0.1", port=9006)