feat(bilibili): 新增 BV 本地缓存,修复视频暂存目录泄漏
- 新增 CACHE_DIR(../cache/bilibili),以 BV 号为键缓存 {BV}.mp4,命中即 FileResponse、不再调用 yt-dlp
- 未命中时下载到缓存目录内的隐藏暂存目录 .dl-XXXX,成功后 os.replace 原子落盘
- finally 中 shutil.rmtree(tmpdir):原实现只清理 cookie 临时文件,导致每次取视频都把完整视频
永久留在 /tmp(无上限磁盘泄漏)
- 淘汰策略:TTL 7 天未访问过期 + 总量超过 2 GiB 按 mtime 从最旧开始删除
- 新增运维接口 GET /bilibili/cache_info(缓存目录/上限/文件数/总量/剩余磁盘)
- 缓存 HIT/MISS/STORE 日志加 flush=True,实时可见
This commit is contained in:
+123
-24
@@ -3,7 +3,9 @@ import sys
|
||||
import base64
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
@@ -12,6 +14,19 @@ from aiohttp import web
|
||||
COOKIE_FILE = str(Path(__file__).parent / "cookies.txt")
|
||||
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
|
||||
|
||||
# ---------------- 本地缓存(与 pixiv 同级:source/cache/bilibili) ----------------
|
||||
CACHE_DIR = (Path(__file__).resolve().parent / "../cache/bilibili").resolve()
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
CACHE_MAX_BYTES = 2 * 1024 ** 3 # 缓存总量上限 2 GiB
|
||||
CACHE_TTL_SECONDS = 7 * 24 * 3600 # 7 天未访问即淘汰
|
||||
VIDEO_EXTS = (".mp4", ".flv", ".mkv", ".webm")
|
||||
CONTENT_TYPES = {
|
||||
".mp4": "video/mp4",
|
||||
".flv": "video/x-flv",
|
||||
".mkv": "video/x-matroska",
|
||||
".webm": "video/webm",
|
||||
}
|
||||
|
||||
|
||||
def _prepare_cookie_file():
|
||||
"""Convert simple name=value cookies.txt to Netscape format for yt-dlp."""
|
||||
@@ -36,6 +51,65 @@ def _prepare_cookie_file():
|
||||
return tmp.name
|
||||
|
||||
|
||||
def _cache_file(bvid):
|
||||
"""缓存命中则返回文件路径(同时刷新 mtime 作为 LRU 依据),否则 None。"""
|
||||
for f in CACHE_DIR.glob(f"{bvid}.*"):
|
||||
if f.is_file() and f.suffix.lower() in VIDEO_EXTS:
|
||||
try:
|
||||
os.utime(f, None)
|
||||
except OSError:
|
||||
pass
|
||||
return f
|
||||
return None
|
||||
|
||||
|
||||
def _cache_files():
|
||||
out = []
|
||||
for f in CACHE_DIR.iterdir():
|
||||
if f.is_file() and f.suffix.lower() in VIDEO_EXTS:
|
||||
out.append(f)
|
||||
return out
|
||||
|
||||
|
||||
def _cache_sweep():
|
||||
"""先按 TTL 淘汰过期文件,再按总量上限从最旧(mtime 最小)开始淘汰。"""
|
||||
now = time.time()
|
||||
for f in _cache_files():
|
||||
try:
|
||||
if now - f.stat().st_mtime > CACHE_TTL_SECONDS:
|
||||
f.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
files = _cache_files()
|
||||
total = 0
|
||||
for f in files:
|
||||
try:
|
||||
total += f.stat().st_size
|
||||
except OSError:
|
||||
pass
|
||||
if total > CACHE_MAX_BYTES:
|
||||
for f in sorted(files, key=lambda p: p.stat().st_mtime):
|
||||
if total <= CACHE_MAX_BYTES:
|
||||
break
|
||||
try:
|
||||
size = f.stat().st_size
|
||||
f.unlink()
|
||||
total -= size
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def _file_response(path, bvid):
|
||||
return web.FileResponse(
|
||||
path=path,
|
||||
headers={
|
||||
"Content-Type": CONTENT_TYPES.get(path.suffix.lower(), "application/octet-stream"),
|
||||
"Content-Disposition": f'attachment; filename="{path.name}"',
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def resolve_bvid(link):
|
||||
# 直接是 BV 号
|
||||
m = re.search(r"(BV[a-zA-Z0-9]{10})", link)
|
||||
@@ -90,11 +164,19 @@ async def get_video(url_b64=None, bv=None):
|
||||
except ValueError as e:
|
||||
return web.json_response({"error": str(e)}, status=400)
|
||||
|
||||
# 1) 缓存命中:直接回文件,不再调用 yt-dlp
|
||||
cached = _cache_file(bvid)
|
||||
if cached:
|
||||
print(f"[bilibili] cache HIT {bvid} -> {cached.name} ({cached.stat().st_size} B)", flush=True)
|
||||
return _file_response(cached, bvid)
|
||||
|
||||
cookie_tmp = _prepare_cookie_file()
|
||||
if not cookie_tmp:
|
||||
return web.json_response({"error": "missing bilibili cookie"}, status=500)
|
||||
|
||||
tmpdir = tempfile.mkdtemp(prefix="bilibili_")
|
||||
print(f"[bilibili] cache MISS {bvid}, downloading...", flush=True)
|
||||
# 2) 下载到缓存目录内的隐藏临时目录(同盘,才能原子 rename 落盘)
|
||||
tmpdir = tempfile.mkdtemp(prefix=".dl-", dir=CACHE_DIR)
|
||||
output_template = os.path.join(tmpdir, "video.%(ext)s")
|
||||
cmd = [
|
||||
os.path.join(os.path.dirname(sys.executable), "yt-dlp"),
|
||||
@@ -113,16 +195,11 @@ async def get_video(url_b64=None, bv=None):
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
try:
|
||||
stdout, stderr = await asyncio.wait_for(
|
||||
proc.communicate(), timeout=300
|
||||
)
|
||||
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=300)
|
||||
except asyncio.TimeoutError:
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
return web.json_response({"error": "download timeout"}, status=504)
|
||||
finally:
|
||||
if cookie_tmp and os.path.exists(cookie_tmp):
|
||||
os.unlink(cookie_tmp)
|
||||
|
||||
video_files = [p for p in Path(tmpdir).glob("*") if p.is_file()]
|
||||
if proc.returncode != 0 and not video_files:
|
||||
@@ -136,26 +213,23 @@ async def get_video(url_b64=None, bv=None):
|
||||
return web.json_response({"error": "no output file"}, status=502)
|
||||
|
||||
video_path = max(video_files, key=lambda p: p.stat().st_size)
|
||||
|
||||
ext = video_path.suffix.lower()
|
||||
content_types = {
|
||||
".mp4": "video/mp4",
|
||||
".flv": "video/x-flv",
|
||||
".mkv": "video/x-matroska",
|
||||
".webm": "video/webm",
|
||||
}
|
||||
content_type = content_types.get(ext, "application/octet-stream")
|
||||
|
||||
return web.FileResponse(
|
||||
path=video_path,
|
||||
headers={
|
||||
"Content-Type": content_type,
|
||||
"Content-Disposition": f'attachment; filename="{video_path.name}"',
|
||||
},
|
||||
)
|
||||
final = CACHE_DIR / f"{bvid}{ext}"
|
||||
try:
|
||||
os.replace(video_path, final) # 同盘原子落盘
|
||||
except OSError:
|
||||
shutil.move(str(video_path), str(final))
|
||||
_cache_sweep()
|
||||
print(f"[bilibili] cache STORE {bvid} -> {final.name} ({final.stat().st_size} B)", flush=True)
|
||||
return _file_response(final, bvid)
|
||||
except Exception as e:
|
||||
print(f"[bilibili] Error: {e}")
|
||||
return web.json_response({"error": str(e)}, status=500)
|
||||
finally:
|
||||
# 关键修复:无论成功/失败/超时都回收临时目录(原实现会把整个视频永久留在 /tmp)
|
||||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||
if cookie_tmp and os.path.exists(cookie_tmp):
|
||||
os.unlink(cookie_tmp)
|
||||
|
||||
|
||||
async def handle_bilibili(request):
|
||||
@@ -168,8 +242,33 @@ async def handle_bilibili(request):
|
||||
return await get_video(url_b64=url_b64)
|
||||
|
||||
|
||||
async def handle_cache_info(request):
|
||||
files = []
|
||||
total = 0
|
||||
for f in _cache_files():
|
||||
try:
|
||||
st = f.stat()
|
||||
except OSError:
|
||||
continue
|
||||
files.append({"name": f.name, "size": st.st_size, "mtime": int(st.st_mtime)})
|
||||
total += st.st_size
|
||||
return web.json_response(
|
||||
{
|
||||
"cache_dir": str(CACHE_DIR),
|
||||
"cache_max_bytes": CACHE_MAX_BYTES,
|
||||
"cache_ttl_seconds": CACHE_TTL_SECONDS,
|
||||
"count": len(files),
|
||||
"total_bytes": total,
|
||||
"free_bytes": shutil.disk_usage(CACHE_DIR).free,
|
||||
"files": sorted(files, key=lambda x: -x["size"]),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
app = web.Application()
|
||||
app.router.add_get("/bilibili/get_video", handle_bilibili)
|
||||
app.router.add_get("/bilibili/cache_info", handle_cache_info)
|
||||
|
||||
if __name__ == "__main__":
|
||||
web.run_app(app, host="127.0.0.1", port=19990)
|
||||
print(f"Cache directory: {CACHE_DIR}")
|
||||
web.run_app(app, host="127.0.0.1", port=9006)
|
||||
|
||||
Reference in New Issue
Block a user