feat(bilibili): 新增 BV 本地缓存,修复视频暂存目录泄漏
- 新增 CACHE_DIR(../cache/bilibili),以 BV 号为键缓存 {BV}.mp4,命中即 FileResponse、不再调用 yt-dlp
- 未命中时下载到缓存目录内的隐藏暂存目录 .dl-XXXX,成功后 os.replace 原子落盘
- finally 中 shutil.rmtree(tmpdir):原实现只清理 cookie 临时文件,导致每次取视频都把完整视频
永久留在 /tmp(无上限磁盘泄漏)
- 淘汰策略:TTL 7 天未访问过期 + 总量超过 2 GiB 按 mtime 从最旧开始删除
- 新增运维接口 GET /bilibili/cache_info(缓存目录/上限/文件数/总量/剩余磁盘)
- 缓存 HIT/MISS/STORE 日志加 flush=True,实时可见
This commit is contained in:
+123
-24
@@ -3,7 +3,9 @@ import sys
|
|||||||
import base64
|
import base64
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
import shutil
|
||||||
import tempfile
|
import tempfile
|
||||||
|
import time
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
|
|
||||||
import httpx
|
import httpx
|
||||||
@@ -12,6 +14,19 @@ from aiohttp import web
|
|||||||
COOKIE_FILE = str(Path(__file__).parent / "cookies.txt")
|
COOKIE_FILE = str(Path(__file__).parent / "cookies.txt")
|
||||||
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
|
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
|
||||||
|
|
||||||
|
# ---------------- 本地缓存(与 pixiv 同级:source/cache/bilibili) ----------------
|
||||||
|
CACHE_DIR = (Path(__file__).resolve().parent / "../cache/bilibili").resolve()
|
||||||
|
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||||
|
CACHE_MAX_BYTES = 2 * 1024 ** 3 # 缓存总量上限 2 GiB
|
||||||
|
CACHE_TTL_SECONDS = 7 * 24 * 3600 # 7 天未访问即淘汰
|
||||||
|
VIDEO_EXTS = (".mp4", ".flv", ".mkv", ".webm")
|
||||||
|
CONTENT_TYPES = {
|
||||||
|
".mp4": "video/mp4",
|
||||||
|
".flv": "video/x-flv",
|
||||||
|
".mkv": "video/x-matroska",
|
||||||
|
".webm": "video/webm",
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def _prepare_cookie_file():
|
def _prepare_cookie_file():
|
||||||
"""Convert simple name=value cookies.txt to Netscape format for yt-dlp."""
|
"""Convert simple name=value cookies.txt to Netscape format for yt-dlp."""
|
||||||
@@ -36,6 +51,65 @@ def _prepare_cookie_file():
|
|||||||
return tmp.name
|
return tmp.name
|
||||||
|
|
||||||
|
|
||||||
|
def _cache_file(bvid):
|
||||||
|
"""缓存命中则返回文件路径(同时刷新 mtime 作为 LRU 依据),否则 None。"""
|
||||||
|
for f in CACHE_DIR.glob(f"{bvid}.*"):
|
||||||
|
if f.is_file() and f.suffix.lower() in VIDEO_EXTS:
|
||||||
|
try:
|
||||||
|
os.utime(f, None)
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
return f
|
||||||
|
return None
|
||||||
|
|
||||||
|
|
||||||
|
def _cache_files():
|
||||||
|
out = []
|
||||||
|
for f in CACHE_DIR.iterdir():
|
||||||
|
if f.is_file() and f.suffix.lower() in VIDEO_EXTS:
|
||||||
|
out.append(f)
|
||||||
|
return out
|
||||||
|
|
||||||
|
|
||||||
|
def _cache_sweep():
|
||||||
|
"""先按 TTL 淘汰过期文件,再按总量上限从最旧(mtime 最小)开始淘汰。"""
|
||||||
|
now = time.time()
|
||||||
|
for f in _cache_files():
|
||||||
|
try:
|
||||||
|
if now - f.stat().st_mtime > CACHE_TTL_SECONDS:
|
||||||
|
f.unlink()
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
files = _cache_files()
|
||||||
|
total = 0
|
||||||
|
for f in files:
|
||||||
|
try:
|
||||||
|
total += f.stat().st_size
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
if total > CACHE_MAX_BYTES:
|
||||||
|
for f in sorted(files, key=lambda p: p.stat().st_mtime):
|
||||||
|
if total <= CACHE_MAX_BYTES:
|
||||||
|
break
|
||||||
|
try:
|
||||||
|
size = f.stat().st_size
|
||||||
|
f.unlink()
|
||||||
|
total -= size
|
||||||
|
except OSError:
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
||||||
|
def _file_response(path, bvid):
|
||||||
|
return web.FileResponse(
|
||||||
|
path=path,
|
||||||
|
headers={
|
||||||
|
"Content-Type": CONTENT_TYPES.get(path.suffix.lower(), "application/octet-stream"),
|
||||||
|
"Content-Disposition": f'attachment; filename="{path.name}"',
|
||||||
|
},
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
async def resolve_bvid(link):
|
async def resolve_bvid(link):
|
||||||
# 直接是 BV 号
|
# 直接是 BV 号
|
||||||
m = re.search(r"(BV[a-zA-Z0-9]{10})", link)
|
m = re.search(r"(BV[a-zA-Z0-9]{10})", link)
|
||||||
@@ -90,11 +164,19 @@ async def get_video(url_b64=None, bv=None):
|
|||||||
except ValueError as e:
|
except ValueError as e:
|
||||||
return web.json_response({"error": str(e)}, status=400)
|
return web.json_response({"error": str(e)}, status=400)
|
||||||
|
|
||||||
|
# 1) 缓存命中:直接回文件,不再调用 yt-dlp
|
||||||
|
cached = _cache_file(bvid)
|
||||||
|
if cached:
|
||||||
|
print(f"[bilibili] cache HIT {bvid} -> {cached.name} ({cached.stat().st_size} B)", flush=True)
|
||||||
|
return _file_response(cached, bvid)
|
||||||
|
|
||||||
cookie_tmp = _prepare_cookie_file()
|
cookie_tmp = _prepare_cookie_file()
|
||||||
if not cookie_tmp:
|
if not cookie_tmp:
|
||||||
return web.json_response({"error": "missing bilibili cookie"}, status=500)
|
return web.json_response({"error": "missing bilibili cookie"}, status=500)
|
||||||
|
|
||||||
tmpdir = tempfile.mkdtemp(prefix="bilibili_")
|
print(f"[bilibili] cache MISS {bvid}, downloading...", flush=True)
|
||||||
|
# 2) 下载到缓存目录内的隐藏临时目录(同盘,才能原子 rename 落盘)
|
||||||
|
tmpdir = tempfile.mkdtemp(prefix=".dl-", dir=CACHE_DIR)
|
||||||
output_template = os.path.join(tmpdir, "video.%(ext)s")
|
output_template = os.path.join(tmpdir, "video.%(ext)s")
|
||||||
cmd = [
|
cmd = [
|
||||||
os.path.join(os.path.dirname(sys.executable), "yt-dlp"),
|
os.path.join(os.path.dirname(sys.executable), "yt-dlp"),
|
||||||
@@ -113,16 +195,11 @@ async def get_video(url_b64=None, bv=None):
|
|||||||
stderr=asyncio.subprocess.PIPE,
|
stderr=asyncio.subprocess.PIPE,
|
||||||
)
|
)
|
||||||
try:
|
try:
|
||||||
stdout, stderr = await asyncio.wait_for(
|
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=300)
|
||||||
proc.communicate(), timeout=300
|
|
||||||
)
|
|
||||||
except asyncio.TimeoutError:
|
except asyncio.TimeoutError:
|
||||||
proc.kill()
|
proc.kill()
|
||||||
await proc.wait()
|
await proc.wait()
|
||||||
return web.json_response({"error": "download timeout"}, status=504)
|
return web.json_response({"error": "download timeout"}, status=504)
|
||||||
finally:
|
|
||||||
if cookie_tmp and os.path.exists(cookie_tmp):
|
|
||||||
os.unlink(cookie_tmp)
|
|
||||||
|
|
||||||
video_files = [p for p in Path(tmpdir).glob("*") if p.is_file()]
|
video_files = [p for p in Path(tmpdir).glob("*") if p.is_file()]
|
||||||
if proc.returncode != 0 and not video_files:
|
if proc.returncode != 0 and not video_files:
|
||||||
@@ -136,26 +213,23 @@ async def get_video(url_b64=None, bv=None):
|
|||||||
return web.json_response({"error": "no output file"}, status=502)
|
return web.json_response({"error": "no output file"}, status=502)
|
||||||
|
|
||||||
video_path = max(video_files, key=lambda p: p.stat().st_size)
|
video_path = max(video_files, key=lambda p: p.stat().st_size)
|
||||||
|
|
||||||
ext = video_path.suffix.lower()
|
ext = video_path.suffix.lower()
|
||||||
content_types = {
|
final = CACHE_DIR / f"{bvid}{ext}"
|
||||||
".mp4": "video/mp4",
|
try:
|
||||||
".flv": "video/x-flv",
|
os.replace(video_path, final) # 同盘原子落盘
|
||||||
".mkv": "video/x-matroska",
|
except OSError:
|
||||||
".webm": "video/webm",
|
shutil.move(str(video_path), str(final))
|
||||||
}
|
_cache_sweep()
|
||||||
content_type = content_types.get(ext, "application/octet-stream")
|
print(f"[bilibili] cache STORE {bvid} -> {final.name} ({final.stat().st_size} B)", flush=True)
|
||||||
|
return _file_response(final, bvid)
|
||||||
return web.FileResponse(
|
|
||||||
path=video_path,
|
|
||||||
headers={
|
|
||||||
"Content-Type": content_type,
|
|
||||||
"Content-Disposition": f'attachment; filename="{video_path.name}"',
|
|
||||||
},
|
|
||||||
)
|
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f"[bilibili] Error: {e}")
|
print(f"[bilibili] Error: {e}")
|
||||||
return web.json_response({"error": str(e)}, status=500)
|
return web.json_response({"error": str(e)}, status=500)
|
||||||
|
finally:
|
||||||
|
# 关键修复:无论成功/失败/超时都回收临时目录(原实现会把整个视频永久留在 /tmp)
|
||||||
|
shutil.rmtree(tmpdir, ignore_errors=True)
|
||||||
|
if cookie_tmp and os.path.exists(cookie_tmp):
|
||||||
|
os.unlink(cookie_tmp)
|
||||||
|
|
||||||
|
|
||||||
async def handle_bilibili(request):
|
async def handle_bilibili(request):
|
||||||
@@ -168,8 +242,33 @@ async def handle_bilibili(request):
|
|||||||
return await get_video(url_b64=url_b64)
|
return await get_video(url_b64=url_b64)
|
||||||
|
|
||||||
|
|
||||||
|
async def handle_cache_info(request):
|
||||||
|
files = []
|
||||||
|
total = 0
|
||||||
|
for f in _cache_files():
|
||||||
|
try:
|
||||||
|
st = f.stat()
|
||||||
|
except OSError:
|
||||||
|
continue
|
||||||
|
files.append({"name": f.name, "size": st.st_size, "mtime": int(st.st_mtime)})
|
||||||
|
total += st.st_size
|
||||||
|
return web.json_response(
|
||||||
|
{
|
||||||
|
"cache_dir": str(CACHE_DIR),
|
||||||
|
"cache_max_bytes": CACHE_MAX_BYTES,
|
||||||
|
"cache_ttl_seconds": CACHE_TTL_SECONDS,
|
||||||
|
"count": len(files),
|
||||||
|
"total_bytes": total,
|
||||||
|
"free_bytes": shutil.disk_usage(CACHE_DIR).free,
|
||||||
|
"files": sorted(files, key=lambda x: -x["size"]),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
app = web.Application()
|
app = web.Application()
|
||||||
app.router.add_get("/bilibili/get_video", handle_bilibili)
|
app.router.add_get("/bilibili/get_video", handle_bilibili)
|
||||||
|
app.router.add_get("/bilibili/cache_info", handle_cache_info)
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
web.run_app(app, host="127.0.0.1", port=19990)
|
print(f"Cache directory: {CACHE_DIR}")
|
||||||
|
web.run_app(app, host="127.0.0.1", port=9006)
|
||||||
|
|||||||
Reference in New Issue
Block a user