From 5400dc0a86ee7a62637cb57be0cd10a3503b847c Mon Sep 17 00:00:00 2001 From: RRRosmontis Date: Sun, 13 Sep 2026 02:26:34 +0800 Subject: [PATCH] =?UTF-8?q?feat(bilibili):=20=E6=96=B0=E5=A2=9E=20BV=20?= =?UTF-8?q?=E6=9C=AC=E5=9C=B0=E7=BC=93=E5=AD=98=EF=BC=8C=E4=BF=AE=E5=A4=8D?= =?UTF-8?q?=E8=A7=86=E9=A2=91=E6=9A=82=E5=AD=98=E7=9B=AE=E5=BD=95=E6=B3=84?= =?UTF-8?q?=E6=BC=8F?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 新增 CACHE_DIR(../cache/bilibili)，以 BV 号为键缓存 {BV}.mp4，命中即 FileResponse、不再调用 yt-dlp - 未命中时下载到缓存目录内的隐藏暂存目录 .dl-XXXX，成功后 os.replace 原子落盘 - finally 中 shutil.rmtree(tmpdir)：原实现只清理 cookie 临时文件，导致每次取视频都把完整视频 永久留在 /tmp（无上限磁盘泄漏） - 淘汰策略：TTL 7 天未访问过期 + 总量超过 2 GiB 按 mtime 从最旧开始删除 - 新增运维接口 GET /bilibili/cache_info（缓存目录/上限/文件数/总量/剩余磁盘） - 缓存 HIT/MISS/STORE 日志加 flush=True，实时可见 --- source/bilibili/bilibili.py | 147 ++++++++++++++++++++++++++++++------ 1 file changed, 123 insertions(+), 24 deletions(-) diff --git a/source/bilibili/bilibili.py b/source/bilibili/bilibili.py index 698d0eb..66471dd 100644 --- a/source/bilibili/bilibili.py +++ b/source/bilibili/bilibili.py @@ -3,7 +3,9 @@ import sys import base64 import os import re +import shutil import tempfile +import time from pathlib import Path import httpx @@ -12,6 +14,19 @@ from aiohttp import web COOKIE_FILE = str(Path(__file__).parent / "cookies.txt") UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36" +# ---------------- 本地缓存（与 pixiv 同级：source/cache/bilibili） ---------------- +CACHE_DIR = (Path(__file__).resolve().parent / "../cache/bilibili").resolve() +CACHE_DIR.mkdir(parents=True, exist_ok=True) +CACHE_MAX_BYTES = 2 * 1024 ** 3 # 缓存总量上限 2 GiB +CACHE_TTL_SECONDS = 7 * 24 * 3600 # 7 天未访问即淘汰 +VIDEO_EXTS = (".mp4", ".flv", ".mkv", ".webm") +CONTENT_TYPES = { + ".mp4": "video/mp4", + ".flv": "video/x-flv", + ".mkv": "video/x-matroska", + ".webm": "video/webm", +} + def _prepare_cookie_file(): """Convert simple name=value cookies.txt to Netscape format for yt-dlp.""" @@ -36,6 +51,65 @@ def _prepare_cookie_file(): return tmp.name +def _cache_file(bvid): + """缓存命中则返回文件路径（同时刷新 mtime 作为 LRU 依据），否则 None。""" + for f in CACHE_DIR.glob(f"{bvid}.*"): + if f.is_file() and f.suffix.lower() in VIDEO_EXTS: + try: + os.utime(f, None) + except OSError: + pass + return f + return None + + +def _cache_files(): + out = [] + for f in CACHE_DIR.iterdir(): + if f.is_file() and f.suffix.lower() in VIDEO_EXTS: + out.append(f) + return out + + +def _cache_sweep(): + """先按 TTL 淘汰过期文件，再按总量上限从最旧（mtime 最小）开始淘汰。""" + now = time.time() + for f in _cache_files(): + try: + if now - f.stat().st_mtime > CACHE_TTL_SECONDS: + f.unlink() + except OSError: + pass + + files = _cache_files() + total = 0 + for f in files: + try: + total += f.stat().st_size + except OSError: + pass + if total > CACHE_MAX_BYTES: + for f in sorted(files, key=lambda p: p.stat().st_mtime): + if total <= CACHE_MAX_BYTES: + break + try: + size = f.stat().st_size + f.unlink() + total -= size + except OSError: + pass + + +def _file_response(path, bvid): + return web.FileResponse( + path=path, + headers={ + "Content-Type": CONTENT_TYPES.get(path.suffix.lower(), "application/octet-stream"), + "Content-Disposition": f'attachment; filename="{path.name}"', + }, + ) + + async def resolve_bvid(link): # 直接是 BV 号 m = re.search(r"(BV[a-zA-Z0-9]{10})", link) @@ -90,11 +164,19 @@ async def get_video(url_b64=None, bv=None): except ValueError as e: return web.json_response({"error": str(e)}, status=400) + # 1) 缓存命中：直接回文件，不再调用 yt-dlp + cached = _cache_file(bvid) + if cached: + print(f"[bilibili] cache HIT {bvid} -> {cached.name} ({cached.stat().st_size} B)", flush=True) + return _file_response(cached, bvid) + cookie_tmp = _prepare_cookie_file() if not cookie_tmp: return web.json_response({"error": "missing bilibili cookie"}, status=500) - tmpdir = tempfile.mkdtemp(prefix="bilibili_") + print(f"[bilibili] cache MISS {bvid}, downloading...", flush=True) + # 2) 下载到缓存目录内的隐藏临时目录（同盘，才能原子 rename 落盘） + tmpdir = tempfile.mkdtemp(prefix=".dl-", dir=CACHE_DIR) output_template = os.path.join(tmpdir, "video.%(ext)s") cmd = [ os.path.join(os.path.dirname(sys.executable), "yt-dlp"), @@ -113,16 +195,11 @@ async def get_video(url_b64=None, bv=None): stderr=asyncio.subprocess.PIPE, ) try: - stdout, stderr = await asyncio.wait_for( - proc.communicate(), timeout=300 - ) + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=300) except asyncio.TimeoutError: proc.kill() await proc.wait() return web.json_response({"error": "download timeout"}, status=504) - finally: - if cookie_tmp and os.path.exists(cookie_tmp): - os.unlink(cookie_tmp) video_files = [p for p in Path(tmpdir).glob("*") if p.is_file()] if proc.returncode != 0 and not video_files: @@ -136,26 +213,23 @@ async def get_video(url_b64=None, bv=None): return web.json_response({"error": "no output file"}, status=502) video_path = max(video_files, key=lambda p: p.stat().st_size) - ext = video_path.suffix.lower() - content_types = { - ".mp4": "video/mp4", - ".flv": "video/x-flv", - ".mkv": "video/x-matroska", - ".webm": "video/webm", - } - content_type = content_types.get(ext, "application/octet-stream") - - return web.FileResponse( - path=video_path, - headers={ - "Content-Type": content_type, - "Content-Disposition": f'attachment; filename="{video_path.name}"', - }, - ) + final = CACHE_DIR / f"{bvid}{ext}" + try: + os.replace(video_path, final) # 同盘原子落盘 + except OSError: + shutil.move(str(video_path), str(final)) + _cache_sweep() + print(f"[bilibili] cache STORE {bvid} -> {final.name} ({final.stat().st_size} B)", flush=True) + return _file_response(final, bvid) except Exception as e: print(f"[bilibili] Error: {e}") return web.json_response({"error": str(e)}, status=500) + finally: + # 关键修复：无论成功/失败/超时都回收临时目录（原实现会把整个视频永久留在 /tmp） + shutil.rmtree(tmpdir, ignore_errors=True) + if cookie_tmp and os.path.exists(cookie_tmp): + os.unlink(cookie_tmp) async def handle_bilibili(request): @@ -168,8 +242,33 @@ async def handle_bilibili(request): return await get_video(url_b64=url_b64) +async def handle_cache_info(request): + files = [] + total = 0 + for f in _cache_files(): + try: + st = f.stat() + except OSError: + continue + files.append({"name": f.name, "size": st.st_size, "mtime": int(st.st_mtime)}) + total += st.st_size + return web.json_response( + { + "cache_dir": str(CACHE_DIR), + "cache_max_bytes": CACHE_MAX_BYTES, + "cache_ttl_seconds": CACHE_TTL_SECONDS, + "count": len(files), + "total_bytes": total, + "free_bytes": shutil.disk_usage(CACHE_DIR).free, + "files": sorted(files, key=lambda x: -x["size"]), + } + ) + + app = web.Application() app.router.add_get("/bilibili/get_video", handle_bilibili) +app.router.add_get("/bilibili/cache_info", handle_cache_info) if __name__ == "__main__": - web.run_app(app, host="127.0.0.1", port=19990) + print(f"Cache directory: {CACHE_DIR}") + web.run_app(app, host="127.0.0.1", port=9006)