RRRosmontis
5400dc0a86
- 新增 CACHE_DIR(../cache/bilibili)，以 BV 号为键缓存 {BV}.mp4，命中即 FileResponse、不再调用 yt-dlp
- 未命中时下载到缓存目录内的隐藏暂存目录 .dl-XXXX，成功后 os.replace 原子落盘
- finally 中 shutil.rmtree(tmpdir)：原实现只清理 cookie 临时文件，导致每次取视频都把完整视频
永久留在 /tmp（无上限磁盘泄漏）
- 淘汰策略：TTL 7 天未访问过期 + 总量超过 2 GiB 按 mtime 从最旧开始删除
- 新增运维接口 GET /bilibili/cache_info（缓存目录/上限/文件数/总量/剩余磁盘）
- 缓存 HIT/MISS/STORE 日志加 flush=True，实时可见
275 lines
8.9 KiB
Python
275 lines
8.9 KiB
Python
import asyncio
|
||
import sys
|
||
import base64
|
||
import os
|
||
import re
|
||
import shutil
|
||
import tempfile
|
||
import time
|
||
from pathlib import Path
|
||
|
||
import httpx
|
||
from aiohttp import web
|
||
|
||
COOKIE_FILE = str(Path(__file__).parent / "cookies.txt")
|
||
UA = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36"
|
||
|
||
# ---------------- 本地缓存（与 pixiv 同级：source/cache/bilibili） ----------------
|
||
CACHE_DIR = (Path(__file__).resolve().parent / "../cache/bilibili").resolve()
|
||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||
CACHE_MAX_BYTES = 2 * 1024 ** 3 # 缓存总量上限 2 GiB
|
||
CACHE_TTL_SECONDS = 7 * 24 * 3600 # 7 天未访问即淘汰
|
||
VIDEO_EXTS = (".mp4", ".flv", ".mkv", ".webm")
|
||
CONTENT_TYPES = {
|
||
".mp4": "video/mp4",
|
||
".flv": "video/x-flv",
|
||
".mkv": "video/x-matroska",
|
||
".webm": "video/webm",
|
||
}
|
||
|
||
|
||
def _prepare_cookie_file():
|
||
"""Convert simple name=value cookies.txt to Netscape format for yt-dlp."""
|
||
try:
|
||
with open(COOKIE_FILE) as f:
|
||
cookies = [line.strip() for line in f if line.strip() and not line.startswith("#")]
|
||
except FileNotFoundError:
|
||
return None
|
||
if not cookies:
|
||
return None
|
||
tmp = tempfile.NamedTemporaryFile(
|
||
mode="w", suffix=".txt", prefix="bilibili_cookie_", delete=False
|
||
)
|
||
tmp.write("# Netscape HTTP Cookie File\n")
|
||
for c in cookies:
|
||
if "=" not in c:
|
||
continue
|
||
name, value = c.split("=", 1)
|
||
# Netscape format: domain flag path secure expires name value
|
||
tmp.write(f".bilibili.com\tTRUE\t/\tFALSE\t0\t{name}\t{value}\n")
|
||
tmp.close()
|
||
return tmp.name
|
||
|
||
|
||
def _cache_file(bvid):
|
||
"""缓存命中则返回文件路径（同时刷新 mtime 作为 LRU 依据），否则 None。"""
|
||
for f in CACHE_DIR.glob(f"{bvid}.*"):
|
||
if f.is_file() and f.suffix.lower() in VIDEO_EXTS:
|
||
try:
|
||
os.utime(f, None)
|
||
except OSError:
|
||
pass
|
||
return f
|
||
return None
|
||
|
||
|
||
def _cache_files():
|
||
out = []
|
||
for f in CACHE_DIR.iterdir():
|
||
if f.is_file() and f.suffix.lower() in VIDEO_EXTS:
|
||
out.append(f)
|
||
return out
|
||
|
||
|
||
def _cache_sweep():
|
||
"""先按 TTL 淘汰过期文件，再按总量上限从最旧（mtime 最小）开始淘汰。"""
|
||
now = time.time()
|
||
for f in _cache_files():
|
||
try:
|
||
if now - f.stat().st_mtime > CACHE_TTL_SECONDS:
|
||
f.unlink()
|
||
except OSError:
|
||
pass
|
||
|
||
files = _cache_files()
|
||
total = 0
|
||
for f in files:
|
||
try:
|
||
total += f.stat().st_size
|
||
except OSError:
|
||
pass
|
||
if total > CACHE_MAX_BYTES:
|
||
for f in sorted(files, key=lambda p: p.stat().st_mtime):
|
||
if total <= CACHE_MAX_BYTES:
|
||
break
|
||
try:
|
||
size = f.stat().st_size
|
||
f.unlink()
|
||
total -= size
|
||
except OSError:
|
||
pass
|
||
|
||
|
||
def _file_response(path, bvid):
|
||
return web.FileResponse(
|
||
path=path,
|
||
headers={
|
||
"Content-Type": CONTENT_TYPES.get(path.suffix.lower(), "application/octet-stream"),
|
||
"Content-Disposition": f'attachment; filename="{path.name}"',
|
||
},
|
||
)
|
||
|
||
|
||
async def resolve_bvid(link):
|
||
# 直接是 BV 号
|
||
m = re.search(r"(BV[a-zA-Z0-9]{10})", link)
|
||
if m:
|
||
return m.group(1)
|
||
|
||
# b23.tv 短链接，跟随跳转
|
||
if "b23.tv" in link:
|
||
async with httpx.AsyncClient(timeout=10) as client:
|
||
resp = await client.get(
|
||
link.strip(),
|
||
follow_redirects=False,
|
||
headers={"User-Agent": UA},
|
||
)
|
||
location = resp.headers.get("Location", "")
|
||
if m := re.search(r"(BV[a-zA-Z0-9]{10})", location):
|
||
return m.group(1)
|
||
if m := re.search(r"(BV[a-zA-Z0-9]{10})", resp.text):
|
||
return m.group(1)
|
||
|
||
# 普通 bilibili.com URL
|
||
if m := re.search(r"(BV[a-zA-Z0-9]{10})", link):
|
||
return m.group(1)
|
||
|
||
raise ValueError(f"unable to resolve BV from: {link}")
|
||
|
||
|
||
def decode_url(url_b64):
|
||
try:
|
||
return base64.urlsafe_b64decode(url_b64.encode()).decode()
|
||
except Exception:
|
||
return base64.b64decode(url_b64.encode()).decode()
|
||
|
||
|
||
async def get_video(url_b64=None, bv=None):
|
||
if bv:
|
||
bvid = bv
|
||
else:
|
||
try:
|
||
url = decode_url(url_b64)
|
||
except Exception:
|
||
return web.json_response({"error": "invalid base64 encoding"}, status=400)
|
||
|
||
if not url:
|
||
return web.json_response({"error": "empty url"}, status=400)
|
||
|
||
if "bilibili.com" not in url and "b23.tv" not in url and not url.startswith("BV"):
|
||
return web.json_response({"error": "not a bilibili url"}, status=400)
|
||
|
||
try:
|
||
bvid = await resolve_bvid(url)
|
||
except ValueError as e:
|
||
return web.json_response({"error": str(e)}, status=400)
|
||
|
||
# 1) 缓存命中：直接回文件，不再调用 yt-dlp
|
||
cached = _cache_file(bvid)
|
||
if cached:
|
||
print(f"[bilibili] cache HIT {bvid} -> {cached.name} ({cached.stat().st_size} B)", flush=True)
|
||
return _file_response(cached, bvid)
|
||
|
||
cookie_tmp = _prepare_cookie_file()
|
||
if not cookie_tmp:
|
||
return web.json_response({"error": "missing bilibili cookie"}, status=500)
|
||
|
||
print(f"[bilibili] cache MISS {bvid}, downloading...", flush=True)
|
||
# 2) 下载到缓存目录内的隐藏临时目录（同盘，才能原子 rename 落盘）
|
||
tmpdir = tempfile.mkdtemp(prefix=".dl-", dir=CACHE_DIR)
|
||
output_template = os.path.join(tmpdir, "video.%(ext)s")
|
||
cmd = [
|
||
os.path.join(os.path.dirname(sys.executable), "yt-dlp"),
|
||
"--cookies", cookie_tmp,
|
||
"--add-header", f"User-Agent:{UA}",
|
||
"--add-header", "Referer:https://www.bilibili.com/",
|
||
"-f", "bestvideo*+bestaudio/best",
|
||
"--merge-output-format", "mp4",
|
||
"-o", output_template,
|
||
f"https://www.bilibili.com/video/{bvid}",
|
||
]
|
||
try:
|
||
proc = await asyncio.create_subprocess_exec(
|
||
*cmd,
|
||
stdout=asyncio.subprocess.PIPE,
|
||
stderr=asyncio.subprocess.PIPE,
|
||
)
|
||
try:
|
||
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=300)
|
||
except asyncio.TimeoutError:
|
||
proc.kill()
|
||
await proc.wait()
|
||
return web.json_response({"error": "download timeout"}, status=504)
|
||
|
||
video_files = [p for p in Path(tmpdir).glob("*") if p.is_file()]
|
||
if proc.returncode != 0 and not video_files:
|
||
err_msg = stderr.decode()[-1000:] if stderr else "unknown error"
|
||
return web.json_response(
|
||
{"error": "download failed", "detail": err_msg},
|
||
status=502,
|
||
)
|
||
|
||
if not video_files:
|
||
return web.json_response({"error": "no output file"}, status=502)
|
||
|
||
video_path = max(video_files, key=lambda p: p.stat().st_size)
|
||
ext = video_path.suffix.lower()
|
||
final = CACHE_DIR / f"{bvid}{ext}"
|
||
try:
|
||
os.replace(video_path, final) # 同盘原子落盘
|
||
except OSError:
|
||
shutil.move(str(video_path), str(final))
|
||
_cache_sweep()
|
||
print(f"[bilibili] cache STORE {bvid} -> {final.name} ({final.stat().st_size} B)", flush=True)
|
||
return _file_response(final, bvid)
|
||
except Exception as e:
|
||
print(f"[bilibili] Error: {e}")
|
||
return web.json_response({"error": str(e)}, status=500)
|
||
finally:
|
||
# 关键修复：无论成功/失败/超时都回收临时目录（原实现会把整个视频永久留在 /tmp）
|
||
shutil.rmtree(tmpdir, ignore_errors=True)
|
||
if cookie_tmp and os.path.exists(cookie_tmp):
|
||
os.unlink(cookie_tmp)
|
||
|
||
|
||
async def handle_bilibili(request):
|
||
bv = request.query.get("bv", "")
|
||
if bv:
|
||
return await get_video(bv=bv)
|
||
url_b64 = request.query.get("url", "")
|
||
if not url_b64:
|
||
return web.json_response({"error": "missing url or bv parameter"}, status=400)
|
||
return await get_video(url_b64=url_b64)
|
||
|
||
|
||
async def handle_cache_info(request):
|
||
files = []
|
||
total = 0
|
||
for f in _cache_files():
|
||
try:
|
||
st = f.stat()
|
||
except OSError:
|
||
continue
|
||
files.append({"name": f.name, "size": st.st_size, "mtime": int(st.st_mtime)})
|
||
total += st.st_size
|
||
return web.json_response(
|
||
{
|
||
"cache_dir": str(CACHE_DIR),
|
||
"cache_max_bytes": CACHE_MAX_BYTES,
|
||
"cache_ttl_seconds": CACHE_TTL_SECONDS,
|
||
"count": len(files),
|
||
"total_bytes": total,
|
||
"free_bytes": shutil.disk_usage(CACHE_DIR).free,
|
||
"files": sorted(files, key=lambda x: -x["size"]),
|
||
}
|
||
)
|
||
|
||
|
||
app = web.Application()
|
||
app.router.add_get("/bilibili/get_video", handle_bilibili)
|
||
app.router.add_get("/bilibili/cache_info", handle_cache_info)
|
||
|
||
if __name__ == "__main__":
|
||
print(f"Cache directory: {CACHE_DIR}")
|
||
web.run_app(app, host="127.0.0.1", port=9006)
|