feat(jm): 新增 JM 漫画下载打包服务（9004，移植自 qqbot jm 模块）
- source/jm/jm.py：aiohttp + jmcomic，GET /jm/get_comic?id=xxx[&range=起-止] 下载至 cache/jm 隐藏暂存目录 → webp 转 jpg 打包 ZIP → HTTP 直返 → 600s 延时删除 单次上限 30 章；per-album 锁防并发；cache_sweep 兜底清理；/jm/cache_info 运维接口 - source/main.py：聚合层注册 /jm/get_comic（新增 JM_BACKEND，复用 proxy()） - systemd/jm.service：独立系统服务（Restart=always，日志 /var/log/jm.log） - README / 部署与运维文档同步更新
This commit is contained in:
RRRosmontis
+280
@@ -0,0 +1,280 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# jm.py — JM 漫画下载 API 模块
|
||||
# 流程: 源站下载 -> 本地临时目录打包 ZIP -> HTTP 返回 -> 延时删除
|
||||
# 移植自 qqbot 的 jm 模块（bj C:/call/bot/utility/jm.py），
|
||||
# 去掉 QQ 机器人 / 代理轮换 / 上传 cf 逻辑，改为与 pixiv/bilibili 一致的 HTTP 直返模式。
|
||||
# hk 服务器可直连源站，无需代理。
|
||||
|
||||
import asyncio
|
||||
import shutil
|
||||
import tempfile
|
||||
import time
|
||||
import unicodedata
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
from aiohttp import web
|
||||
|
||||
# ---------------- 本地临时缓存（与 pixiv/bilibili 同级：source/cache/jm） ----------------
|
||||
CACHE_DIR = (Path(__file__).resolve().parent / "../cache/jm").resolve()
|
||||
CACHE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
MAX_CHAPTERS = 30 # 单次最多章节（与 qqbot 一致，同时避免超过聚合层 300s 超时）
|
||||
ZIP_TTL_SECONDS = 600 # ZIP 返回后保留 10 分钟再删除
|
||||
PORT = 9004
|
||||
|
||||
_image_exts = (".webp", ".jpg", ".png")
|
||||
|
||||
# 同一本子防并发（album_id -> asyncio.Lock）
|
||||
_locks = {}
|
||||
_locks_guard = asyncio.Lock()
|
||||
|
||||
|
||||
def _sanitize_filename(name):
|
||||
"""清理文件名非法字符，保留中英文/数字/常见符号"""
|
||||
cleaned = unicodedata.normalize("NFKC", name or "")
|
||||
result = []
|
||||
for ch in cleaned:
|
||||
if ch.isalnum() or ch in ("_", "-", ".", "(", ")", "[", "]", "（", "）", "【", "】"):
|
||||
result.append(ch)
|
||||
elif ch.isspace():
|
||||
result.append("_")
|
||||
else:
|
||||
result.append("-")
|
||||
s = "".join(result)
|
||||
while "__" in s:
|
||||
s = s.replace("__", "_")
|
||||
while "--" in s:
|
||||
s = s.replace("--", "-")
|
||||
return s[:80].strip("_-") or "unnamed"
|
||||
|
||||
|
||||
def _zip_files():
|
||||
return [f for f in CACHE_DIR.glob("*.zip") if f.is_file()]
|
||||
|
||||
|
||||
def _cache_sweep():
|
||||
"""兜底清理：进程重启后残留的过期 ZIP（mtime 超过 TTL 即删）"""
|
||||
now = time.time()
|
||||
for f in _zip_files():
|
||||
try:
|
||||
if now - f.stat().st_mtime > ZIP_TTL_SECONDS:
|
||||
f.unlink()
|
||||
print(f"[jm] sweep expired zip: {f.name}", flush=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
async def _delete_later(path, delay):
|
||||
"""返回响应后延时删除 ZIP"""
|
||||
await asyncio.sleep(delay)
|
||||
try:
|
||||
Path(path).unlink(missing_ok=True)
|
||||
print(f"[jm] ttl delete: {Path(path).name}", flush=True)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
class _ParamError(RuntimeError):
|
||||
"""参数类错误，返回 400"""
|
||||
|
||||
|
||||
def _resolve_album_plan(album_id, range_start, range_end):
|
||||
"""获取元数据并确定章节范围与 ZIP 文件名。
|
||||
返回 (title, episodes, range_start, range_end, zip_name)，失败抛异常"""
|
||||
from jmcomic import JmModuleConfig
|
||||
|
||||
option = JmModuleConfig.option_class().default()
|
||||
client = option.build_jm_client()
|
||||
album = client.get_album_detail(int(album_id))
|
||||
title = getattr(album, "title", "") or str(album_id)
|
||||
episodes = getattr(album, "episode_list", []) or []
|
||||
total = len(episodes)
|
||||
if total == 0:
|
||||
raise RuntimeError("未找到任何章节")
|
||||
|
||||
if range_start is None:
|
||||
if total > MAX_CHAPTERS:
|
||||
raise _ParamError(
|
||||
f"共 {total} 章, 超过单次上限 {MAX_CHAPTERS} 章, 请用 range 参数分段下载 (如 range=1-{MAX_CHAPTERS})"
|
||||
)
|
||||
range_start, range_end = 1, total
|
||||
else:
|
||||
if range_start < 1 or range_end < range_start:
|
||||
raise _ParamError("章节范围不合法")
|
||||
if range_end - range_start + 1 > MAX_CHAPTERS:
|
||||
raise _ParamError(f"章节跨度超过 {MAX_CHAPTERS} 章, 请缩小范围")
|
||||
if range_start > total:
|
||||
raise _ParamError(f"起始章节超出总章节数 {total}")
|
||||
range_end = min(range_end, total)
|
||||
|
||||
zip_name = f"{_sanitize_filename(title)}_{album_id}_ch{range_start}-{range_end}.zip"
|
||||
return title, episodes, range_start, range_end, zip_name
|
||||
|
||||
|
||||
def _download_and_pack(album_id, plan):
|
||||
"""下载并打包 ZIP。成功返回 (zip_path, image_count, ok_ch, fail_ch)，失败抛异常"""
|
||||
from jmcomic import JmModuleConfig
|
||||
from PIL import Image
|
||||
|
||||
title, episodes, range_start, range_end, zip_name = plan
|
||||
|
||||
work_dir = Path(tempfile.mkdtemp(prefix=f".dl-{album_id}-", dir=CACHE_DIR))
|
||||
try:
|
||||
# 逐章下载（hk 直连源站，无需代理）
|
||||
ok_count = 0
|
||||
fail_count = 0
|
||||
for idx in range(range_start - 1, range_end):
|
||||
photo_id = episodes[idx][0]
|
||||
try:
|
||||
dl_option = JmModuleConfig.option_class().construct({
|
||||
"dir_rule": {"rule": "Bd_Pid", "base_dir": str(work_dir)},
|
||||
"download": {"image": {"suffix": ".webp"}},
|
||||
})
|
||||
dler = JmModuleConfig.downloader_class()(dl_option)
|
||||
dler.download_photo(int(photo_id))
|
||||
ok_count += 1
|
||||
print(f"[jm] 章节下载完成: {album_id}/{photo_id}", flush=True)
|
||||
except Exception as e:
|
||||
fail_count += 1
|
||||
print(f"[jm] 章节下载失败: {album_id}/{photo_id}: {e}", flush=True)
|
||||
|
||||
# 收集图片
|
||||
images = []
|
||||
for ext in _image_exts:
|
||||
images.extend(sorted(work_dir.rglob(f"*{ext}")))
|
||||
if not images:
|
||||
raise RuntimeError("未下载到任何图片")
|
||||
|
||||
# 打包 ZIP (webp 统一转 jpg, 无密码)
|
||||
zip_path = CACHE_DIR / zip_name
|
||||
tmp_dir = Path(tempfile.mkdtemp(prefix=".pack-", dir=CACHE_DIR))
|
||||
try:
|
||||
packed = []
|
||||
for i, p in enumerate(images, 1):
|
||||
out = tmp_dir / f"{i:04d}.jpg"
|
||||
try:
|
||||
img = Image.open(str(p))
|
||||
if img.mode != "RGB":
|
||||
img = img.convert("RGB")
|
||||
img.save(str(out), "JPEG", quality=95)
|
||||
except Exception:
|
||||
out = tmp_dir / f"{i:04d}{p.suffix.lower()}"
|
||||
shutil.copyfile(str(p), str(out))
|
||||
packed.append(out)
|
||||
with zipfile.ZipFile(zip_path, "w", zipfile.ZIP_DEFLATED) as zf:
|
||||
for j in packed:
|
||||
zf.write(j, j.name)
|
||||
finally:
|
||||
shutil.rmtree(tmp_dir, ignore_errors=True)
|
||||
|
||||
print(f"[jm] 打包完成: {zip_name} ({len(images)} 张, 成功 {ok_count} 章 / 失败 {fail_count} 章)", flush=True)
|
||||
return zip_path, len(images), ok_count, fail_count
|
||||
finally:
|
||||
# 无论成功/失败都回收图片临时目录
|
||||
shutil.rmtree(work_dir, ignore_errors=True)
|
||||
|
||||
|
||||
def _file_response(zip_path):
|
||||
return web.FileResponse(
|
||||
path=zip_path,
|
||||
headers={
|
||||
"Content-Type": "application/zip",
|
||||
"Content-Disposition": f'attachment; filename="{zip_path.name}"',
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
async def handle_jm(request):
|
||||
album_id = request.query.get("id", "").strip()
|
||||
if not album_id or not album_id.isdigit():
|
||||
return web.json_response({"error": "missing or invalid id parameter"}, status=400)
|
||||
|
||||
range_start = range_end = None
|
||||
range_str = request.query.get("range", "").strip()
|
||||
if range_str:
|
||||
import re
|
||||
m = re.fullmatch(r"(\d{1,4})-(\d{1,4})", range_str)
|
||||
if not m:
|
||||
return web.json_response(
|
||||
{"error": "invalid range format, expected: start-end (e.g. 1-30)"}, status=400
|
||||
)
|
||||
range_start, range_end = int(m.group(1)), int(m.group(2))
|
||||
|
||||
_cache_sweep()
|
||||
|
||||
# 同本子串行，防止并发重复下载
|
||||
async with _locks_guard:
|
||||
lock = _locks.setdefault(album_id, asyncio.Lock())
|
||||
if lock.locked():
|
||||
return web.json_response(
|
||||
{"error": "same album is being processed, please retry later"}, status=409
|
||||
)
|
||||
|
||||
async with lock:
|
||||
try:
|
||||
plan = await asyncio.to_thread(
|
||||
_resolve_album_plan, album_id, range_start, range_end
|
||||
)
|
||||
except _ParamError as e:
|
||||
return web.json_response({"error": str(e)}, status=400)
|
||||
except Exception as e:
|
||||
return web.json_response(
|
||||
{"error": "failed to fetch album info", "detail": str(e)[:200]}, status=502
|
||||
)
|
||||
|
||||
zip_name = plan[4]
|
||||
existing = CACHE_DIR / zip_name
|
||||
if existing.exists():
|
||||
# TTL 窗口内重复请求：直接复用并重新计时
|
||||
try:
|
||||
existing.touch()
|
||||
except OSError:
|
||||
pass
|
||||
print(f"[jm] reuse zip: {zip_name}", flush=True)
|
||||
asyncio.create_task(_delete_later(existing, ZIP_TTL_SECONDS))
|
||||
return _file_response(existing)
|
||||
|
||||
try:
|
||||
zip_path, img_count, ok_c, fail_c = await asyncio.to_thread(
|
||||
_download_and_pack, album_id, plan
|
||||
)
|
||||
except Exception as e:
|
||||
return web.json_response(
|
||||
{"error": "download or pack failed", "detail": str(e)[:200]}, status=502
|
||||
)
|
||||
|
||||
asyncio.create_task(_delete_later(zip_path, ZIP_TTL_SECONDS))
|
||||
return _file_response(zip_path)
|
||||
|
||||
|
||||
async def handle_cache_info(request):
|
||||
files = []
|
||||
total = 0
|
||||
for f in _zip_files():
|
||||
try:
|
||||
st = f.stat()
|
||||
except OSError:
|
||||
continue
|
||||
files.append({"name": f.name, "size": st.st_size, "mtime": int(st.st_mtime)})
|
||||
total += st.st_size
|
||||
return web.json_response(
|
||||
{
|
||||
"cache_dir": str(CACHE_DIR),
|
||||
"zip_ttl_seconds": ZIP_TTL_SECONDS,
|
||||
"max_chapters": MAX_CHAPTERS,
|
||||
"count": len(files),
|
||||
"total_bytes": total,
|
||||
"free_bytes": shutil.disk_usage(CACHE_DIR).free,
|
||||
"files": sorted(files, key=lambda x: -x["size"]),
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
app = web.Application()
|
||||
app.router.add_get("/jm/get_comic", handle_jm)
|
||||
app.router.add_get("/jm/cache_info", handle_cache_info)
|
||||
|
||||
if __name__ == "__main__":
|
||||
print(f"Cache directory: {CACHE_DIR}")
|
||||
web.run_app(app, host="127.0.0.1", port=PORT)
|
||||
@@ -4,6 +4,7 @@ from aiohttp import web, ClientSession, ClientTimeout
|
||||
PIXIV_BACKEND = "http://127.0.0.1:9007"
|
||||
BILIBILI_BACKEND = "http://127.0.0.1:9006"
|
||||
DUFE_BACKEND = "http://127.0.0.1:9005"
|
||||
JM_BACKEND = "http://127.0.0.1:9004"
|
||||
|
||||
|
||||
async def handle_pixiv(request):
|
||||
@@ -30,6 +31,14 @@ async def handle_dufe_sign_in(request):
|
||||
return await proxy_forward(request, target)
|
||||
|
||||
|
||||
async def handle_jm(request):
|
||||
qs = request.query_string
|
||||
target = f"{JM_BACKEND}/jm/get_comic"
|
||||
if qs:
|
||||
target += "?" + qs
|
||||
return await proxy(request, target)
|
||||
|
||||
|
||||
async def proxy(request, target):
|
||||
timeout = ClientTimeout(total=300)
|
||||
async with ClientSession(timeout=timeout) as session:
|
||||
@@ -92,6 +101,7 @@ app.router.add_get("/pixiv/get_image", handle_pixiv)
|
||||
app.router.add_get("/bilibili/get_video", handle_bilibili)
|
||||
app.router.add_get("/dufe/ginkgostu_sign_in", handle_dufe_sign_in)
|
||||
app.router.add_post("/dufe/ginkgostu_sign_in", handle_dufe_sign_in)
|
||||
app.router.add_get("/jm/get_comic", handle_jm)
|
||||
|
||||
if __name__ == "__main__":
|
||||
web.run_app(app, host="127.0.0.1", port=9008)
|
||||
|
||||
Reference in New Issue
Block a user