CareerNexus 职联：岗位能力图谱构建与动态演化分析平台（挑战杯 XH-202621）
- 全链路：多源JD接入/清洗交叉验证/结构化抽取(星火+离线双通道)/时序知识图谱/新岗位发现/演化分析/人岗匹配/学习路径 - WebUI 五视图：驾驶舱/图谱工作台/岗位演化/匹配(单人+批量)/学习路径 - 设置中心(config.json)/Token鉴权/SQLite状态库/语义向量通道(BGE-M3本地或联网API) - 71 个 pytest 用例；数据目录仅保留结构(.gitkeep)，数据由 scripts/generate_dataset.py 复现
This commit is contained in:
RoyirinRosmontis
@@ -0,0 +1,3 @@
|
||||
"""CareerNexus 职联 —— 多源异构数据驱动的岗位与能力图谱平台。"""
|
||||
|
||||
__version__ = "0.1.0"
|
||||
@@ -0,0 +1,812 @@
|
||||
"""FastAPI 接口层：统一响应 {code, message, data, request_id}。
|
||||
|
||||
- 耗时任务（JD 导入/全链路 pipeline）使用 任务ID + 轮询；
|
||||
- 托管 webui 静态文件（/ -> webui/index.html）；
|
||||
- 服务启动时若 data/graph.json 已存在则直接加载图谱，否则等首次导入。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from urllib.parse import quote
|
||||
|
||||
import json
|
||||
|
||||
from fastapi import FastAPI, File, Form, HTTPException, Request, Response, UploadFile
|
||||
from fastapi.responses import FileResponse, JSONResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
from pydantic import BaseModel, ValidationError
|
||||
|
||||
from app.config import ROOT, load_config, load_runtime_settings, resolve, save_runtime_settings
|
||||
from app.discover.cluster import apply_review, discover_new_jobs, load_pending, save_pending
|
||||
from app.extract.base import SkillMatcher, load_lexicon
|
||||
from app.extract.llm import llm_ready
|
||||
from app.graph.tkg import TemporalKG, now_str
|
||||
from app.ingest.loader import load_jd_dir
|
||||
from app.match import semantic
|
||||
from app.match.engine import MatchEngine
|
||||
from app.match.resume import parse_resume, parse_resume_file
|
||||
from app.models import JDRecord, ResumeProfile
|
||||
from app.services.pipeline import run_pipeline
|
||||
from app.storage import db as state_db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
app = FastAPI(title="CareerNexus 职联", version="0.1.0")
|
||||
|
||||
_cfg = load_config()
|
||||
_state: dict = {
|
||||
"kg": None,
|
||||
"records": None,
|
||||
"extractions": None,
|
||||
"tasks": {},
|
||||
}
|
||||
|
||||
|
||||
def _ok(data=None, message: str = "ok") -> dict:
|
||||
return {"code": 0, "message": message, "data": data,
|
||||
"request_id": uuid.uuid4().hex[:12]}
|
||||
|
||||
|
||||
def _get_kg() -> TemporalKG:
|
||||
kg = _state["kg"]
|
||||
if kg is None:
|
||||
raise HTTPException(status_code=503, detail="图谱尚未构建，请先调用 /api/jd/import")
|
||||
return kg
|
||||
|
||||
|
||||
def _db_path() -> Path:
|
||||
"""SQLite 应用状态库路径（配置键 state_db_path，默认 data/careernexus.db）。"""
|
||||
return resolve(_cfg, "state_db_path")
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
def _startup() -> None:
|
||||
graph_path = resolve(_cfg, "graph_path")
|
||||
if graph_path.exists():
|
||||
_state["kg"] = TemporalKG.load(graph_path)
|
||||
logger.info("已加载图谱：%s", _state["kg"].stats())
|
||||
else:
|
||||
logger.warning("图谱文件不存在（%s），请先执行 pipeline 或 /api/jd/import", graph_path)
|
||||
# SQLite 状态库：幂等建表 + 旧 JSONL/JSON 一次性迁移（原文件保留）
|
||||
state_db.connect(
|
||||
_db_path(),
|
||||
legacy_audit=resolve(_cfg, "audit_path"),
|
||||
legacy_pending=resolve(_cfg, "pending_path"),
|
||||
).close()
|
||||
# 重启后从 tasks 表恢复任务清单；中断的任务标记为失败
|
||||
for row in state_db.list_tasks(_db_path()):
|
||||
payload = row["payload"]
|
||||
if payload.get("status") in ("pending", "running"):
|
||||
payload = {**payload, "status": "failed", "error": "服务重启，任务中断"}
|
||||
_state["tasks"][row["task_id"]] = payload
|
||||
|
||||
|
||||
# ---------- 健康与图谱 ----------
|
||||
@app.get("/api/health")
|
||||
def health() -> dict:
|
||||
kg = _state["kg"]
|
||||
ready = llm_ready(_cfg)
|
||||
return _ok({"status": "ok", "graph": kg.stats() if kg else None,
|
||||
"llm_ready": ready, "offline_fallback": not ready,
|
||||
"embed": semantic.status(_cfg),
|
||||
"guided": bool(_cfg.get("guided")),
|
||||
"mode": "online" if ready else "offline"})
|
||||
|
||||
|
||||
# ---------- 静态资源缓存控制（避免浏览器用旧版前端） ----------
|
||||
@app.middleware("http")
|
||||
async def static_no_cache(request, call_next):
|
||||
resp = await call_next(request)
|
||||
if not request.url.path.startswith("/api/"):
|
||||
resp.headers["Cache-Control"] = "no-cache" # 每次加载都回源校验（ETag 命中仍 304，成本低）
|
||||
return resp
|
||||
|
||||
|
||||
# ---------- 访问令牌鉴权 ----------
|
||||
@app.middleware("http")
|
||||
async def access_token_guard(request, call_next):
|
||||
"""设置 access_token 后，/api/*（health 除外）必须携带 X-Access-Token 头。
|
||||
|
||||
仅在用户显式配置令牌时启用；默认无令牌、本地使用无感知。
|
||||
"""
|
||||
token = str(_cfg.get("access_token") or "").strip()
|
||||
path = request.url.path
|
||||
# 豁免：health 探活、GET /api/config（只读脱敏值，打开设置面板需要）
|
||||
exempt = path == "/api/health" or (path == "/api/config" and request.method == "GET")
|
||||
if token and path.startswith("/api/") and not exempt:
|
||||
if request.headers.get("x-access-token", "") != token:
|
||||
return JSONResponse({"detail": "未授权：缺少或错误的访问令牌"}, status_code=401)
|
||||
return await call_next(request)
|
||||
|
||||
|
||||
# ---------- 语义环境一键配置（按需下载，基础镜像不打包重依赖） ----------
|
||||
_semantic_setup: dict = {"running": False, "stage": "", "log": [], "error": "", "done": False}
|
||||
_semantic_setup_lock = threading.Lock()
|
||||
|
||||
|
||||
def _setup_log(msg: str) -> None:
|
||||
_semantic_setup["log"].append(msg)
|
||||
_semantic_setup["log"] = _semantic_setup["log"][-20:]
|
||||
logger.info("语义环境配置: %s", msg)
|
||||
|
||||
|
||||
def _run_semantic_setup() -> None:
|
||||
"""后台执行：装依赖 → 下载权重 → 写入配置并热生效。任一阶段失败记 error。"""
|
||||
import importlib
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
weights_dir = ROOT / "models" / "bge-m3"
|
||||
try:
|
||||
# 已就绪则直接写配置收尾（幂等）
|
||||
if not semantic.status(_cfg)["ready"]:
|
||||
_semantic_setup["stage"] = "deps"
|
||||
_setup_log("安装 Python 依赖 sentence-transformers / modelscope（镜像站）…")
|
||||
pip = [sys.executable, "-m", "pip", "install", "-q",
|
||||
"sentence-transformers", "modelscope",
|
||||
"-i", "https://pypi.tuna.tsinghua.edu.cn/simple"]
|
||||
proc = subprocess.run(pip, capture_output=True, text=True, timeout=1800)
|
||||
if proc.returncode != 0: # 镜像站失败退回官方源
|
||||
_setup_log("镜像站安装失败，改用官方源重试…")
|
||||
proc = subprocess.run(pip[:-2], capture_output=True, text=True, timeout=1800)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"依赖安装失败：{(proc.stderr or proc.stdout)[-300:]}")
|
||||
importlib.invalidate_caches()
|
||||
_setup_log("依赖安装完成")
|
||||
|
||||
if not semantic.weights_present(weights_dir):
|
||||
_semantic_setup["stage"] = "download"
|
||||
_setup_log("从 ModelScope 下载 BGE-M3 权重（约 2.2GB，数分钟）…")
|
||||
from modelscope import snapshot_download
|
||||
weights_dir.parent.mkdir(parents=True, exist_ok=True)
|
||||
snapshot_download("BAAI/bge-m3", local_dir=str(weights_dir))
|
||||
_setup_log("权重下载完成")
|
||||
else:
|
||||
_setup_log("检测到已有权重，跳过下载")
|
||||
|
||||
_semantic_setup["stage"] = "config"
|
||||
save_runtime_settings({"embed_provider": "local",
|
||||
"embed_model_path": str(weights_dir)})
|
||||
_cfg["embed_provider"] = "local"
|
||||
_cfg["embed_model_path"] = str(weights_dir)
|
||||
ready = semantic.status(_cfg)["ready"]
|
||||
_semantic_setup["done"] = ready
|
||||
_semantic_setup["stage"] = "done" if ready else "error"
|
||||
if not ready:
|
||||
_semantic_setup["error"] = "配置已写入但就绪检测未通过，请检查依赖与权重文件"
|
||||
_setup_log("配置完成" if ready else "就绪检测未通过")
|
||||
except Exception as exc: # noqa: BLE001 - 任何失败都要落到状态里供前端展示
|
||||
_semantic_setup["error"] = str(exc)
|
||||
_semantic_setup["stage"] = "error"
|
||||
_setup_log(f"失败：{exc}")
|
||||
finally:
|
||||
_semantic_setup["running"] = False
|
||||
|
||||
|
||||
@app.post("/api/semantic/setup")
|
||||
def semantic_setup() -> dict:
|
||||
"""启动语义环境一键配置（异步）。已运行中则返回当前状态。"""
|
||||
with _semantic_setup_lock:
|
||||
if not _semantic_setup["running"]:
|
||||
_semantic_setup.update({"running": True, "stage": "start",
|
||||
"log": [], "error": "", "done": False})
|
||||
threading.Thread(target=_run_semantic_setup, daemon=True).start()
|
||||
return _ok({"started": True})
|
||||
|
||||
|
||||
@app.get("/api/semantic/setup/status")
|
||||
def semantic_setup_status() -> dict:
|
||||
return _ok({** _semantic_setup, "embed": semantic.status(_cfg)})
|
||||
|
||||
|
||||
# ---------- 运行时设置（config.json） ----------
|
||||
def _mask(secret: str) -> str:
|
||||
s = (secret or "").strip()
|
||||
return ("••••" + s[-4:]) if s else ""
|
||||
|
||||
|
||||
class ConfigBody(BaseModel):
|
||||
host: str | None = None # 127.0.0.1 | 0.0.0.0
|
||||
port: int | None = None
|
||||
access_token: str | None = None # 空串 = 清除令牌
|
||||
llm_api_key: str | None = None
|
||||
llm_base_url: str | None = None
|
||||
llm_model: str | None = None
|
||||
embed_model_path: str | None = None # BGE-M3 本地权重目录，空串 = 清除
|
||||
embed_provider: str | None = None # local | api
|
||||
embed_api_key: str | None = None # 联网 Embedding API 密钥（如 DashScope）
|
||||
embed_api_base_url: str | None = None
|
||||
embed_api_model: str | None = None
|
||||
guided: bool | None = None # 初始设置引导完成标记
|
||||
mode: str | None = None # demo | live
|
||||
|
||||
|
||||
def _public_config() -> dict:
|
||||
saved = load_runtime_settings()
|
||||
ready = llm_ready(_cfg)
|
||||
running = {"host": str(_cfg.get("host")), "port": int(_cfg.get("port", 9008))}
|
||||
saved_hp = {"host": saved.get("host", running["host"]), "port": int(saved.get("port", running["port"]))}
|
||||
return {
|
||||
"running": running, # 当前进程实际监听
|
||||
"saved": saved_hp, # config.json 中保存的监听（重启后生效）
|
||||
"restart_required": saved_hp != running,
|
||||
"has_access_token": bool(str(_cfg.get("access_token") or "").strip()),
|
||||
"access_token_masked": _mask(str(_cfg.get("access_token") or "")),
|
||||
"llm": {
|
||||
"configured": ready,
|
||||
"api_key_masked": _mask(str(_cfg.get("llm_api_key") or "")),
|
||||
"base_url": str(_cfg.get("llm_base_url") or ""),
|
||||
"model": str(_cfg.get("llm_model") or ""),
|
||||
},
|
||||
"embed": semantic.status(_cfg),
|
||||
"mode": str(_cfg.get("mode", "demo")),
|
||||
"offline_fallback": not ready,
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/config")
|
||||
def get_config() -> dict:
|
||||
return _ok(_public_config())
|
||||
|
||||
|
||||
@app.post("/api/config")
|
||||
def save_config(body: ConfigBody) -> dict:
|
||||
"""保存运行时设置到 config.json；LLM/令牌热生效，监听地址与端口需重启。"""
|
||||
updates: dict = {}
|
||||
if body.host is not None:
|
||||
if body.host not in ("127.0.0.1", "0.0.0.0"):
|
||||
raise HTTPException(status_code=400, detail="host 仅支持 127.0.0.1 或 0.0.0.0")
|
||||
updates["host"] = body.host
|
||||
if body.port is not None:
|
||||
if not (1024 <= body.port <= 65535):
|
||||
raise HTTPException(status_code=400, detail="端口需在 1024-65535 之间")
|
||||
updates["port"] = body.port
|
||||
if body.mode is not None:
|
||||
if body.mode not in ("demo", "live"):
|
||||
raise HTTPException(status_code=400, detail="mode 仅支持 demo 或 live")
|
||||
updates["mode"] = body.mode
|
||||
if body.access_token is not None:
|
||||
updates["access_token"] = body.access_token.strip()
|
||||
if body.llm_api_key is not None:
|
||||
updates["llm_api_key"] = body.llm_api_key.strip()
|
||||
if body.llm_base_url is not None:
|
||||
updates["llm_base_url"] = body.llm_base_url.strip()
|
||||
if body.llm_model is not None:
|
||||
updates["llm_model"] = body.llm_model.strip()
|
||||
if body.embed_model_path is not None:
|
||||
updates["embed_model_path"] = body.embed_model_path.strip()
|
||||
if body.embed_provider is not None:
|
||||
if body.embed_provider not in ("local", "api"):
|
||||
raise HTTPException(status_code=400, detail="embed_provider 仅支持 local 或 api")
|
||||
updates["embed_provider"] = body.embed_provider
|
||||
if body.embed_api_key is not None:
|
||||
updates["embed_api_key"] = body.embed_api_key.strip()
|
||||
if body.embed_api_base_url is not None:
|
||||
updates["embed_api_base_url"] = body.embed_api_base_url.strip()
|
||||
if body.embed_api_model is not None:
|
||||
updates["embed_api_model"] = body.embed_api_model.strip()
|
||||
if body.guided is not None:
|
||||
updates["guided"] = bool(body.guided)
|
||||
|
||||
save_runtime_settings(updates)
|
||||
# 热生效：令牌、模式与 LLM/语义模型配置立即更新到运行配置（下次调用即读取）
|
||||
for k in ("access_token", "mode", "llm_api_key", "llm_base_url", "llm_model",
|
||||
"embed_model_path", "embed_provider", "embed_api_key",
|
||||
"embed_api_base_url", "embed_api_model", "guided"):
|
||||
if k in updates:
|
||||
if updates[k]:
|
||||
_cfg[k] = updates[k]
|
||||
else:
|
||||
_cfg.pop(k, None)
|
||||
logger.info("运行时设置已保存到 config.json: %s", sorted(updates))
|
||||
return _ok(_public_config(), message="设置已保存")
|
||||
|
||||
|
||||
@app.post("/api/config/restart")
|
||||
def restart_service() -> dict:
|
||||
"""延迟 1 秒后原地重启进程（os.execv），使监听地址/端口变更生效。"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
def _reexec() -> None:
|
||||
threading.Event().wait(1.0)
|
||||
logger.info("按用户请求重启服务……")
|
||||
os.execv(sys.executable, [sys.executable] + sys.argv)
|
||||
|
||||
threading.Thread(target=_reexec, daemon=True).start()
|
||||
return _ok({"restarting": True}, message="服务将在 1 秒后重启，请稍后刷新页面")
|
||||
|
||||
|
||||
@app.get("/api/graph")
|
||||
def graph() -> dict:
|
||||
"""返回节点与边（含时间区间）与演化事件，供前端渲染全景图谱。"""
|
||||
kg = _get_kg()
|
||||
nodes = [{"id": n, **d} for n, d in kg.g.nodes(data=True)]
|
||||
edges = [{"source": u, "target": v, **d} for u, v, d in kg.g.edges(data=True)]
|
||||
events = sorted((e.model_dump() for e in kg.events), key=lambda e: e["window"])
|
||||
return _ok({"nodes": nodes, "edges": edges, "events": events, "stats": kg.stats()})
|
||||
|
||||
|
||||
# ---------- JD 导入（异步任务） ----------
|
||||
def _run_task(task_id: str) -> None:
|
||||
task = _state["tasks"][task_id]
|
||||
try:
|
||||
task["status"] = "running"
|
||||
state_db.upsert_task(_db_path(), task_id, kind="jd_import",
|
||||
status="running", payload=task)
|
||||
stats = run_pipeline(_cfg)
|
||||
graph_path = resolve(_cfg, "graph_path")
|
||||
_state["kg"] = TemporalKG.load(graph_path)
|
||||
task.update(status="done", result=stats)
|
||||
except Exception as exc: # noqa: BLE001 - 任务失败必须可查询原因
|
||||
logger.exception("pipeline 任务失败")
|
||||
task.update(status="failed", error=str(exc))
|
||||
state_db.upsert_task(_db_path(), task_id, kind="jd_import",
|
||||
status=task["status"], payload=task)
|
||||
|
||||
|
||||
def _start_import_task(import_stats: dict | None = None) -> str:
|
||||
"""登记并启动异步 pipeline 重建任务，返回 task_id。"""
|
||||
task_id = uuid.uuid4().hex[:12]
|
||||
payload: dict = {"status": "pending"}
|
||||
if import_stats:
|
||||
payload["import"] = import_stats
|
||||
_state["tasks"][task_id] = payload
|
||||
state_db.upsert_task(_db_path(), task_id, kind="jd_import",
|
||||
status="pending", payload=payload)
|
||||
threading.Thread(target=_run_task, args=(task_id,), daemon=True).start()
|
||||
return task_id
|
||||
|
||||
|
||||
@app.post("/api/jd/import")
|
||||
async def import_jd(request: Request) -> dict:
|
||||
"""JD 导入：支持 multipart 文件上传（.jsonl/.csv/.txt）、JSON 文本粘贴，
|
||||
或不带内容直接触发全量重建（旧行为）。
|
||||
|
||||
合格记录写入 data/jd_raw/import_时间戳.jsonl（保留来源标记），
|
||||
坏行返回前端并写入 logs/import_bad_时间戳.jsonl；随后异步重建图谱。
|
||||
"""
|
||||
from app.ingest.loader import IMPORT_EXTS, IMPORT_MAX_BYTES, save_import
|
||||
text, filename = "", "paste.txt"
|
||||
if "multipart/form-data" in request.headers.get("content-type", ""):
|
||||
form = await request.form()
|
||||
upload = form.get("file")
|
||||
if upload is not None and hasattr(upload, "read"):
|
||||
filename = upload.filename or ""
|
||||
suffix = Path(filename).suffix.lower()
|
||||
if suffix not in IMPORT_EXTS:
|
||||
raise HTTPException(status_code=400,
|
||||
detail=f"不支持的格式 {suffix or '(无扩展名)'}，支持 {sorted(IMPORT_EXTS)}")
|
||||
content = await upload.read()
|
||||
if not content:
|
||||
raise HTTPException(status_code=400, detail="文件为空")
|
||||
if len(content) > IMPORT_MAX_BYTES:
|
||||
raise HTTPException(status_code=400, detail="文件超过 10MB 限制")
|
||||
try:
|
||||
text = content.decode("utf-8-sig")
|
||||
except UnicodeDecodeError as exc:
|
||||
raise HTTPException(status_code=400, detail="文件需为 UTF-8 编码") from exc
|
||||
elif form.get("text"):
|
||||
text = str(form["text"])
|
||||
else:
|
||||
raw = await request.body()
|
||||
if raw:
|
||||
try:
|
||||
payload = json.loads(raw)
|
||||
except json.JSONDecodeError as exc:
|
||||
raise HTTPException(status_code=400, detail=f"请求体不是合法 JSON：{exc}") from exc
|
||||
text = str(payload.get("text") or "")
|
||||
|
||||
import_stats: dict | None = None
|
||||
if text.strip():
|
||||
import_stats = save_import(text, filename, resolve(_cfg, "jd_raw_dir"), ROOT / "logs")
|
||||
if not import_stats["imported"] and not import_stats["bad_lines"]:
|
||||
raise HTTPException(status_code=400, detail="没有可导入的内容")
|
||||
task_id = _start_import_task(import_stats)
|
||||
data: dict = {"task_id": task_id}
|
||||
if import_stats:
|
||||
data.update(imported=import_stats["imported"], skipped=import_stats["skipped"],
|
||||
bad_lines=import_stats["bad_lines"],
|
||||
file=import_stats["file"], bad_log=import_stats["bad_log"])
|
||||
return _ok(data, message="导入任务已启动")
|
||||
|
||||
|
||||
@app.get("/api/tasks/{task_id}")
|
||||
def task_status(task_id: str) -> dict:
|
||||
task = _state["tasks"].get(task_id)
|
||||
if task is None:
|
||||
row = state_db.get_task(_db_path(), task_id) # 重启前的任务从状态库回查
|
||||
task = row["payload"] if row else None
|
||||
if task is None:
|
||||
raise HTTPException(status_code=404, detail="任务不存在")
|
||||
return _ok(task)
|
||||
|
||||
|
||||
# ---------- 岗位 ----------
|
||||
@app.get("/api/jobs")
|
||||
def list_jobs(category: str = "", skill: str = "", keyword: str = "") -> dict:
|
||||
kg = _get_kg()
|
||||
jobs = list(kg.jobs.values())
|
||||
if category:
|
||||
jobs = [j for j in jobs if j.category == category]
|
||||
if skill:
|
||||
jobs = [j for j in jobs if skill in j.required_skills or skill in j.preferred_skills]
|
||||
if keyword:
|
||||
jobs = [j for j in jobs if keyword in j.title]
|
||||
return _ok([j.model_dump() for j in jobs])
|
||||
|
||||
|
||||
@app.get("/api/jobs/{job_id}")
|
||||
def job_detail(job_id: str) -> dict:
|
||||
kg = _get_kg()
|
||||
job = kg.jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="岗位不存在")
|
||||
return _ok({"definition": job.model_dump(),
|
||||
"snapshot": kg.latest_snapshot(job_id)})
|
||||
|
||||
|
||||
@app.get("/api/jobs/{job_id}/evolution")
|
||||
def job_evolution(job_id: str) -> dict:
|
||||
kg = _get_kg()
|
||||
if job_id not in kg.jobs:
|
||||
raise HTTPException(status_code=404, detail="岗位不存在")
|
||||
events = [e.model_dump() for e in kg.events_of(job_id)]
|
||||
events.sort(key=lambda e: e["window"])
|
||||
return _ok({"job_id": job_id, "events": events})
|
||||
|
||||
|
||||
@app.get("/api/jobs/{job_id}/snapshot")
|
||||
def job_snapshot(job_id: str, at: str = "") -> dict:
|
||||
"""任意时间点能力画像切片（用于演化页时点对比）。"""
|
||||
kg = _get_kg()
|
||||
if job_id not in kg.jobs:
|
||||
raise HTTPException(status_code=404, detail="岗位不存在")
|
||||
return _ok(kg.snapshot(job_id, at or "9999"))
|
||||
|
||||
|
||||
# ---------- 新岗位发现与审核 ----------
|
||||
@app.post("/api/jobs/discover")
|
||||
def discover() -> dict:
|
||||
"""对当前原始数据重新跑新岗位发现，结果进待审核分区。"""
|
||||
jd_dir = resolve(_cfg, "jd_raw_dir")
|
||||
if not jd_dir.exists():
|
||||
raise HTTPException(status_code=400, detail="原始数据目录不存在")
|
||||
from app.extract.offline import OfflineExtractor
|
||||
records, _ = load_jd_dir(jd_dir)
|
||||
extractor = OfflineExtractor(load_lexicon(resolve(_cfg, "lexicon_path")))
|
||||
extractions = {r.jid: extractor.extract(r) for r in records if not r.is_noise}
|
||||
known = {j.title for j in (_state["kg"].jobs.values() if _state["kg"] else [])}
|
||||
candidates = discover_new_jobs(records, extractions, known)
|
||||
save_pending(candidates, _db_path())
|
||||
return _ok({"candidates": [c.model_dump() for c in candidates]})
|
||||
|
||||
|
||||
class ReviewBody(BaseModel):
|
||||
candidate_id: str
|
||||
action: str # accept | modify | reject
|
||||
operator: str
|
||||
reason: str = ""
|
||||
modify: dict | None = None
|
||||
|
||||
|
||||
@app.post("/api/review")
|
||||
def review(body: ReviewBody) -> dict:
|
||||
"""人机协同审核：通过后写入正式图谱（真实区间/来源/场景行业边/演化事件），
|
||||
并写入覆写层保证跨重建持久化，全程留痕。"""
|
||||
from app.services.overrides import apply_accepted_job, record_job_accept
|
||||
kg = _get_kg()
|
||||
try:
|
||||
accepted, record = apply_review(
|
||||
_db_path(), resolve(_cfg, "audit_path"),
|
||||
body.candidate_id, body.action, body.operator, body.reason, body.modify)
|
||||
except KeyError as exc:
|
||||
raise HTTPException(status_code=404, detail=str(exc)) from exc
|
||||
if accepted is not None:
|
||||
from app.services.overrides import normalize_job_id
|
||||
candidate_id = accepted.job_id
|
||||
accepted.job_id = normalize_job_id(accepted) # cand_X → job_<岗位名>，全图唯一节点
|
||||
apply_accepted_job(kg, accepted) # 真实窗口/来源入库 + 场景行业边 + 新增事件
|
||||
record_job_accept(_db_path(), accepted, candidate_id=candidate_id) # 覆写层持久化，重建后自动重放
|
||||
kg.save(resolve(_cfg, "graph_path"))
|
||||
return _ok({"review": record.model_dump(),
|
||||
"accepted": accepted.model_dump() if accepted else None})
|
||||
|
||||
|
||||
@app.get("/api/review/pending")
|
||||
def pending_list() -> dict:
|
||||
return _ok([c.model_dump() for c in load_pending(_db_path())])
|
||||
|
||||
|
||||
# ---------- 图谱节点编辑（人机协同，留痕） ----------
|
||||
ALLOWED_NODE_FIELDS = {"title", "category", "level", "importance", "weight"}
|
||||
|
||||
|
||||
class NodeUpdateBody(BaseModel):
|
||||
node_id: str
|
||||
updates: dict # 仅允许 ALLOWED_NODE_FIELDS 中的键
|
||||
operator: str
|
||||
reason: str = ""
|
||||
|
||||
|
||||
def _append_audit(record: dict) -> None:
|
||||
"""向审核留痕追加一条记录（与审核流同一 SQLite 状态库 audit_log 表）。"""
|
||||
state_db.append_audit(_db_path(), record)
|
||||
|
||||
|
||||
@app.post("/api/graph/node/update")
|
||||
def node_update(body: NodeUpdateBody) -> dict:
|
||||
"""编辑节点展示属性（名称/类别/级别/重要性/权重），写图谱并留痕。
|
||||
|
||||
只改属性不改结构：节点 id、边与演化历史不受影响。
|
||||
Job 节点同步 kg.jobs 的 JobDefinition（title/category/级别），
|
||||
编辑写入覆写层（job_overrides 表），pipeline 重建后自动重放。
|
||||
"""
|
||||
from app.services.overrides import apply_node_updates, record_node_update
|
||||
kg = _get_kg()
|
||||
updates = {k: v for k, v in body.updates.items() if k in ALLOWED_NODE_FIELDS}
|
||||
if body.node_id not in kg.g:
|
||||
raise HTTPException(status_code=404, detail="节点不存在")
|
||||
if not updates:
|
||||
raise HTTPException(status_code=400, detail="没有可更新的字段")
|
||||
attrs = kg.g.nodes[body.node_id]
|
||||
before = {k: attrs.get(k) for k in updates}
|
||||
apply_node_updates(kg, body.node_id, updates)
|
||||
record_node_update(_db_path(), body.node_id, updates)
|
||||
kg.save(resolve(_cfg, "graph_path"))
|
||||
record = {"kind": "node_update", "node_id": body.node_id,
|
||||
"operator": body.operator, "reason": body.reason,
|
||||
"before": before, "after": updates, "time": now_str()}
|
||||
_append_audit(record)
|
||||
return _ok({"node": {"id": body.node_id, **attrs}, "audit": record},
|
||||
message="节点已更新")
|
||||
|
||||
|
||||
@app.get("/api/audit")
|
||||
def audit_list(node_id: str = "", limit: int = 50) -> dict:
|
||||
"""读取审核/编辑留痕（新→旧排序），可按节点过滤。返回结构与旧 audit.jsonl 行一致。"""
|
||||
records = state_db.get_audit(_db_path(), node_id=node_id or None,
|
||||
limit=max(1, min(limit, 200)))
|
||||
return _ok(records)
|
||||
|
||||
|
||||
# ---------- JD 原文回查（证据来源弹窗） ----------
|
||||
@app.get("/api/jd")
|
||||
def jd_by_source(sid: str) -> dict:
|
||||
"""按 source_id（形如 batch_2019.jsonl#17）回查 JD 原文片段。"""
|
||||
if "#" not in sid:
|
||||
raise HTTPException(status_code=400, detail="source_id 格式应为 文件名#行号")
|
||||
fname, _, lineno_s = sid.partition("#")
|
||||
if Path(fname).name != fname or not fname.endswith(".jsonl"):
|
||||
raise HTTPException(status_code=400, detail="非法文件名")
|
||||
try:
|
||||
lineno = int(lineno_s)
|
||||
except ValueError as exc:
|
||||
raise HTTPException(status_code=400, detail="非法行号") from exc
|
||||
path = resolve(_cfg, "jd_raw_dir") / fname
|
||||
if not path.exists():
|
||||
raise HTTPException(status_code=404, detail="数据文件不存在")
|
||||
lines = path.read_text(encoding="utf-8").splitlines()
|
||||
if not (1 <= lineno <= len(lines)):
|
||||
raise HTTPException(status_code=404, detail="记录不存在")
|
||||
try:
|
||||
rec = JDRecord(**json.loads(lines[lineno - 1]))
|
||||
except (json.JSONDecodeError, ValidationError, TypeError) as exc:
|
||||
raise HTTPException(status_code=404, detail=f"记录解析失败：{exc}") from exc
|
||||
rec.source_id = sid
|
||||
return _ok(rec.model_dump())
|
||||
|
||||
|
||||
# ---------- 简历 / 匹配 / 学习路径 ----------
|
||||
class ResumeBody(BaseModel):
|
||||
text: str
|
||||
|
||||
|
||||
@app.post("/api/resume/parse")
|
||||
def resume_parse(body: ResumeBody) -> dict:
|
||||
matcher = SkillMatcher(load_lexicon(resolve(_cfg, "lexicon_path")))
|
||||
profile = parse_resume(body.text, matcher)
|
||||
return _ok(profile.model_dump())
|
||||
|
||||
|
||||
_RESUME_EXTS = {".txt", ".md", ".docx", ".pdf"}
|
||||
_RESUME_MAX_BYTES = 5 * 1024 * 1024 # 5MB
|
||||
|
||||
|
||||
@app.post("/api/resume/parse/file")
|
||||
async def resume_parse_file(file: UploadFile) -> dict:
|
||||
"""上传简历文件（.txt/.md/.docx/.pdf）并解析：脱敏 + 技能抽取归一化。
|
||||
|
||||
文件名随机化落临时目录，解析完成即删除；临时文件不进入日志。
|
||||
"""
|
||||
suffix = Path(file.filename or "").suffix.lower()
|
||||
if suffix not in _RESUME_EXTS:
|
||||
raise HTTPException(status_code=400,
|
||||
detail=f"不支持的格式 {suffix or '(无扩展名)'}，支持 {sorted(_RESUME_EXTS)}")
|
||||
content = await file.read()
|
||||
if not content:
|
||||
raise HTTPException(status_code=400, detail="文件为空")
|
||||
if len(content) > _RESUME_MAX_BYTES:
|
||||
raise HTTPException(status_code=400, detail="文件超过 5MB 限制")
|
||||
tmp_dir = ROOT / "data" / "tmp"
|
||||
tmp_dir.mkdir(parents=True, exist_ok=True)
|
||||
tmp_path = tmp_dir / f"resume_{uuid.uuid4().hex[:12]}{suffix}"
|
||||
try:
|
||||
tmp_path.write_bytes(content)
|
||||
matcher = SkillMatcher(load_lexicon(resolve(_cfg, "lexicon_path")))
|
||||
profile = parse_resume_file(tmp_path, matcher)
|
||||
except RuntimeError as exc: # 如 pdf 依赖缺失、文件损坏等
|
||||
raise HTTPException(status_code=422, detail=str(exc)) from exc
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=422, detail=f"简历文件解析失败：{exc}") from exc
|
||||
finally:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
logger.info("简历文件解析完成: 格式=%s, 技能=%d 项", suffix, len(profile.skills))
|
||||
return _ok(profile.model_dump())
|
||||
|
||||
|
||||
class MatchBody(BaseModel):
|
||||
job_id: str
|
||||
text: str = "" # 简历原文（与 skills 二选一）
|
||||
skills: list[str] = []
|
||||
years: float = 0.0
|
||||
scenarios: list[str] = []
|
||||
|
||||
|
||||
class ResumeExportBody(BaseModel):
|
||||
profile: dict # ResumeProfile 序列化结果
|
||||
match: dict | None = None # 可选：匹配结果，用于关键词高亮与诊断摘要
|
||||
|
||||
|
||||
_BATCH_MAX_FILES = 50
|
||||
|
||||
|
||||
@app.post("/api/match/batch")
|
||||
async def match_batch(job_id: str = Form(...), files: list[UploadFile] = File(...)) -> dict:
|
||||
"""批量人岗匹配：多份简历（.txt/.md/.docx/.pdf，≤50 份）对同一岗位打分排序。
|
||||
|
||||
每份简历独立解析（含脱敏）与匹配，单份失败不影响整批（记 error 返回）。
|
||||
"""
|
||||
kg = _get_kg()
|
||||
job = kg.jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="岗位不存在")
|
||||
if not files:
|
||||
raise HTTPException(status_code=400, detail="未收到简历文件")
|
||||
if len(files) > _BATCH_MAX_FILES:
|
||||
raise HTTPException(status_code=400, detail=f"单批最多 {_BATCH_MAX_FILES} 份简历")
|
||||
|
||||
engine = _engine()
|
||||
matcher = SkillMatcher(load_lexicon(resolve(_cfg, "lexicon_path")))
|
||||
tmp_dir = ROOT / "data" / "tmp"
|
||||
tmp_dir.mkdir(parents=True, exist_ok=True)
|
||||
results: list[dict] = []
|
||||
for f in files:
|
||||
name = f.filename or "未命名"
|
||||
suffix = Path(name).suffix.lower()
|
||||
if suffix not in _RESUME_EXTS:
|
||||
results.append({"filename": name, "error": f"不支持的格式 {suffix or '(无扩展名)'}"})
|
||||
continue
|
||||
content = await f.read()
|
||||
if not content or len(content) > _RESUME_MAX_BYTES:
|
||||
results.append({"filename": name, "error": "文件为空或超过 5MB"})
|
||||
continue
|
||||
tmp_path = tmp_dir / f"batch_{uuid.uuid4().hex[:12]}{suffix}"
|
||||
try:
|
||||
tmp_path.write_bytes(content)
|
||||
profile = parse_resume_file(tmp_path, matcher)
|
||||
m = engine.match(profile, job)
|
||||
results.append({
|
||||
"filename": name,
|
||||
"name": profile.name or "（未识别）",
|
||||
"years": profile.years,
|
||||
"skills_count": len(profile.skills),
|
||||
"total_score": m.total_score,
|
||||
"dimension_scores": m.dimension_scores,
|
||||
"matched": m.matched_required + m.matched_preferred,
|
||||
"missing": m.missing_required + m.missing_preferred,
|
||||
"is_match": m.is_match,
|
||||
})
|
||||
except Exception as exc: # noqa: BLE001 - 单份失败不拖垮整批
|
||||
logger.warning("批量匹配：%s 解析失败: %s", name, exc)
|
||||
results.append({"filename": name, "error": f"解析失败：{exc}"})
|
||||
finally:
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
ok = [r for r in results if "error" not in r]
|
||||
ok.sort(key=lambda r: r["total_score"], reverse=True)
|
||||
logger.info("批量匹配完成: 岗位=%s, 成功 %d/%d", job.title, len(ok), len(files))
|
||||
return _ok({"job_id": job_id, "job_title": job.title,
|
||||
"total": len(results), "ok": len(ok), "results": ok + [r for r in results if "error" in r]})
|
||||
|
||||
|
||||
@app.post("/api/resume/export")
|
||||
def resume_export(body: ResumeExportBody) -> Response:
|
||||
"""把脱敏简历画像导出为规范 DOCX（照片+基本信息+分节+关键词高亮）。"""
|
||||
from app.match.resume_docx import build_resume_docx
|
||||
try:
|
||||
profile = ResumeProfile(**body.profile)
|
||||
except ValidationError as exc:
|
||||
raise HTTPException(status_code=400, detail=f"简历画像格式错误：{exc}") from exc
|
||||
data = build_resume_docx(profile, body.match)
|
||||
fname = quote("简历_脱敏结构化.docx")
|
||||
return Response(
|
||||
content=data,
|
||||
media_type="application/vnd.openxmlformats-officedocument.wordprocessingml.document",
|
||||
headers={"Content-Disposition": f"attachment; filename*=utf-8''{fname}"})
|
||||
|
||||
|
||||
def _profile_of(body: MatchBody) -> ResumeProfile:
|
||||
matcher = SkillMatcher(load_lexicon(resolve(_cfg, "lexicon_path")))
|
||||
if body.text:
|
||||
return parse_resume(body.text, matcher)
|
||||
# 手动录入的技能清单：三级归一化（大小写/别名→包含容错）；
|
||||
# 仍未命中的，若语义通道就绪则做 BGE-M3 语义召回兜底（可选增强）
|
||||
lexicon = load_lexicon(resolve(_cfg, "lexicon_path"))
|
||||
skills: list[str] = []
|
||||
for s in body.skills:
|
||||
if not s or not s.strip():
|
||||
continue
|
||||
norm = matcher.normalize(s)
|
||||
if norm == s.strip():
|
||||
sem = semantic.most_similar(norm, _cfg, lexicon)
|
||||
if sem:
|
||||
logger.info("语义召回: %s → %s (%.3f)", norm, sem[0], sem[1])
|
||||
norm = sem[0]
|
||||
skills.append(norm)
|
||||
return ResumeProfile(skills=skills, years=body.years, scenarios=body.scenarios)
|
||||
|
||||
|
||||
def _engine() -> MatchEngine:
|
||||
return MatchEngine(
|
||||
lexicon=load_lexicon(resolve(_cfg, "lexicon_path")),
|
||||
weights={
|
||||
"required": float(_cfg.get("match_weight_required", 0.5)),
|
||||
"preferred": float(_cfg.get("match_weight_preferred", 0.15)),
|
||||
"experience": float(_cfg.get("match_weight_experience", 0.2)),
|
||||
"scenario": float(_cfg.get("match_weight_scenario", 0.15)),
|
||||
},
|
||||
threshold=float(_cfg.get("match_threshold", 60.0)),
|
||||
)
|
||||
|
||||
|
||||
@app.post("/api/match")
|
||||
def match(body: MatchBody) -> dict:
|
||||
kg = _get_kg()
|
||||
job = kg.jobs.get(body.job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="岗位不存在")
|
||||
result = _engine().match(_profile_of(body), job)
|
||||
return _ok(result.model_dump())
|
||||
|
||||
|
||||
class LearnBody(BaseModel):
|
||||
job_id: str
|
||||
known_skills: list[str] = []
|
||||
|
||||
|
||||
@app.post("/api/learning-path")
|
||||
def learning_path(body: LearnBody) -> dict:
|
||||
kg = _get_kg()
|
||||
job = kg.jobs.get(body.job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="岗位不存在")
|
||||
profile = ResumeProfile(skills=body.known_skills)
|
||||
result = _engine().match(profile, job)
|
||||
return _ok({"job_id": job.job_id, "title": job.title,
|
||||
"learning_path": result.learning_path,
|
||||
"missing_required": result.missing_required})
|
||||
|
||||
|
||||
# ---------- webui 静态托管 ----------
|
||||
WEBUI = ROOT / "webui"
|
||||
|
||||
|
||||
@app.get("/", include_in_schema=False)
|
||||
def index():
|
||||
return FileResponse(WEBUI / "index.html")
|
||||
|
||||
|
||||
app.mount("/static", StaticFiles(directory=WEBUI), name="static")
|
||||
@@ -0,0 +1,62 @@
|
||||
"""应用装配层：日志、图谱初始化、服务启动。
|
||||
|
||||
main.py 只做一件事——调用本模块的 run()。分层关系：
|
||||
|
||||
main.py 唯一入口（装配）
|
||||
└─ app.bootstrap 启动流程（本模块）
|
||||
├─ app.config 配置加载（config.yaml + config.json）
|
||||
├─ app.services 端到端 pipeline 编排
|
||||
└─ app.api.routes FastAPI 接口层（uvicorn 托管）
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import webbrowser
|
||||
|
||||
import uvicorn
|
||||
|
||||
from app.config import load_config, resolve
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def setup_logging() -> None:
|
||||
"""全局日志：时间 | 级别 | 模块 | 消息。"""
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
||||
)
|
||||
|
||||
|
||||
def ensure_graph(cfg: dict) -> None:
|
||||
"""图谱不存在时先跑一遍端到端 pipeline 构建。"""
|
||||
graph_path = resolve(cfg, "graph_path")
|
||||
if graph_path.exists():
|
||||
logger.info("检测到已有图谱 %s，直接加载", graph_path)
|
||||
return
|
||||
logger.info("图谱不存在，先执行端到端 pipeline 构建……")
|
||||
from app.services.pipeline import run_pipeline
|
||||
stats = run_pipeline(cfg)
|
||||
logger.info("图谱构建完成：%s", stats["stages"].get("graph"))
|
||||
|
||||
|
||||
def _open_browser_later(url: str, delay: float = 1.5) -> None:
|
||||
time.sleep(delay)
|
||||
try:
|
||||
webbrowser.open(url)
|
||||
except Exception as exc: # 浏览器打不开不影响服务
|
||||
logger.warning("自动打开浏览器失败：%s，请手动访问 %s", exc, url)
|
||||
|
||||
|
||||
def run() -> None:
|
||||
"""启动流程：日志 → 配置 → 图谱 → 服务 → 自动开浏览器。"""
|
||||
setup_logging()
|
||||
cfg = load_config()
|
||||
ensure_graph(cfg)
|
||||
host, port = str(cfg["host"]), int(cfg["port"])
|
||||
url = f"http://{host}:{port}"
|
||||
logger.info("CareerNexus 职联启动中：%s", url)
|
||||
threading.Thread(target=_open_browser_later, args=(url,), daemon=True).start()
|
||||
uvicorn.run("app.api.routes:app", host=host, port=port, log_level="info")
|
||||
@@ -0,0 +1,33 @@
|
||||
"""时间衰减：招聘数据存在时滞，越旧的 JD 对当前能力刻画的贡献越低。
|
||||
|
||||
权重公式：w = exp(-λ · age)，age 为发布日期距参考日期的年数。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from datetime import date
|
||||
|
||||
from app.models import JDRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def decay_weight(publish_date: date, ref_date: date, lam: float = 0.35) -> float:
|
||||
"""计算单条 JD 的时间衰减权重。"""
|
||||
age_years = max(0.0, (ref_date - publish_date).days / 365.25)
|
||||
return round(math.exp(-lam * age_years), 4)
|
||||
|
||||
|
||||
def apply_decay(
|
||||
records: list[JDRecord],
|
||||
ref_date: date | None = None,
|
||||
lam: float = 0.35,
|
||||
) -> list[JDRecord]:
|
||||
"""就地更新每条记录的 weight（与已有降权相乘，不会覆盖抄袭降权）。"""
|
||||
if ref_date is None:
|
||||
ref_date = max((r.publish_date for r in records), default=date.today())
|
||||
for r in records:
|
||||
r.weight = round(r.weight * decay_weight(r.publish_date, ref_date, lam), 4)
|
||||
logger.info("时间衰减完成：λ=%.3f，参考日期 %s", lam, ref_date.isoformat())
|
||||
return records
|
||||
@@ -0,0 +1,119 @@
|
||||
"""近重复（抄袭）JD 检测：纯 Python MinHash（64 哈希、5-gram shingles）+ LSH banding。
|
||||
|
||||
招聘网站之间存在大量"同文不同公司"的抄袭 JD，直接入图会导致技能频次虚高。
|
||||
本模块把近重复 JD 聚簇：簇内保留最早来源，其余标记 is_duplicate 并降权。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import random
|
||||
|
||||
from app.models import JDRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
NUM_HASHES = 128 # MinHash 哈希函数个数（>=64，越高估计越稳）
|
||||
SHINGLE_N = 5 # 字符 5-gram shingles
|
||||
LSH_BANDS = 16 # 16 bands × 8 rows（严格分带，抑制模板化文本误配）
|
||||
_PRIME = (1 << 61) - 1 # 梅森素数，用于 (a*x+b) mod p 哈希族
|
||||
_MAXINT = (1 << 32) - 1
|
||||
|
||||
|
||||
def shingles(text: str, n: int = SHINGLE_N) -> set[int]:
|
||||
"""把文本切成字符 n-gram shingles，并映射为 32 位整数集合。"""
|
||||
text = "".join(text.split()) # 去掉空白，抵抗排版差异
|
||||
if len(text) < n:
|
||||
text = text + " " * (n - len(text))
|
||||
out = set()
|
||||
for i in range(len(text) - n + 1):
|
||||
h = hashlib.md5(text[i:i + n].encode("utf-8")).digest()
|
||||
out.add(int.from_bytes(h[:4], "little"))
|
||||
return out
|
||||
|
||||
|
||||
class MinHash:
|
||||
"""(a*x+b) mod p 置换哈希族实现的 MinHash 签名。"""
|
||||
|
||||
def __init__(self, num_hashes: int = NUM_HASHES, seed: int = 42):
|
||||
self.num_hashes = num_hashes
|
||||
# 确定性大随机系数，保证 (a*x+b) mod p 是真置换哈希且可复现
|
||||
rng = random.Random(seed)
|
||||
self._a = [rng.randrange(1, _PRIME) for _ in range(num_hashes)]
|
||||
self._b = [rng.randrange(0, _PRIME) for _ in range(num_hashes)]
|
||||
|
||||
def signature(self, shingle_set: set[int]) -> list[int]:
|
||||
return [min((a * x + b) % _PRIME for x in shingle_set)
|
||||
for a, b in zip(self._a, self._b)]
|
||||
|
||||
def estimate(self, s1: list[int], s2: list[int]) -> float:
|
||||
return sum(1 for x, y in zip(s1, s2) if x == y) / self.num_hashes
|
||||
|
||||
|
||||
def _lsh_candidates(sig: list[int], bands: int, buckets: dict) -> None:
|
||||
"""把签名按 band 切段放入哈希桶，同桶者成为候选对。"""
|
||||
rows = len(sig) // bands
|
||||
for b in range(bands):
|
||||
key = (b, tuple(sig[b * rows:(b + 1) * rows]))
|
||||
buckets.setdefault(key, []).append(sig)
|
||||
|
||||
|
||||
def find_near_duplicates(
|
||||
records: list[JDRecord],
|
||||
threshold: float = 0.85,
|
||||
num_hashes: int = NUM_HASHES,
|
||||
bands: int = LSH_BANDS,
|
||||
) -> list[list[str]]:
|
||||
"""对全部 JD 做近重复聚簇，返回簇列表（每簇为 jid 列表，保留者排第一）。
|
||||
|
||||
簇内按 (publish_date, jid) 排序，最早来源为保留者；其余记录被标记
|
||||
``is_duplicate=True``、``dup_of=<保留者 jid>`` 并降权 0.2。
|
||||
"""
|
||||
mh = MinHash(num_hashes)
|
||||
sigs: dict[str, list[int]] = {}
|
||||
for r in records:
|
||||
sigs[r.jid] = mh.signature(shingles(r.text))
|
||||
|
||||
buckets: dict = {}
|
||||
for sig in sigs.values():
|
||||
_lsh_candidates(sig, bands, buckets)
|
||||
|
||||
# 并查集聚簇：候选对需通过完整签名相似度复核
|
||||
parent = {r.jid: r.jid for r in records}
|
||||
|
||||
def find(x: str) -> str:
|
||||
while parent[x] != x:
|
||||
parent[x] = parent[parent[x]]
|
||||
x = parent[x]
|
||||
return x
|
||||
|
||||
def union(x: str, y: str) -> None:
|
||||
parent[find(x)] = find(y)
|
||||
|
||||
jid_of_sig = {id(sig): jid for jid, sig in sigs.items()}
|
||||
for members in buckets.values():
|
||||
for i in range(len(members)):
|
||||
for j in range(i + 1, len(members)):
|
||||
if mh.estimate(members[i], members[j]) >= threshold:
|
||||
union(jid_of_sig[id(members[i])], jid_of_sig[id(members[j])])
|
||||
|
||||
groups: dict[str, list[JDRecord]] = {}
|
||||
by_jid = {r.jid: r for r in records}
|
||||
for r in records:
|
||||
groups.setdefault(find(r.jid), []).append(r)
|
||||
|
||||
clusters: list[list[str]] = []
|
||||
for members in groups.values():
|
||||
if len(members) < 2:
|
||||
continue
|
||||
members.sort(key=lambda r: (r.publish_date, r.jid))
|
||||
keeper = members[0]
|
||||
cluster = [keeper.jid]
|
||||
for dup in members[1:]:
|
||||
dup.is_duplicate = True
|
||||
dup.dup_of = keeper.jid
|
||||
dup.weight = round(dup.weight * 0.2, 4) # 抄袭来源降权
|
||||
cluster.append(dup.jid)
|
||||
clusters.append(cluster)
|
||||
logger.info("检出近重复簇：保留 %s，降权 %d 条", keeper.jid, len(members) - 1)
|
||||
return clusters
|
||||
@@ -0,0 +1,47 @@
|
||||
"""噪声过滤：识别广告尾注、无关条款等非岗位描述内容。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
from app.models import JDRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 噪声特征：广告导流、扫码领资料、与岗位无关的商业条款等
|
||||
NOISE_PATTERNS = [
|
||||
r"扫码|点击关注|关注公众号|领取.{0,6}(资料|福利|红包)",
|
||||
r"【广告】|推广信息|商务合作请加",
|
||||
r"长期回收|代办|加盟代理|刷单",
|
||||
r"内部推荐奖.{0,10}扫码",
|
||||
]
|
||||
_NOISE_RE = re.compile("|".join(NOISE_PATTERNS))
|
||||
|
||||
|
||||
def is_noise(text: str) -> bool:
|
||||
"""判断文本是否含有噪声内容。"""
|
||||
return bool(_NOISE_RE.search(text))
|
||||
|
||||
|
||||
def strip_noise_lines(text: str) -> str:
|
||||
"""逐行剔除噪声行，保留正文（用于噪声行与正文混排的情况）。"""
|
||||
kept = [ln for ln in text.splitlines() if ln.strip() and not is_noise(ln)]
|
||||
return "\n".join(kept)
|
||||
|
||||
|
||||
def filter_noise(records: list[JDRecord]) -> list[JDRecord]:
|
||||
"""标记噪声记录：整条以噪声为主的记录 is_noise=True 并降权；混排的只剔除噪声行。"""
|
||||
marked = 0
|
||||
for r in records:
|
||||
lines = [ln for ln in r.text.splitlines() if ln.strip()]
|
||||
noisy = [ln for ln in lines if is_noise(ln)]
|
||||
if not noisy:
|
||||
continue
|
||||
if len(noisy) >= max(1, len(lines) // 2):
|
||||
r.is_noise = True
|
||||
r.weight = round(r.weight * 0.1, 4)
|
||||
marked += 1
|
||||
else:
|
||||
r.text = strip_noise_lines(r.text)
|
||||
logger.info("噪声过滤：%d 条记录被标记为噪声", marked)
|
||||
return records
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
"""极简配置加载：解析平铺的 ``key: value`` 格式 config.yaml，避免额外依赖。
|
||||
|
||||
运行时设置（监听地址/端口/访问令牌/LLM 密钥）保存在 config.json，
|
||||
由设置接口写入、启动时合并读取，优先级：config.json > config.yaml > DEFAULTS。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
CONFIG_JSON = ROOT / "config.json" # 运行时设置文件（测试可 monkeypatch 此路径）
|
||||
|
||||
# 允许通过设置面板/config.json 覆盖的键白名单
|
||||
RUNTIME_KEYS = {
|
||||
"mode", "host", "port", "access_token",
|
||||
"llm_api_key", "llm_base_url", "llm_model", "llm_timeout",
|
||||
"embed_model_path", "embed_provider", "embed_api_key",
|
||||
"embed_api_base_url", "embed_api_model",
|
||||
"guided",
|
||||
}
|
||||
|
||||
DEFAULTS: dict = {
|
||||
"mode": "demo", # demo | live
|
||||
"host": "127.0.0.1",
|
||||
"port": 9008,
|
||||
"data_dir": "data",
|
||||
"jd_raw_dir": "data/jd_raw",
|
||||
"jd_extra_dirs": "", # 附加 JD 数据源目录（逗号分隔，可切换到真实数据集）
|
||||
"graph_path": "data/graph.json",
|
||||
"lexicon_path": "data/skills_lexicon.json",
|
||||
"pending_path": "data/review/pending.json", # 旧待审核文件（仅作 SQLite 一次性迁移种子）
|
||||
"audit_path": "data/review/audit.jsonl", # 旧留痕文件（仅作 SQLite 一次性迁移种子）
|
||||
"state_db_path": "data/careernexus.db", # SQLite 应用状态库（审核留痕/候选/任务）
|
||||
"decay_lambda": 0.35, # 时间衰减系数 w = exp(-λ·age)
|
||||
"match_weight_required": 0.50, # 人岗匹配四维权重
|
||||
"match_weight_preferred": 0.15,
|
||||
"match_weight_experience": 0.20,
|
||||
"match_weight_scenario": 0.15,
|
||||
"match_threshold": 60.0, # 总分 >= 阈值视为匹配
|
||||
"llm_api_key_env": "SPARK_API_KEY", # 密钥只从环境变量读取
|
||||
"llm_base_url": "https://spark-api-open.xf-yun.com/v1",
|
||||
"llm_model": "4.0Ultra",
|
||||
"llm_timeout": 20,
|
||||
}
|
||||
|
||||
|
||||
def _coerce(value: str):
|
||||
v = value.strip()
|
||||
# 行内注释：值中第一个 " #" 之后的内容丢弃（URL 等含 # 的值不受影响）
|
||||
if " #" in v:
|
||||
v = v.split(" #", 1)[0].strip()
|
||||
if v.lower() in ("true", "false"):
|
||||
return v.lower() == "true"
|
||||
try:
|
||||
return int(v)
|
||||
except ValueError:
|
||||
pass
|
||||
try:
|
||||
return float(v)
|
||||
except ValueError:
|
||||
pass
|
||||
return v
|
||||
|
||||
|
||||
def load_config(path: str | Path | None = None) -> dict:
|
||||
"""加载配置：DEFAULTS ← config.yaml ← config.json（运行时设置，优先级最高）。"""
|
||||
cfg = dict(DEFAULTS)
|
||||
p = Path(path) if path else ROOT / "config.yaml"
|
||||
if p.exists():
|
||||
for line in p.read_text(encoding="utf-8").splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or ":" not in line:
|
||||
continue
|
||||
key, _, value = line.partition(":")
|
||||
cfg[key.strip()] = _coerce(value)
|
||||
cfg.update(load_runtime_settings())
|
||||
return cfg
|
||||
|
||||
|
||||
def load_runtime_settings() -> dict:
|
||||
"""读取 config.json 中的白名单键；文件不存在或损坏时返回空 dict。"""
|
||||
try:
|
||||
if CONFIG_JSON.exists():
|
||||
data = json.loads(CONFIG_JSON.read_text(encoding="utf-8"))
|
||||
return {k: v for k, v in data.items() if k in RUNTIME_KEYS}
|
||||
except (json.JSONDecodeError, OSError):
|
||||
pass
|
||||
return {}
|
||||
|
||||
|
||||
def save_runtime_settings(updates: dict) -> dict:
|
||||
"""把白名单内的设置合并写入 config.json，返回保存后的完整运行时设置。"""
|
||||
current = load_runtime_settings()
|
||||
for k, v in updates.items():
|
||||
if k not in RUNTIME_KEYS:
|
||||
continue
|
||||
if v is None or (isinstance(v, str) and not v.strip()):
|
||||
current.pop(k, None) # 空值视为清除该设置
|
||||
else:
|
||||
current[k] = v
|
||||
CONFIG_JSON.write_text(json.dumps(current, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8")
|
||||
return current
|
||||
|
||||
|
||||
def resolve(cfg: dict, key: str) -> Path:
|
||||
"""把配置中的相对路径解析为工程根目录下的绝对路径。"""
|
||||
p = Path(str(cfg[key]))
|
||||
return p if p.is_absolute() else ROOT / p
|
||||
@@ -0,0 +1,251 @@
|
||||
"""新岗位发现：纯 Python TF-IDF（字符 n-gram）向量化 + 凝聚聚类。
|
||||
|
||||
流程：无法归入既有岗位的 JD → TF-IDF 向量 → 余弦相似度凝聚聚类 →
|
||||
簇同时满足四项指标（支撑 JD 数 / 独立企业数 / 近期增长率 / 与既有岗位差异度）→
|
||||
候选新岗位 → 进入“待审核”分区，人工 accept/modify/reject 后才写入正式图谱，审核留痕。
|
||||
阈值与《项目方案》第七章对齐：≥15 条 JD / ≥5 家企业 / 增长率 ≥50% / 差异度 ≥0.4。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from app.graph.tkg import now_str, quarter_of
|
||||
from app.models import ExtractionResult, JDRecord, JobDefinition, ReviewRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 候选新岗位阈值（与方案文档一致）
|
||||
MIN_CLUSTER_SIZE = 15 # 支撑 JD 数
|
||||
MIN_COMPANIES = 5 # 独立企业数（防单一企业造词）
|
||||
MIN_GROWTH = 1.5 # 近期增长率（最近一年 JD 数 / 前一年，≥50% 增长）
|
||||
MIN_DISTINCTNESS = 0.4 # 与既有岗位差异度：1 - 与最近既有岗位质心的余弦相似度
|
||||
MERGE_SIM = 0.50 # 凝聚合并的质心余弦相似度阈值
|
||||
|
||||
|
||||
# ---------- TF-IDF（字符 2/3-gram） ----------
|
||||
def _char_ngrams(text: str, ns=(2, 3)) -> list[str]:
|
||||
text = "".join(text.split())
|
||||
out: list[str] = []
|
||||
for n in ns:
|
||||
out.extend(text[i:i + n] for i in range(max(0, len(text) - n + 1)))
|
||||
return out
|
||||
|
||||
|
||||
class TfidfVectorizer:
|
||||
"""极简 TF-IDF：字符 n-gram 词表 + IDF 加权 + L2 归一。"""
|
||||
|
||||
def __init__(self):
|
||||
self.idf: dict[str, float] = {}
|
||||
|
||||
def fit(self, docs: list[str]) -> "TfidfVectorizer":
|
||||
df: Counter = Counter()
|
||||
for d in docs:
|
||||
df.update(set(_char_ngrams(d)))
|
||||
n = max(1, len(docs))
|
||||
self.idf = {t: math.log((n + 1) / (c + 1)) + 1.0 for t, c in df.items()}
|
||||
return self
|
||||
|
||||
def transform(self, doc: str) -> dict[str, float]:
|
||||
tf: Counter = Counter(_char_ngrams(doc))
|
||||
vec = {t: (c / max(1, sum(tf.values()))) * self.idf.get(t, 1.0)
|
||||
for t, c in tf.items()}
|
||||
norm = math.sqrt(sum(v * v for v in vec.values())) or 1.0
|
||||
return {t: v / norm for t, v in vec.items()}
|
||||
|
||||
|
||||
def cosine(a: dict[str, float], b: dict[str, float]) -> float:
|
||||
"""稀疏向量余弦相似度。"""
|
||||
if len(a) > len(b):
|
||||
a, b = b, a
|
||||
return sum(v * b.get(t, 0.0) for t, v in a.items())
|
||||
|
||||
|
||||
def _centroid(vecs: list[dict[str, float]]) -> dict[str, float]:
|
||||
c: dict[str, float] = {}
|
||||
for v in vecs:
|
||||
for t, x in v.items():
|
||||
c[t] = c.get(t, 0.0) + x
|
||||
n = max(1, len(vecs))
|
||||
return {t: x / n for t, x in c.items()}
|
||||
|
||||
|
||||
def agglomerative(vecs: list[dict[str, float]], merge_sim: float = MERGE_SIM) -> list[list[int]]:
|
||||
"""凝聚聚类（平均链接的贪心实现）：每轮合并质心最相似的一对簇。"""
|
||||
clusters: list[list[int]] = [[i] for i in range(len(vecs))]
|
||||
centroids = [_centroid([v]) for v in vecs]
|
||||
while True:
|
||||
best, pair = merge_sim, None
|
||||
for i in range(len(clusters)):
|
||||
for j in range(i + 1, len(clusters)):
|
||||
s = cosine(centroids[i], centroids[j])
|
||||
if s > best:
|
||||
best, pair = s, (i, j)
|
||||
if pair is None:
|
||||
return clusters
|
||||
i, j = pair
|
||||
clusters[i].extend(clusters[j])
|
||||
centroids[i] = _centroid([vecs[k] for k in clusters[i]])
|
||||
del clusters[j], centroids[j]
|
||||
|
||||
|
||||
# ---------- 候选生成与审核 ----------
|
||||
def discover_new_jobs(
|
||||
records: list[JDRecord],
|
||||
extractions: dict[str, ExtractionResult],
|
||||
known_titles: set[str],
|
||||
min_size: int = MIN_CLUSTER_SIZE,
|
||||
min_companies: int = MIN_COMPANIES,
|
||||
min_growth: float = MIN_GROWTH,
|
||||
min_distinctness: float = MIN_DISTINCTNESS,
|
||||
) -> list[JobDefinition]:
|
||||
"""从未知岗位 JD 中聚类发现候选新岗位（status=pending）。
|
||||
|
||||
四指标判定：支撑 JD 数 / 独立企业数 / 近期增长率 / 与既有岗位差异度。
|
||||
"""
|
||||
unknown = [r for r in records
|
||||
if not r.is_noise and not r.is_duplicate
|
||||
and r.jid in extractions
|
||||
and extractions[r.jid].job_title not in known_titles]
|
||||
if len(unknown) < min_size:
|
||||
logger.info("未知岗位 JD 不足 %d 条，跳过新岗位发现", min_size)
|
||||
return []
|
||||
|
||||
# 岗位名称是强信号：重复十次提升其在 TF-IDF 中的权重；
|
||||
# 再叠加抽取出的必备技能（重复三次），保证长文本、多模板句式的 JD 仍能按岗位聚拢
|
||||
def doc_of(r: JDRecord) -> str:
|
||||
e = extractions[r.jid]
|
||||
return f"{r.job_title} " * 10 + " ".join(e.required_skills * 3) + " " + r.text
|
||||
|
||||
docs = [doc_of(r) for r in unknown]
|
||||
vectorizer = TfidfVectorizer().fit(docs)
|
||||
vecs = [vectorizer.transform(d) for d in docs]
|
||||
clusters = agglomerative(vecs)
|
||||
|
||||
# 既有岗位质心（差异度基准）：按已知标题聚合并向量化
|
||||
known_docs: dict[str, list[str]] = {}
|
||||
for r in records:
|
||||
e = extractions.get(r.jid)
|
||||
if e and e.job_title in known_titles and not r.is_noise and not r.is_duplicate:
|
||||
known_docs.setdefault(e.job_title, []).append(doc_of(r))
|
||||
known_centroids = {t: _centroid([vectorizer.transform(d) for d in ds])
|
||||
for t, ds in known_docs.items()}
|
||||
|
||||
candidates: list[JobDefinition] = []
|
||||
for members in clusters:
|
||||
if len(members) < min_size:
|
||||
continue
|
||||
group = [unknown[i] for i in members]
|
||||
companies = {r.company for r in group}
|
||||
if len(companies) < min_companies:
|
||||
continue
|
||||
years = Counter(r.publish_date.year for r in group)
|
||||
latest_year = max(years)
|
||||
prev_year_n = years.get(latest_year - 1, 0)
|
||||
# 前一年没有基数视为“从无到有”，增长率视为达标
|
||||
growth = years[latest_year] / prev_year_n if prev_year_n else float("inf")
|
||||
if growth < min_growth:
|
||||
continue
|
||||
# 差异度：与最近既有岗位质心的距离，过近说明只是既有岗位的变体
|
||||
cvec = _centroid([vecs[i] for i in members])
|
||||
max_sim = max((cosine(cvec, kc) for kc in known_centroids.values()), default=0.0)
|
||||
distinctness = 1.0 - max_sim
|
||||
if distinctness < min_distinctness:
|
||||
logger.info("簇与既有岗位差异度不足（%.2f < %.2f），不列为新岗位", distinctness, min_distinctness)
|
||||
continue
|
||||
# 簇内最常见的抽取标题作为候选岗位名；技能取簇内高频
|
||||
titles = Counter(extractions[r.jid].job_title for r in group)
|
||||
title = titles.most_common(1)[0][0]
|
||||
req = Counter(s for r in group for s in extractions[r.jid].required_skills)
|
||||
pref = Counter(s for r in group for s in extractions[r.jid].preferred_skills)
|
||||
scen = Counter(s for r in group for s in extractions[r.jid].scenarios)
|
||||
ind = Counter(r.industry for r in group if r.industry)
|
||||
n = len(group)
|
||||
candidates.append(JobDefinition(
|
||||
job_id=f"cand_{title}",
|
||||
title=title,
|
||||
category="新岗位候选",
|
||||
industries=[s for s, _ in ind.most_common(2)],
|
||||
required_skills=[s for s, c in req.most_common() if c >= n * 0.4],
|
||||
preferred_skills=[s for s, c in pref.most_common() if c >= n * 0.25],
|
||||
scenarios=[s for s, _ in scen.most_common(5)],
|
||||
status="pending",
|
||||
source_count=n,
|
||||
company_count=len(companies),
|
||||
support_source_ids=[r.source_id for r in group],
|
||||
support_windows=sorted({quarter_of(r.publish_date) for r in group}),
|
||||
))
|
||||
logger.info("候选新岗位：%s（支撑 %d 条 JD / %d 家企业 / 增长率 %.2f / 差异度 %.2f）",
|
||||
title, n, len(companies), growth, distinctness)
|
||||
return candidates
|
||||
|
||||
|
||||
# ---------- 待审核分区与审核留痕（SQLite 状态库，见 app/storage/db.py） ----------
|
||||
def save_pending(candidates: list[JobDefinition], path: str | Path) -> None:
|
||||
"""整体覆写待审核分区。``path`` 为 SQLite 状态库路径（配置键 state_db_path）。"""
|
||||
from app.storage import db
|
||||
keep = set()
|
||||
for c in candidates:
|
||||
db.save_candidate(path, c.model_dump())
|
||||
keep.add(c.job_id)
|
||||
for old in db.list_candidates(path):
|
||||
if old["job_id"] not in keep:
|
||||
db.delete_candidate(path, old["job_id"])
|
||||
|
||||
|
||||
def load_pending(path: str | Path) -> list[JobDefinition]:
|
||||
"""读取全部待审核候选。``path`` 为 SQLite 状态库路径（配置键 state_db_path）。"""
|
||||
from app.storage import db
|
||||
return [JobDefinition(**d) for d in db.list_candidates(path)]
|
||||
|
||||
|
||||
def apply_review(
|
||||
pending_path: str | Path,
|
||||
audit_path: str | Path,
|
||||
candidate_id: str,
|
||||
action: str,
|
||||
operator: str,
|
||||
reason: str = "",
|
||||
modify: dict | None = None,
|
||||
) -> tuple[JobDefinition | None, ReviewRecord]:
|
||||
"""人工审核：accept/modify/reject。返回 (通过的岗位定义或 None, 审核留痕)。
|
||||
|
||||
``pending_path`` 为 SQLite 状态库路径；``audit_path`` 为旧 audit.jsonl 路径，
|
||||
仅在状态库首次打开时作为一次性迁移种子使用（原文件保留）。
|
||||
"""
|
||||
from app.storage import db
|
||||
db.connect(pending_path, legacy_audit=audit_path).close() # 首次打开触发迁移（幂等）
|
||||
candidates = load_pending(pending_path)
|
||||
target = next((c for c in candidates if c.job_id == candidate_id), None)
|
||||
if target is None:
|
||||
raise KeyError(f"候选不存在：{candidate_id}")
|
||||
before = target.model_dump()
|
||||
accepted: JobDefinition | None = None
|
||||
after: dict = {}
|
||||
if action == "accept":
|
||||
target.status = "active"
|
||||
accepted = target
|
||||
after = target.model_dump()
|
||||
elif action == "modify":
|
||||
if modify:
|
||||
target = target.model_copy(update=modify)
|
||||
target.status = "active"
|
||||
accepted = target
|
||||
after = target.model_dump()
|
||||
elif action == "reject":
|
||||
target.status = "rejected"
|
||||
after = target.model_dump()
|
||||
# 驳回：若该候选曾入库（覆写层有记录），一并清理，避免重建后复活
|
||||
db.delete_override(pending_path, candidate_id)
|
||||
else:
|
||||
raise ValueError(f"未知审核动作：{action}")
|
||||
|
||||
record = ReviewRecord(candidate_id=candidate_id, action=action, operator=operator,
|
||||
reason=reason, before=before, after=after, time=now_str())
|
||||
db.append_audit(pending_path, record.model_dump())
|
||||
# 已审核的候选从待审核分区移除
|
||||
db.delete_candidate(pending_path, candidate_id)
|
||||
logger.info("审核完成：%s -> %s（操作人 %s）", candidate_id, action, operator)
|
||||
return accepted, record
|
||||
@@ -0,0 +1,143 @@
|
||||
"""岗位能力演化分析：按季度窗口聚合岗位技能频次，相邻窗口对比产生事件。
|
||||
|
||||
事件类型：新增 / 增强 / 减弱 / 退出；每个事件保存前后权重、时间窗、
|
||||
来源 JD，并生成注明数据源的中文更新说明。事件追加进图谱，不覆盖历史。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
|
||||
from app.graph.tkg import TemporalKG, new_event_id, now_str, quarter_of
|
||||
from app.models import ChangeEvent, ExtractionResult, JDRecord, GraphObservation
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 事件判定阈值（频次基于窗口内 JD 计数）
|
||||
MIN_SUPPORT = 2 # 新增/退出需要的最小支撑次数
|
||||
GROW_RATIO = 1.5 # 增强：curr >= prev * 1.5 且差值 >= MIN_DIFF
|
||||
SHRINK_RATIO = 0.5 # 减弱：curr <= prev * 0.5 且差值 >= MIN_DIFF
|
||||
MIN_DIFF = 3 # 增强/减弱的最小绝对差，抑制稀疏窗口抖动
|
||||
MIN_ESTABLISHED_WINDOWS = 3 # 技能至少稳定出现 3 个窗口，才允许判定“退出”
|
||||
|
||||
|
||||
def build_observations(
|
||||
records: list[JDRecord],
|
||||
extractions: dict[str, ExtractionResult],
|
||||
job_id_of_title: dict[str, str],
|
||||
) -> list[GraphObservation]:
|
||||
"""把“JD × 抽取结果”转换为图谱观察流（按季度窗口，带时间衰减权重）。"""
|
||||
observations: list[GraphObservation] = []
|
||||
for r in records:
|
||||
if r.is_noise:
|
||||
continue
|
||||
ext = extractions.get(r.jid)
|
||||
if ext is None:
|
||||
continue
|
||||
job_id = job_id_of_title.get(ext.job_title)
|
||||
if job_id is None:
|
||||
continue # 未知岗位走“新岗位发现”链路，不进正式图谱
|
||||
window = quarter_of(r.publish_date)
|
||||
for skill in ext.required_skills:
|
||||
observations.append(GraphObservation(
|
||||
job_id=job_id, skill=skill, importance="required",
|
||||
weight=r.weight, window=window, source_ids=[r.source_id]))
|
||||
for skill in ext.preferred_skills:
|
||||
observations.append(GraphObservation(
|
||||
job_id=job_id, skill=skill, importance="preferred",
|
||||
weight=r.weight * 0.6, window=window, source_ids=[r.source_id]))
|
||||
return observations
|
||||
|
||||
|
||||
def aggregate_windows(
|
||||
observations: list[GraphObservation],
|
||||
) -> dict[str, dict[str, dict[str, dict]]]:
|
||||
"""聚合为 {job_id: {window: {skill: {count, weight, sources}}}}。"""
|
||||
agg: dict[str, dict[str, dict[str, dict]]] = defaultdict(lambda: defaultdict(dict))
|
||||
for o in observations:
|
||||
cell = agg[o.job_id][o.window].setdefault(
|
||||
o.skill, {"count": 0, "weight": 0.0, "sources": []})
|
||||
cell["count"] += 1
|
||||
cell["weight"] = round(cell["weight"] + o.weight, 4)
|
||||
for sid in o.source_ids:
|
||||
if sid not in cell["sources"]:
|
||||
cell["sources"].append(sid)
|
||||
return agg
|
||||
|
||||
|
||||
def _describe(change_type: str, job_title: str, skill: str, window: str,
|
||||
prev: float, curr: float, n_sources: int) -> str:
|
||||
"""生成注明数据源的中文更新说明。"""
|
||||
base = f"{window}，岗位【{job_title}】技能【{skill}】"
|
||||
tail = f"（前后权重 {prev:.2f}→{curr:.2f}；数据源：招聘 JD，本窗口支撑样本 {n_sources} 条）"
|
||||
return base + {
|
||||
"新增": "出现新增能力要求",
|
||||
"增强": "要求显著增强",
|
||||
"减弱": "要求明显减弱",
|
||||
"退出": "不再出现，判定退出该岗位能力图谱",
|
||||
}[change_type] + tail
|
||||
|
||||
|
||||
def analyze(kg: TemporalKG, agg: dict | None = None,
|
||||
observations: list[GraphObservation] | None = None) -> list[ChangeEvent]:
|
||||
"""相邻季度窗口对比，产生演化事件并写入图谱（追加）。返回本次新事件。"""
|
||||
if agg is None:
|
||||
agg = aggregate_windows(observations or [])
|
||||
new_events: list[ChangeEvent] = []
|
||||
for job_id, windows in agg.items():
|
||||
job = kg.jobs.get(job_id)
|
||||
title = job.title if job else job_id
|
||||
seen_before: set[str] = set()
|
||||
exited: set[str] = set() # 已判定退出的技能，避免重复报
|
||||
windows_seen: dict[str, int] = defaultdict(int) # 技能历史出现窗口数
|
||||
prev_map: dict[str, dict] = {}
|
||||
sorted_windows = sorted(windows)
|
||||
for wi, window in enumerate(sorted_windows):
|
||||
curr_map = windows[window]
|
||||
# 下一窗口计数（用于“退出”前瞻：下一窗口仍为 0 才确认退出）
|
||||
next_map = windows[sorted_windows[wi + 1]] if wi + 1 < len(sorted_windows) else {}
|
||||
skills = set(curr_map) | set(prev_map)
|
||||
for skill in sorted(skills):
|
||||
prev = prev_map.get(skill, {"count": 0, "weight": 0.0, "sources": []})
|
||||
curr = curr_map.get(skill, {"count": 0, "weight": 0.0, "sources": []})
|
||||
pc, cc = prev["count"], curr["count"]
|
||||
next_c = next_map.get(skill, {"count": 0})["count"]
|
||||
change = None
|
||||
if cc >= MIN_SUPPORT and pc == 0 and skill not in seen_before:
|
||||
change = "新增"
|
||||
elif (cc == 0 and next_c == 0 and skill not in exited
|
||||
and windows_seen.get(skill, 0) >= MIN_ESTABLISHED_WINDOWS):
|
||||
# 已稳定出现多个窗口的技能，连续两窗口消失 → 退出
|
||||
change = "退出"
|
||||
elif pc >= 3 and cc >= pc * GROW_RATIO and cc - pc >= MIN_DIFF:
|
||||
change = "增强"
|
||||
elif pc >= 3 and 0 < cc <= pc * SHRINK_RATIO and pc - cc >= MIN_DIFF:
|
||||
change = "减弱"
|
||||
if change is None:
|
||||
continue
|
||||
ev = ChangeEvent(
|
||||
event_id=new_event_id(), job_id=job_id, skill=skill,
|
||||
change_type=change, window=window,
|
||||
prev_weight=round(prev["weight"], 4),
|
||||
curr_weight=round(curr["weight"], 4),
|
||||
source_ids=curr["sources"][:10],
|
||||
description=_describe(change, title, skill, window,
|
||||
prev["weight"], curr["weight"],
|
||||
len(curr["sources"])),
|
||||
created_at=now_str(),
|
||||
)
|
||||
kg.add_event(ev)
|
||||
new_events.append(ev)
|
||||
if change == "退出":
|
||||
kg.close_interval(job_id, skill, window)
|
||||
exited.add(skill)
|
||||
for s, c in curr_map.items():
|
||||
if c["count"] > 0:
|
||||
exited.discard(s) # 重新出现则解除退出态
|
||||
seen_before |= {s for s, c in curr_map.items() if c["count"] > 0}
|
||||
for s, c in curr_map.items():
|
||||
if c["count"] > 0:
|
||||
windows_seen[s] += 1
|
||||
prev_map = curr_map
|
||||
logger.info("演化分析完成：新增事件 %d 条", len(new_events))
|
||||
return new_events
|
||||
@@ -0,0 +1,131 @@
|
||||
"""抽取器基础：Extractor 协议、技能词表加载与匹配、字段级多源投票。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Protocol
|
||||
|
||||
from app.config import ROOT
|
||||
from app.models import ExtractionResult, JDRecord
|
||||
|
||||
|
||||
class Extractor(Protocol):
|
||||
"""抽取器协议：任何实现（离线/大模型）都必须遵守同一输出 Schema。"""
|
||||
|
||||
name: str
|
||||
|
||||
def extract(self, record: JDRecord) -> ExtractionResult:
|
||||
...
|
||||
|
||||
|
||||
def load_lexicon(path: str | Path | None = None) -> dict:
|
||||
"""加载技能词表 data/skills_lexicon.json。
|
||||
|
||||
结构：{"skills": [{name, category, aliases, level, prereqs}], "scenarios": [...]}
|
||||
"""
|
||||
p = Path(path) if path else ROOT / "data" / "skills_lexicon.json"
|
||||
return json.loads(p.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
class SkillMatcher:
|
||||
"""基于词表（含别名）的技能归一化匹配器。
|
||||
|
||||
- 含 ASCII 的词使用“非字母数字边界”匹配，避免 Java 误中 JavaScript、C 误中 C++；
|
||||
- 纯中文词直接子串匹配；
|
||||
- 重叠命中按“起点靠前、长度最长”优先，保证 Spring Boot 优先于 Spring。
|
||||
"""
|
||||
|
||||
def __init__(self, lexicon: dict):
|
||||
self.lexicon = lexicon
|
||||
self.alias2name: dict[str, str] = {}
|
||||
for s in lexicon["skills"]:
|
||||
self.alias2name[s["name"]] = s["name"]
|
||||
for a in s.get("aliases", []):
|
||||
self.alias2name[a] = s["name"]
|
||||
# 长词优先编译，Java 与 JavaScript 这类前缀词靠边界 + 长度双重保险
|
||||
self._patterns: list[tuple[str, re.Pattern]] = []
|
||||
for alias in sorted(self.alias2name, key=len, reverse=True):
|
||||
esc = re.escape(alias)
|
||||
if re.search(r"[A-Za-z0-9+#.]", alias):
|
||||
pat = re.compile(rf"(?<![A-Za-z0-9+#.]){esc}(?![A-Za-z0-9+#.])", re.IGNORECASE)
|
||||
else:
|
||||
pat = re.compile(esc)
|
||||
self._patterns.append((alias, pat))
|
||||
|
||||
def normalize(self, name: str) -> str:
|
||||
"""把用户输入的技能名归一到词表标准名，三级容错：
|
||||
|
||||
1. 精确/别名匹配（大小写不敏感）：redis→Redis、K8s→Kubernetes；
|
||||
2. 包含匹配（语义近似兜底）：输入包含某词表技能时取其最长者，
|
||||
如 微服务架构→微服务、Java语言→Java、SpringBoot项目→Spring Boot；
|
||||
3. 均未命中则原样返回（交由上层标记为"词表外技能"）。
|
||||
"""
|
||||
if not hasattr(self, "_alias_lower"):
|
||||
self._alias_lower = {k.lower(): v for k, v in self.alias2name.items()}
|
||||
key = name.strip()
|
||||
hit = self._alias_lower.get(key.lower())
|
||||
if hit:
|
||||
return hit
|
||||
hits = self.find(key)
|
||||
if hits:
|
||||
# 取覆盖最长（最具体）的命中，避免 JavaScript 被 Java 截胡
|
||||
best = max(hits, key=lambda h: h[1] - h[0])
|
||||
return best[2]
|
||||
return key
|
||||
|
||||
def find(self, text: str) -> list[tuple[int, int, str]]:
|
||||
"""返回 [(start, end, canonical_name)]，重叠命中已消解。"""
|
||||
hits: list[tuple[int, int, str]] = []
|
||||
for alias, pat in self._patterns:
|
||||
canonical = self.alias2name[alias]
|
||||
for m in pat.finditer(text):
|
||||
hits.append((m.start(), m.end(), canonical))
|
||||
hits.sort(key=lambda h: (h[0], -(h[1] - h[0])))
|
||||
out: list[tuple[int, int, str]] = []
|
||||
last_end = -1
|
||||
for start, end, name in hits:
|
||||
if start >= last_end:
|
||||
out.append((start, end, name))
|
||||
last_end = end
|
||||
return out
|
||||
|
||||
def find_names(self, text: str) -> list[str]:
|
||||
"""只返回归一化后的技能名（保序去重）。"""
|
||||
seen: list[str] = []
|
||||
for _, _, name in self.find(text):
|
||||
if name not in seen:
|
||||
seen.append(name)
|
||||
return seen
|
||||
|
||||
def find_scenarios(self, text: str) -> list[str]:
|
||||
"""应用场景词表匹配（纯子串，保序去重）。"""
|
||||
out: list[str] = []
|
||||
for sc in self.lexicon.get("scenarios", []):
|
||||
if sc in text and sc not in out:
|
||||
out.append(sc)
|
||||
return out
|
||||
|
||||
|
||||
def vote_field(
|
||||
candidates: list[tuple[list[str], float]],
|
||||
min_agreement: float = 0.5,
|
||||
) -> tuple[list[str], float]:
|
||||
"""字段级多源投票：关键字段多源一致才给高置信。
|
||||
|
||||
candidates: [(某来源抽到的技能列表, 该来源权重)]，例如离线规则与大模型双通道。
|
||||
返回 (通过投票的技能列表, 置信度)。技能按加权支持率 >= min_agreement 入选。
|
||||
"""
|
||||
if not candidates:
|
||||
return [], 0.0
|
||||
total = sum(w for _, w in candidates) or 1.0
|
||||
support: dict[str, float] = {}
|
||||
order: list[str] = []
|
||||
for skills, w in candidates:
|
||||
for s in dict.fromkeys(skills):
|
||||
support[s] = support.get(s, 0.0) + w
|
||||
if s not in order:
|
||||
order.append(s)
|
||||
chosen = [s for s in order if support[s] / total >= min_agreement]
|
||||
confidence = max((support[s] / total for s in chosen), default=0.0)
|
||||
return chosen, round(confidence, 4)
|
||||
@@ -0,0 +1,124 @@
|
||||
"""星火 / OpenAI 兼容 API 抽取器：JSON 模式、Schema 校验、失败重试 2 次、超时降级。
|
||||
|
||||
- 密钥只从环境变量读取（默认 SPARK_API_KEY），绝不写入代码或配置；
|
||||
- 无密钥时 get_extractor() 显式降级到离线抽取器；
|
||||
- 返回结果先过 ExtractionResult（pydantic）强校验，不合法即重试；
|
||||
- 重试耗尽或超时 → 降级到离线通道，保证链路永不断。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
|
||||
import httpx
|
||||
|
||||
from app.extract.base import Extractor, load_lexicon
|
||||
from app.extract.offline import OfflineExtractor
|
||||
from app.extract.prompts import PROMPT_VERSION, SYSTEM_PROMPT, build_prompt
|
||||
from app.models import ExtractionResult, JDRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_RETRIES = 2 # 失败重试 2 次
|
||||
|
||||
|
||||
class LLMExtractor:
|
||||
"""大模型抽取器（需配置密钥才启用）。"""
|
||||
|
||||
name = "llm"
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
api_key: str,
|
||||
base_url: str,
|
||||
model: str,
|
||||
timeout: float = 20.0,
|
||||
fallback: Extractor | None = None,
|
||||
):
|
||||
self.api_key = api_key
|
||||
self.base_url = base_url.rstrip("/")
|
||||
self.model = model
|
||||
self.timeout = timeout
|
||||
self.fallback = fallback or OfflineExtractor(load_lexicon())
|
||||
|
||||
def _call_api(self, jd_text: str) -> str:
|
||||
"""调用 OpenAI 兼容 chat/completions，返回模型输出文本。"""
|
||||
resp = httpx.post(
|
||||
f"{self.base_url}/chat/completions",
|
||||
headers={"Authorization": f"Bearer {self.api_key}"},
|
||||
json={
|
||||
"model": self.model,
|
||||
"messages": [
|
||||
{"role": "system", "content": SYSTEM_PROMPT},
|
||||
{"role": "user", "content": build_prompt(jd_text)},
|
||||
],
|
||||
"response_format": {"type": "json_object"},
|
||||
"temperature": 0.1,
|
||||
},
|
||||
timeout=self.timeout,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
return resp.json()["choices"][0]["message"]["content"]
|
||||
|
||||
@staticmethod
|
||||
def _parse(content: str, record: JDRecord) -> ExtractionResult:
|
||||
"""Schema 强校验：坏 JSON / 缺字段都会抛异常，由上层重试。
|
||||
|
||||
先剥离模型常见的 ```json 代码块包裹再解析。
|
||||
"""
|
||||
text = content.strip()
|
||||
if text.startswith("```"):
|
||||
text = text.split("\n", 1)[-1] if "\n" in text else text.strip("`")
|
||||
if text.rstrip().endswith("```"):
|
||||
text = text.rstrip()[:-3]
|
||||
data = json.loads(text.strip())
|
||||
result = ExtractionResult(**data)
|
||||
result.jid = record.jid
|
||||
result.extractor = "llm"
|
||||
result.prompt_version = PROMPT_VERSION
|
||||
result.confidence = 0.9
|
||||
return result
|
||||
|
||||
def extract(self, record: JDRecord) -> ExtractionResult:
|
||||
last_err: Exception | None = None
|
||||
for attempt in range(MAX_RETRIES + 1):
|
||||
try:
|
||||
return self._parse(self._call_api(record.text), record)
|
||||
except Exception as exc: # noqa: BLE001 - 网络/解析/校验任何失败都要可重试可降级
|
||||
last_err = exc
|
||||
logger.warning("LLM 抽取第 %d 次失败（%s），%s",
|
||||
attempt + 1, type(exc).__name__,
|
||||
"重试" if attempt < MAX_RETRIES else "重试耗尽")
|
||||
logger.warning("LLM 不可用（%s），降级离线抽取 jid=%s", last_err, record.jid)
|
||||
return self.fallback.extract(record)
|
||||
|
||||
|
||||
def get_extractor(config: dict) -> Extractor:
|
||||
"""按配置选择抽取通道：配置了密钥（config.json 或环境变量）→ LLM，否则显式降级离线。
|
||||
|
||||
密钥优先级：config.json 的 llm_api_key > 环境变量（默认 SPARK_API_KEY）。
|
||||
LLM 通道失败时自动降级离线抽取（见 LLMExtractor.extract），链路永不断。
|
||||
"""
|
||||
key_env = str(config.get("llm_api_key_env", "SPARK_API_KEY"))
|
||||
api_key = (str(config.get("llm_api_key") or "").strip()
|
||||
or os.environ.get(key_env, "").strip())
|
||||
offline = OfflineExtractor(load_lexicon(config.get("lexicon_path") or None))
|
||||
if api_key:
|
||||
logger.info("检测到 LLM 密钥，启用大模型抽取通道（模型 %s）", config.get("llm_model"))
|
||||
return LLMExtractor(
|
||||
api_key=api_key,
|
||||
base_url=str(config.get("llm_base_url")),
|
||||
model=str(config.get("llm_model")),
|
||||
timeout=float(config.get("llm_timeout", 20)),
|
||||
fallback=offline,
|
||||
)
|
||||
logger.info("未配置 LLM 密钥，显式降级到离线确定性抽取器")
|
||||
return offline
|
||||
|
||||
|
||||
def llm_ready(config: dict) -> bool:
|
||||
"""是否已配置 LLM 密钥（config.json 或环境变量）。"""
|
||||
key_env = str(config.get("llm_api_key_env", "SPARK_API_KEY"))
|
||||
return bool(str(config.get("llm_api_key") or "").strip()
|
||||
or os.environ.get(key_env, "").strip())
|
||||
@@ -0,0 +1,128 @@
|
||||
"""离线确定性抽取器：技能词表 + 版面规则，从 JD 文本抽出结构化岗位定义。
|
||||
|
||||
版面约定（数据集与真实主流招聘平台格式兼容）：
|
||||
- “岗位名称：X”行给出岗位名称；
|
||||
- 职责段（岗位职责/工作职责/工作内容/职位职责）逐条编号；
|
||||
- 要求段（任职要求/岗位要求/任职条件/任职资格）出现必备技能；
|
||||
- 加分段（加分项/优先考虑/优先条件）出现加分技能；
|
||||
- 尾部段落（团队与福利/福利待遇/岗位福利/公司福利）不属于任何技能区，
|
||||
作为加分段的结束边界，避免福利文案中的技能词混入加分技能；
|
||||
- 应用场景词表全文匹配。
|
||||
每个字段都附原文证据片段，供人机协同审核防“幻觉”。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
from app.extract.base import SkillMatcher, load_lexicon
|
||||
from app.models import ExtractionResult, JDRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 版面标题词表（生成器与抽取器共用，保证规则与数据一致）
|
||||
DUTY_HEADERS = ["岗位职责", "工作职责", "工作内容", "职位职责"]
|
||||
REQ_HEADERS = ["任职要求", "岗位要求", "任职条件", "任职资格"]
|
||||
PREF_HEADERS = ["加分项", "优先考虑", "优先条件"]
|
||||
# 尾部段落标题：出现在加分段之后的团队/福利介绍，作为版面区域结束边界
|
||||
TAIL_HEADERS = ["团队与福利", "福利待遇", "岗位福利", "公司福利"]
|
||||
|
||||
_TITLE_RE = re.compile(r"岗位名称[:：]\s*([^\n]+)")
|
||||
_NUMBER_RE = re.compile(r"^\s*(?:\d+\s*[、.．)）]|[-•·])\s*")
|
||||
|
||||
|
||||
def _find_header(text: str, headers: list[str]) -> tuple[int, int] | None:
|
||||
"""返回第一处标题的 (起, 止)，找不到返回 None。"""
|
||||
best: tuple[int, int] | None = None
|
||||
for h in headers:
|
||||
i = text.find(h)
|
||||
if i >= 0 and (best is None or i < best[0]):
|
||||
best = (i, i + len(h))
|
||||
return best
|
||||
|
||||
|
||||
class OfflineExtractor:
|
||||
"""离线确定性抽取器（默认通道，无需任何外部依赖）。"""
|
||||
|
||||
name = "offline"
|
||||
|
||||
def __init__(self, lexicon: dict | None = None):
|
||||
self.lexicon = lexicon or load_lexicon()
|
||||
self.matcher = SkillMatcher(self.lexicon)
|
||||
|
||||
def _zones(self, text: str) -> dict[str, str]:
|
||||
"""把文本切成 职责/要求/加分 三个版面区域（尾部福利段落不计入）。"""
|
||||
duty = _find_header(text, DUTY_HEADERS)
|
||||
req = _find_header(text, REQ_HEADERS)
|
||||
pref = _find_header(text, PREF_HEADERS)
|
||||
tail = _find_header(text, TAIL_HEADERS)
|
||||
|
||||
def zone(start_h, *end_hs):
|
||||
if start_h is None:
|
||||
return ""
|
||||
s = start_h[1]
|
||||
ends = [h[0] for h in end_hs if h is not None and h[0] >= s]
|
||||
e = min(ends) if ends else len(text)
|
||||
return text[s:e]
|
||||
|
||||
return {
|
||||
"duty": zone(duty, req, pref, tail),
|
||||
"req": zone(req, pref, tail),
|
||||
"pref": zone(pref, tail),
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _parse_duties(duty_zone: str) -> list[str]:
|
||||
"""逐行解析编号职责，去掉编号前缀。"""
|
||||
out: list[str] = []
|
||||
for line in duty_zone.splitlines():
|
||||
line = _NUMBER_RE.sub("", line).strip().strip("：:")
|
||||
if line:
|
||||
out.append(line)
|
||||
return out
|
||||
|
||||
def extract(self, record: JDRecord) -> ExtractionResult:
|
||||
text = record.text
|
||||
zones = self._zones(text)
|
||||
|
||||
m = _TITLE_RE.search(text)
|
||||
title = m.group(1).strip() if m else record.job_title
|
||||
|
||||
duties = self._parse_duties(zones["duty"])
|
||||
if zones["req"]:
|
||||
required = self.matcher.find_names(zones["req"])
|
||||
else:
|
||||
# 无版面结构的真实 JD（如 AutoWeakS 语料）没有"任职要求"段，
|
||||
# 兜底为全文扫描，避免技能字段整体漏抽
|
||||
required = self.matcher.find_names(text)
|
||||
preferred = self.matcher.find_names(zones["pref"]) if zones["pref"] else []
|
||||
# 加分技能不应重复出现在必备里（极少数版面混排的兜底）
|
||||
preferred = [s for s in preferred if s not in required]
|
||||
scenarios = self.matcher.find_scenarios(text)
|
||||
|
||||
def snippet(zone_text: str, limit: int = 80) -> str:
|
||||
return " ".join(zone_text.split())[:limit]
|
||||
|
||||
evidence = {
|
||||
"job_title": m.group(0) if m else "",
|
||||
"responsibilities": snippet(zones["duty"]),
|
||||
"required_skills": snippet(zones["req"]),
|
||||
"preferred_skills": snippet(zones["pref"]),
|
||||
"scenarios": "、".join(scenarios),
|
||||
}
|
||||
# 置信度：五个字段命中比例（离线规则全命中即 1.0）
|
||||
hits = sum([bool(title), bool(duties), bool(required), True, bool(scenarios)])
|
||||
confidence = round(hits / 5.0, 4)
|
||||
|
||||
return ExtractionResult(
|
||||
jid=record.jid,
|
||||
job_title=title,
|
||||
responsibilities=duties,
|
||||
required_skills=required,
|
||||
preferred_skills=preferred,
|
||||
scenarios=scenarios,
|
||||
evidence=evidence,
|
||||
confidence=confidence,
|
||||
extractor=self.name,
|
||||
prompt_version="offline-v1",
|
||||
)
|
||||
@@ -0,0 +1,43 @@
|
||||
"""大模型抽取的固化提示词模板与输出 Schema。
|
||||
|
||||
prompt_version 是抽取结果血缘的一部分：提示词变更必须升版本号，
|
||||
评测与图谱中都能据此追溯。
|
||||
"""
|
||||
|
||||
PROMPT_VERSION = "spark-jd-v1.1"
|
||||
|
||||
SYSTEM_PROMPT = (
|
||||
"你是招聘信息结构化专家。请从招聘 JD 文本中抽取岗位定义，"
|
||||
"只输出符合给定 JSON Schema 的 JSON，不要输出任何多余文字。"
|
||||
"技能名必须归一化到常见写法（如 LLM应用开发 归一化为 大模型应用开发），"
|
||||
"每个字段都要给出原文证据片段，不得编造原文没有的内容。"
|
||||
)
|
||||
|
||||
# 输出 Schema（JSON Schema 强校验的目标结构，与 ExtractionResult 对齐）
|
||||
OUTPUT_SCHEMA: dict = {
|
||||
"type": "object",
|
||||
"required": ["job_title", "responsibilities", "required_skills", "preferred_skills", "scenarios"],
|
||||
"properties": {
|
||||
"job_title": {"type": "string", "description": "岗位标准名称"},
|
||||
"responsibilities": {"type": "array", "items": {"type": "string"}, "description": "核心职责"},
|
||||
"required_skills": {"type": "array", "items": {"type": "string"}, "description": "必备技能"},
|
||||
"preferred_skills": {"type": "array", "items": {"type": "string"}, "description": "加分技能"},
|
||||
"scenarios": {"type": "array", "items": {"type": "string"}, "description": "应用场景"},
|
||||
"evidence": {"type": "object", "description": "字段名 -> 原文证据片段"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def build_prompt(jd_text: str) -> str:
|
||||
"""拼接待抽取 JD 的用户提示词（含显式 Schema，字段名必须为英文键）。"""
|
||||
return (
|
||||
"请从以下招聘 JD 文本中抽取结构化岗位定义。\n"
|
||||
"要求：\n"
|
||||
"1. 只输出一个 JSON 对象，不要 markdown 代码块，不要任何解释文字；\n"
|
||||
"2. 字段名必须使用以下英文键：job_title（岗位标准名称）、responsibilities（核心职责，字符串数组）、"
|
||||
"required_skills（必备技能，字符串数组）、preferred_skills（加分/优先技能，字符串数组）、"
|
||||
"scenarios（应用场景，字符串数组）、evidence（对象，键为字段名，值为原文证据片段）；\n"
|
||||
"3. 技能名归一化到常见写法（如 LLM应用开发→大模型应用开发）；\n"
|
||||
"4. 不得编造原文没有的内容。\n\n"
|
||||
"JD 文本：\n" + jd_text
|
||||
)
|
||||
@@ -0,0 +1,192 @@
|
||||
"""时序知识图谱（TemporalKG）。
|
||||
|
||||
- 节点：Job / Skill / Industry / Scenario；
|
||||
- 关系：REQUIRES，属性含 importance(必备/加分)、weight、sources，
|
||||
以及 intervals：[[valid_from, valid_to|null]] 生效时间区间列表；
|
||||
- 演化事件只追加不覆盖：同一 (岗位, 技能) 的历史通过区间与事件流共同表达；
|
||||
- snapshot(job_id, date)：任意时间点（或季度窗口）切片查询；
|
||||
- JSON 持久化与加载。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
import networkx as nx
|
||||
|
||||
from app.models import ChangeEvent, GraphObservation, JobDefinition
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def quarter_of(dt) -> str:
|
||||
"""日期 -> 季度窗口字符串，如 2022Q1。窗口串按字典序即时间序。"""
|
||||
return f"{dt.year}Q{(dt.month - 1) // 3 + 1}"
|
||||
|
||||
|
||||
def to_window(d: str) -> str:
|
||||
"""把 'YYYY-MM-DD' 或 'YYYYQn' 统一为季度窗口串。"""
|
||||
if "Q" in d:
|
||||
return d
|
||||
parts = d.split("-")
|
||||
return f"{parts[0]}Q{(int(parts[1]) - 1) // 3 + 1}"
|
||||
|
||||
|
||||
def job_level(expected_years: int | None) -> str:
|
||||
"""由期望年限推出岗位级别：≤2 初级 / 3-5 中级 / >5 高级。"""
|
||||
y = expected_years if expected_years is not None else 3
|
||||
return "初级" if y <= 2 else ("中级" if y <= 5 else "高级")
|
||||
|
||||
|
||||
# 级别 -> 代表年限（人工编辑级别时同步回 expected_years，保持筛选一致）
|
||||
LEVEL_TO_YEARS = {"初级": 2, "中级": 4, "高级": 6}
|
||||
|
||||
|
||||
class TemporalKG:
|
||||
"""基于 networkx DiGraph 的时序知识图谱。"""
|
||||
|
||||
def __init__(self):
|
||||
self.g = nx.DiGraph()
|
||||
self.jobs: dict[str, JobDefinition] = {}
|
||||
self.events: list[ChangeEvent] = []
|
||||
|
||||
# ---------- 节点 ----------
|
||||
def add_job(self, job: JobDefinition) -> None:
|
||||
self.jobs[job.job_id] = job
|
||||
self.g.add_node(job.job_id, type="Job", title=job.title,
|
||||
category=job.category, status=job.status,
|
||||
expected_years=job.expected_years,
|
||||
level=job_level(job.expected_years),
|
||||
source_count=job.source_count,
|
||||
company_count=job.company_count)
|
||||
|
||||
def add_skill(self, name: str, category: str = "") -> None:
|
||||
if name not in self.g:
|
||||
self.g.add_node(name, type="Skill", category=category)
|
||||
|
||||
def add_scenario(self, name: str) -> None:
|
||||
node = f"场景::{name}"
|
||||
if node not in self.g:
|
||||
self.g.add_node(node, type="Scenario", title=name)
|
||||
|
||||
def add_industry(self, name: str) -> None:
|
||||
node = f"行业::{name}"
|
||||
if node not in self.g:
|
||||
self.g.add_node(node, type="Industry", title=name)
|
||||
|
||||
# ---------- 关系（追加不覆盖） ----------
|
||||
def observe(self, obs: GraphObservation) -> None:
|
||||
"""追加一次岗位-技能观察：新关系开区间，已有关系更新权重与来源。"""
|
||||
self.add_skill(obs.skill)
|
||||
if self.g.has_edge(obs.job_id, obs.skill):
|
||||
attrs = self.g[obs.job_id][obs.skill]
|
||||
# 最新权重按观察权重滑动更新；sources 累积不去重覆盖历史
|
||||
attrs["weight"] = round(max(attrs["weight"], obs.weight), 4)
|
||||
for sid in obs.source_ids:
|
||||
if sid not in attrs["sources"]:
|
||||
attrs["sources"].append(sid)
|
||||
attrs["importance"] = obs.importance if obs.importance == "required" else attrs["importance"]
|
||||
# 重新出现：若上一区间已关闭，则开新区间
|
||||
last = attrs["intervals"][-1]
|
||||
if last[1] is not None and last[1] <= obs.window:
|
||||
attrs["intervals"].append([obs.window, None])
|
||||
else:
|
||||
self.g.add_edge(
|
||||
obs.job_id, obs.skill, rel="REQUIRES",
|
||||
importance=obs.importance, weight=round(obs.weight, 4),
|
||||
intervals=[[obs.window, None]],
|
||||
sources=list(obs.source_ids),
|
||||
)
|
||||
|
||||
def close_interval(self, job_id: str, skill: str, end_window: str) -> None:
|
||||
"""关闭 (岗位, 技能) 当前生效区间（技能退出）。"""
|
||||
if self.g.has_edge(job_id, skill):
|
||||
last = self.g[job_id][skill]["intervals"][-1]
|
||||
if last[1] is None:
|
||||
last[1] = end_window
|
||||
|
||||
# ---------- 事件 ----------
|
||||
def add_event(self, ev: ChangeEvent) -> None:
|
||||
self.events.append(ev)
|
||||
|
||||
def events_of(self, job_id: str) -> list[ChangeEvent]:
|
||||
return [e for e in self.events if e.job_id == job_id]
|
||||
|
||||
# ---------- 切片查询 ----------
|
||||
def snapshot(self, job_id: str, date_str: str) -> dict:
|
||||
"""任意时间点切片：返回该时刻生效的技能关系。"""
|
||||
window = to_window(date_str)
|
||||
skills = []
|
||||
if job_id in self.g:
|
||||
for _, skill, attrs in self.g.out_edges(job_id, data=True):
|
||||
if attrs.get("rel") != "REQUIRES":
|
||||
continue
|
||||
alive = any(s <= window and (e is None or window < e)
|
||||
for s, e in attrs["intervals"])
|
||||
if alive:
|
||||
skills.append({
|
||||
"skill": skill,
|
||||
"importance": attrs["importance"],
|
||||
"weight": attrs["weight"],
|
||||
"sources": attrs["sources"],
|
||||
})
|
||||
job = self.jobs.get(job_id)
|
||||
return {
|
||||
"job_id": job_id,
|
||||
"title": job.title if job else job_id,
|
||||
"window": window,
|
||||
"skills": skills,
|
||||
}
|
||||
|
||||
def latest_snapshot(self, job_id: str) -> dict:
|
||||
latest = "9999Q4"
|
||||
return self.snapshot(job_id, latest)
|
||||
|
||||
# ---------- 统计 ----------
|
||||
def stats(self) -> dict:
|
||||
return {
|
||||
"nodes": self.g.number_of_nodes(),
|
||||
"edges": self.g.number_of_edges(),
|
||||
"jobs": len(self.jobs),
|
||||
"skills": sum(1 for _, d in self.g.nodes(data=True) if d.get("type") == "Skill"),
|
||||
"events": len(self.events),
|
||||
}
|
||||
|
||||
# ---------- 持久化 ----------
|
||||
def save(self, path: str | Path) -> None:
|
||||
path = Path(path)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
payload = {
|
||||
"nodes": [{"id": n, **d} for n, d in self.g.nodes(data=True)],
|
||||
"edges": [{"source": u, "target": v, **d} for u, v, d in self.g.edges(data=True)],
|
||||
"jobs": {k: v.model_dump() for k, v in self.jobs.items()},
|
||||
"events": [e.model_dump() for e in self.events],
|
||||
}
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
logger.info("图谱已持久化到 %s（%d 节点 / %d 边）", path,
|
||||
self.g.number_of_nodes(), self.g.number_of_edges())
|
||||
|
||||
@classmethod
|
||||
def load(cls, path: str | Path) -> "TemporalKG":
|
||||
kg = cls()
|
||||
payload = json.loads(Path(path).read_text(encoding="utf-8"))
|
||||
for n in payload["nodes"]:
|
||||
nid = n.pop("id")
|
||||
kg.g.add_node(nid, **n)
|
||||
for e in payload["edges"]:
|
||||
s, t = e.pop("source"), e.pop("target")
|
||||
kg.g.add_edge(s, t, **e)
|
||||
kg.jobs = {k: JobDefinition(**v) for k, v in payload["jobs"].items()}
|
||||
kg.events = [ChangeEvent(**e) for e in payload["events"]]
|
||||
return kg
|
||||
|
||||
|
||||
def new_event_id() -> str:
|
||||
return uuid.uuid4().hex[:12]
|
||||
|
||||
|
||||
def now_str() -> str:
|
||||
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
@@ -0,0 +1,134 @@
|
||||
"""JD 原始数据接入：读取 data/jd_raw/*.jsonl，分配不可变 source_id，坏行进错误清单。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.models import JDRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def load_jd_file(path: str | Path) -> tuple[list[JDRecord], list[dict]]:
|
||||
"""读取单个 jsonl 文件，返回 (合法记录, 错误清单)。"""
|
||||
path = Path(path)
|
||||
records: list[JDRecord] = []
|
||||
errors: list[dict] = []
|
||||
for lineno, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1):
|
||||
source_id = f"{path.name}#{lineno}"
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(line)
|
||||
record = JDRecord(**payload)
|
||||
# source_id 由接入层分配，外部输入不可伪造
|
||||
record.source_id = source_id
|
||||
records.append(record)
|
||||
except (json.JSONDecodeError, ValidationError, TypeError) as exc:
|
||||
errors.append({"source_id": source_id, "error": str(exc)[:200], "raw": line[:120]})
|
||||
logger.warning("坏行已进错误清单: %s", source_id)
|
||||
return records, errors
|
||||
|
||||
|
||||
def load_jd_dir(directory: str | Path) -> tuple[list[JDRecord], list[dict]]:
|
||||
"""读取目录下全部 *.jsonl（按文件名排序，保证可复现）。"""
|
||||
directory = Path(directory)
|
||||
records: list[JDRecord] = []
|
||||
errors: list[dict] = []
|
||||
files = sorted(directory.glob("*.jsonl"))
|
||||
if not files:
|
||||
logger.warning("目录 %s 下没有 jsonl 文件", directory)
|
||||
for f in files:
|
||||
r, e = load_jd_file(f)
|
||||
records.extend(r)
|
||||
errors.extend(e)
|
||||
logger.info("接入完成：%d 条记录，%d 条坏行", len(records), len(errors))
|
||||
return records, errors
|
||||
|
||||
|
||||
# ---------- 手动导入（文件上传 / 文本粘贴 → data/jd_raw/import_*.jsonl） ----------
|
||||
IMPORT_EXTS = {".jsonl", ".csv", ".txt"}
|
||||
IMPORT_MAX_BYTES = 10 * 1024 * 1024 # 10MB
|
||||
_CSV_REQUIRED = ("jid", "job_title", "company", "publish_date", "text")
|
||||
|
||||
|
||||
def _validate_payload(payload: dict) -> JDRecord:
|
||||
"""校验单条导入记录（source_id 由接入层在后续加载时统一分配）。"""
|
||||
record = JDRecord(**payload)
|
||||
record.source_id = ""
|
||||
return record
|
||||
|
||||
|
||||
def parse_import_text(text: str, filename: str = "paste.txt") -> tuple[list[JDRecord], list[dict], int]:
|
||||
"""解析手动导入内容，返回 (合法记录, 坏行清单, 跳过空行数)。
|
||||
|
||||
.csv 按表头映射字段（必需列见 _CSV_REQUIRED）；.jsonl/.txt 逐行 JSON。
|
||||
"""
|
||||
suffix = Path(filename or "paste.txt").suffix.lower()
|
||||
records: list[JDRecord] = []
|
||||
bad: list[dict] = []
|
||||
skipped = 0
|
||||
if suffix == ".csv":
|
||||
rows = list(csv.reader(io.StringIO(text)))
|
||||
if not rows:
|
||||
return records, [{"line": 0, "error": "CSV 内容为空", "raw": ""}], skipped
|
||||
header = [h.strip() for h in rows[0]]
|
||||
missing = [c for c in _CSV_REQUIRED if c not in header]
|
||||
if missing:
|
||||
return records, [{"line": 1, "error": f"CSV 缺少必需列: {missing}", "raw": ",".join(header)[:120]}], skipped
|
||||
for lineno, row in enumerate(rows[1:], start=2):
|
||||
if not row or not any(c.strip() for c in row):
|
||||
skipped += 1
|
||||
continue
|
||||
payload = {h: v.strip() for h, v in zip(header, row) if h}
|
||||
try:
|
||||
records.append(_validate_payload(payload))
|
||||
except (ValidationError, TypeError, ValueError) as exc:
|
||||
bad.append({"line": lineno, "error": str(exc)[:200], "raw": ",".join(row)[:120]})
|
||||
return records, bad, skipped
|
||||
for lineno, line in enumerate(text.splitlines(), start=1):
|
||||
if not line.strip():
|
||||
skipped += 1
|
||||
continue
|
||||
try:
|
||||
records.append(_validate_payload(json.loads(line)))
|
||||
except (json.JSONDecodeError, ValidationError, TypeError) as exc:
|
||||
bad.append({"line": lineno, "error": str(exc)[:200], "raw": line[:120]})
|
||||
return records, bad, skipped
|
||||
|
||||
|
||||
def save_import(text: str, filename: str, jd_raw_dir: str | Path,
|
||||
logs_dir: str | Path) -> dict:
|
||||
"""手动导入落盘：合格记录写 data/jd_raw/import_时间戳.jsonl，坏行写 logs。
|
||||
|
||||
返回 {"imported", "skipped", "bad_lines", "file", "bad_log"}。
|
||||
"""
|
||||
records, bad, skipped = parse_import_text(text, filename)
|
||||
stamp = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||||
result: dict = {"imported": len(records), "skipped": skipped,
|
||||
"bad_lines": bad, "file": "", "bad_log": ""}
|
||||
if records:
|
||||
jd_raw_dir = Path(jd_raw_dir)
|
||||
jd_raw_dir.mkdir(parents=True, exist_ok=True)
|
||||
out = jd_raw_dir / f"import_{stamp}.jsonl"
|
||||
with out.open("w", encoding="utf-8") as f:
|
||||
for r in records:
|
||||
f.write(json.dumps(r.model_dump(mode="json"), ensure_ascii=False) + "\n")
|
||||
result["file"] = out.name
|
||||
logger.info("手动导入：%d 条写入 %s", len(records), out)
|
||||
if bad:
|
||||
logs_dir = Path(logs_dir)
|
||||
logs_dir.mkdir(parents=True, exist_ok=True)
|
||||
log_path = logs_dir / f"import_bad_{stamp}.jsonl"
|
||||
with log_path.open("w", encoding="utf-8") as f:
|
||||
for b in bad:
|
||||
f.write(json.dumps({"source": filename, **b}, ensure_ascii=False) + "\n")
|
||||
result["bad_log"] = log_path.name
|
||||
logger.warning("手动导入坏行 %d 条，已写入 %s", len(bad), log_path)
|
||||
return result
|
||||
@@ -0,0 +1,286 @@
|
||||
"""人岗匹配引擎：技能覆盖率召回 + 四维加权评分 + 差距分析 + 学习路径。
|
||||
|
||||
四维权重默认 必备 50% / 加分 15% / 经验 20% / 场景 15%，可通过 config 覆盖。
|
||||
学习路径：缺失技能按图谱/词表中的前置关系拓扑排序，分阶段输出。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
|
||||
from app.extract.base import load_lexicon
|
||||
from app.models import JobDefinition, MatchResult, ResumeProfile
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DEFAULT_WEIGHTS = {
|
||||
"required": 0.50,
|
||||
"preferred": 0.15,
|
||||
"experience": 0.20,
|
||||
"scenario": 0.15,
|
||||
}
|
||||
|
||||
|
||||
def _coverage(have: set[str], want: list[str]) -> tuple[list[str], list[str], float]:
|
||||
matched = [s for s in want if s in have]
|
||||
missing = [s for s in want if s not in have]
|
||||
cov = len(matched) / len(want) if want else 1.0
|
||||
return matched, missing, cov
|
||||
|
||||
|
||||
def topological_stages(missing: list[str], lexicon: dict) -> list[dict]:
|
||||
"""把缺失技能按前置关系拓扑排序分阶段（Kahn 分层）。
|
||||
|
||||
前置关系取自技能词表 prereqs；前置技能若已掌握或不在缺失集中则视为已满足。
|
||||
每个阶段附带结构化验收标准（产出物 / 量化验收项 / 自测问答），按技能类别生成。
|
||||
"""
|
||||
prereq = {s["name"]: [p for p in s.get("prereqs", []) if p in missing]
|
||||
for s in lexicon["skills"] if s["name"] in missing}
|
||||
cat_of = {s["name"]: s.get("category", "") for s in lexicon["skills"]}
|
||||
remaining = set(missing)
|
||||
stages: list[dict] = []
|
||||
stage_no = 1
|
||||
while remaining:
|
||||
ready = sorted(s for s in remaining if not [p for p in prereq.get(s, []) if p in remaining])
|
||||
if not ready: # 有环兜底：按字典序放出，保证终止
|
||||
ready = sorted(remaining)
|
||||
stages.append({"stage": stage_no, "skills": ready,
|
||||
"goal": f"第 {stage_no} 阶段：掌握 {'、'.join(ready)}",
|
||||
"acceptance": build_acceptance(ready, cat_of)})
|
||||
remaining -= set(ready)
|
||||
stage_no += 1
|
||||
return stages
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 验收标准模板
|
||||
# 按技能类别给出差异化的验收过程：产出物、量化验收项、自测问答。
|
||||
# {skills} 占位为本阶段技能名（顿号连接），{s1}/{s2} 为前两个技能。
|
||||
_ACCEPTANCE: dict[str, dict] = {
|
||||
"编程语言": {
|
||||
"deliverable": "用 {skills} 独立完成一个 300 行以上、可运行的小工具或 CLI 程序，源码提交仓库并附 README",
|
||||
"criteria": ["覆盖核心语法与常用标准库，代码可一次性运行无报错",
|
||||
"编写不少于 5 个单元测试用例并全部通过",
|
||||
"README 说明设计取舍与至少 2 个踩坑记录",
|
||||
"代码符合该语言主流风格规范（通过对应 linter 检查）"],
|
||||
"self_check": ["{s1} 与一门相近语言在类型与内存模型上的核心差异是什么？",
|
||||
"{s1} 最常见的 3 个性能陷阱是什么，如何规避？",
|
||||
"如果把这个工具的数据规模放大 100 倍，瓶颈会先出现在哪里？"],
|
||||
},
|
||||
"后端框架": {
|
||||
"deliverable": "基于 {skills} 实现一个不少于 3 个接口的 REST 服务，含鉴权与参数校验",
|
||||
"criteria": ["分层清晰（路由/服务/数据访问分离），无不合理耦合",
|
||||
"关键接口有自动化测试，覆盖率不低于 60%",
|
||||
"完成一次简单压测并记录 QPS 与耗时分布",
|
||||
"能手绘并讲解一次请求的完整生命周期"],
|
||||
"self_check": ["{s1} 的依赖注入/中间件机制是如何工作的？",
|
||||
"生产环境下 {s1} 服务如何做优雅停机与连接管理？",
|
||||
"哪些场景下你会放弃 {s1} 而选择更轻的框架？"],
|
||||
},
|
||||
"前端": {
|
||||
"deliverable": "用 {skills} 完成一个可运行的单页应用（不少于 2 个页面），公开访问或录屏演示",
|
||||
"criteria": ["组件拆分合理、状态管理方案有明确选型理由",
|
||||
"主流浏览器无 console 报错，页面响应式适配两种宽度",
|
||||
"完成一次构建优化（分包/懒加载），记录优化前后体积",
|
||||
"核心交互有至少 3 个组件级测试"],
|
||||
"self_check": ["{s1} 的渲染/更新机制是怎样的，什么操作会触发不必要的重渲染？",
|
||||
"状态提升与全局状态各自适合什么场景？",
|
||||
"首屏白屏时间可以从哪几个环节优化？"],
|
||||
},
|
||||
"数据库": {
|
||||
"deliverable": "围绕 {skills} 设计并实现一个业务库表方案与查询集，附索引设计说明",
|
||||
"criteria": ["给出 ER 图与范式/反范式取舍的书面说明",
|
||||
"用 EXPLAIN 分析并优化至少 2 条慢查询，记录前后对比",
|
||||
"通过实验验证事务隔离级别的差异（附复现 SQL）",
|
||||
"完成一次备份与恢复演练"],
|
||||
"self_check": ["什么情况下索引会失效？各举一个例子",
|
||||
"{s1} 的 redo/undo 或持久化机制保证了什么？",
|
||||
"千万级数据量下，分页查询如何优化？"],
|
||||
},
|
||||
"中间件": {
|
||||
"deliverable": "本地搭建 {skills} 环境并跑通生产/消费（或读写）完整 demo",
|
||||
"criteria": ["给出关键配置项（持久化/高可用/过期策略）的逐项说明",
|
||||
"演练消息丢失、重复消费或缓存穿透场景并给出防护方案",
|
||||
"接入监控并解读 3 个核心指标",
|
||||
"整理一份与同类中间件的选型对比表"],
|
||||
"self_check": ["{s1} 在什么条件下会丢数据？如何避免？",
|
||||
"集群模式下数据是如何分布与迁移的？",
|
||||
"什么业务场景下不该使用 {s1}？"],
|
||||
},
|
||||
"大数据": {
|
||||
"deliverable": "用 {skills} 跑通一个批处理或流处理作业，输入输出可复现",
|
||||
"criteria": ["说明分区与并行度策略及其依据",
|
||||
"制造并解决一次数据倾斜，记录处理前后耗时",
|
||||
"给出作业的容错与重跑方案",
|
||||
"输出结果有数据质量校验（行数/空值/分布）"],
|
||||
"self_check": ["{s1} 的 shuffle 过程发生了什么，为什么昂贵？",
|
||||
"批处理与流处理的语义差异体现在哪里？",
|
||||
"如何判断该作业该垂直扩容还是水平扩容？"],
|
||||
},
|
||||
"AI算法": {
|
||||
"deliverable": "基于 {skills} 复现一个 baseline 模型或算法 demo，附实验记录",
|
||||
"criteria": ["数据集划分规范（训练/验证/测试），杜绝数据泄漏",
|
||||
"输出完整指标报告（P/R/F1 或 Loss/Acc 曲线）",
|
||||
"完成至少 1 组消融或调参对比并记录结论",
|
||||
"能讲清模型的适用边界与失败案例"],
|
||||
"self_check": ["如何判断模型过拟合？你用了哪些手段缓解？",
|
||||
"指标不变差的前提下，推理成本可以怎么降？",
|
||||
"{s1} 与传统方法相比，优势和代价各是什么？"],
|
||||
},
|
||||
"云原生": {
|
||||
"deliverable": "将一个应用容器化并部署到 {skills} 环境，完成一次滚动更新与回滚",
|
||||
"criteria": ["Dockerfile 使用多阶段构建，镜像体积有优化记录",
|
||||
"编排配置包含资源限额、健康探针与副本策略",
|
||||
"演练滚动更新与回滚全过程并截图/录屏",
|
||||
"日志与监控接入，能定位一次人为注入的故障"],
|
||||
"self_check": ["三种探针的区别是什么，配错会导致什么后果？",
|
||||
"有状态服务上 {s1} 要注意什么？",
|
||||
"镜像分层缓存如何影响 CI 构建速度？"],
|
||||
},
|
||||
"运维": {
|
||||
"deliverable": "用 {skills} 实现部署/巡检/告警中的至少一个自动化工具并投入使用演示",
|
||||
"criteria": ["工具覆盖部署、监控、告警三个环节中的至少两个",
|
||||
"完成一次故障注入演练（如杀进程/打满磁盘）并记录处置时长",
|
||||
"编写运行手册（部署步骤、回滚步骤、常见告警处置）",
|
||||
"脚本具备幂等性，重复执行不产生副作用"],
|
||||
"self_check": ["负载飙升时你的排查顺序是什么？",
|
||||
"监控覆盖率和告警疲劳之间怎么平衡？",
|
||||
"{s1} 自动化过程中最需要防的人为风险是什么？"],
|
||||
},
|
||||
"测试": {
|
||||
"deliverable": "为一个真实模块编写测试计划与自动化用例集，并在 CI 中跑通",
|
||||
"criteria": ["用例设计覆盖等价类、边界值与异常路径，附设计说明",
|
||||
"自动化用例不少于 10 条，CI 触发全绿",
|
||||
"提交至少 2 份规范缺陷报告（复现步骤/期望/实际/环境）",
|
||||
"输出覆盖率报告并说明未覆盖部分的风险"],
|
||||
"self_check": ["如何为一个你从未见过的系统设计测试策略？",
|
||||
"自动化测试的投入产出比怎么评估？",
|
||||
"线上问题复盘时，测试环节应该如何改进？"],
|
||||
},
|
||||
"数据分析": {
|
||||
"deliverable": "围绕 {skills} 完成一份端到端数据分析报告（数据获取→清洗→分析→结论）",
|
||||
"criteria": ["清洗与处理过程代码化、可一键复现",
|
||||
"给出不少于 3 个有业务含义的洞察，每个都有数据支撑",
|
||||
"图表选型恰当，坐标轴与口径标注完整",
|
||||
"明确写出结论的适用范围与可能的偏差来源"],
|
||||
"self_check": ["什么是辛普森悖论？如何在分析中规避？",
|
||||
"相关关系和因果关系的差异对结论有什么影响？",
|
||||
"如果数据缺失 30%，你的处理策略是什么？"],
|
||||
},
|
||||
"产品": {
|
||||
"deliverable": "围绕 {skills} 输出一份完整 PRD 或实验方案（含原型或流程图）",
|
||||
"criteria": ["明确定义北极星指标与至少 2 个护栏指标",
|
||||
"实验类方案需给出样本量估算、周期与显著性判定规则",
|
||||
"包含灰度发布与回滚预案",
|
||||
"预设结果判定规则：什么数据下继续/迭代/放弃"],
|
||||
"self_check": ["A/A 测试的目的是什么？不做会有什么风险？",
|
||||
"p 值和置信区间分别说明什么，常见误用有哪些？",
|
||||
"指标提升但用户体验下降时，决策依据是什么？"],
|
||||
},
|
||||
"嵌入式": {
|
||||
"deliverable": "基于 {skills} 完成一个可烧录运行的固件 demo",
|
||||
"criteria": ["完成交叉编译与烧录，功能演示正常",
|
||||
"给出关键中断/时序的分析记录",
|
||||
"测量并记录功耗或资源占用数据",
|
||||
"代码有看门狗与异常恢复处理"],
|
||||
"self_check": ["裸机与 RTOS 的核心差异是什么，何时必须上 RTOS？",
|
||||
"如何定位一次偶发的硬件通信失败？",
|
||||
"Flash/RAM 受限时的优化手段有哪些？"],
|
||||
},
|
||||
"物联网": {
|
||||
"deliverable": "用 {skills} 搭建一条设备数据采集→上报→可视化的端到端链路",
|
||||
"criteria": ["说明通信协议选型理由（如 MQTT/CoAP/HTTP）",
|
||||
"实现断线重连与本地缓存补传",
|
||||
"完成弱网/断电场景下的鲁棒性演练",
|
||||
"数据可视化看板可实时刷新"],
|
||||
"self_check": ["弱网环境下消息的可靠性与时延如何权衡？",
|
||||
"设备身份认证与防伪造怎么做？",
|
||||
"海量设备接入时，架构的瓶颈会先出现在哪？"],
|
||||
},
|
||||
}
|
||||
|
||||
_DEFAULT_ACCEPTANCE = {
|
||||
"deliverable": "独立完成一个综合运用 {skills} 的迷你项目，源码与说明文档齐全",
|
||||
"criteria": ["项目可运行、核心流程演示无阻塞",
|
||||
"能完整讲解 {s1} 的核心原理与常见坑",
|
||||
"整理不少于 5 条学习笔记或踩坑记录",
|
||||
"向他人做一次 10 分钟分享并通过提问环节"],
|
||||
"self_check": ["{s1} 解决了什么问题？没有它时人们怎么做？",
|
||||
"学习 {s1} 过程中最反直觉的一点是什么？",
|
||||
"如何把本阶段技能组合应用到目标岗位的真实场景中？"],
|
||||
}
|
||||
|
||||
|
||||
def build_acceptance(skills: list[str], cat_of: dict[str, str]) -> dict:
|
||||
"""按阶段内技能的主导类别选择模板，生成结构化验收标准。"""
|
||||
cats = [cat_of.get(s, "") for s in skills if cat_of.get(s)]
|
||||
if cats:
|
||||
# 主导类别（并列时取阶段内先出现的技能类别）
|
||||
cat = max(cats, key=lambda c: (cats.count(c), -cats.index(c)))
|
||||
else:
|
||||
cat = ""
|
||||
tpl = _ACCEPTANCE.get(cat, _DEFAULT_ACCEPTANCE)
|
||||
s1 = skills[0]
|
||||
s2 = skills[1] if len(skills) > 1 else skills[0]
|
||||
joined = "、".join(skills)
|
||||
|
||||
def fmt(text: str) -> str:
|
||||
return text.format(skills=joined, s1=s1, s2=s2)
|
||||
|
||||
return {
|
||||
"category": cat or "通用",
|
||||
"deliverable": fmt(tpl["deliverable"]),
|
||||
"criteria": [fmt(c) for c in tpl["criteria"]],
|
||||
"self_check": [fmt(q) for q in tpl["self_check"]],
|
||||
}
|
||||
|
||||
|
||||
class MatchEngine:
|
||||
"""人岗匹配引擎。"""
|
||||
|
||||
def __init__(self, lexicon: dict | None = None, weights: dict | None = None,
|
||||
threshold: float = 60.0):
|
||||
self.lexicon = lexicon or load_lexicon()
|
||||
self.weights = {**DEFAULT_WEIGHTS, **(weights or {})}
|
||||
self.threshold = threshold
|
||||
self.category_of = {s["name"]: s.get("category", "") for s in self.lexicon["skills"]}
|
||||
|
||||
def match(self, profile: ResumeProfile, job: JobDefinition) -> MatchResult:
|
||||
have = set(profile.skills)
|
||||
m_req, miss_req, cov_req = _coverage(have, job.required_skills)
|
||||
m_pref, miss_pref, cov_pref = _coverage(have, job.preferred_skills)
|
||||
exp_score = min(1.0, profile.years / max(1, job.expected_years))
|
||||
scen_matched = [s for s in job.scenarios if s in set(profile.scenarios)]
|
||||
cov_scen = len(scen_matched) / len(job.scenarios) if job.scenarios else 1.0
|
||||
|
||||
dims = {
|
||||
"required": round(100 * cov_req, 2),
|
||||
"preferred": round(100 * cov_pref, 2),
|
||||
"experience": round(100 * exp_score, 2),
|
||||
"scenario": round(100 * cov_scen, 2),
|
||||
}
|
||||
total = round(100 * (
|
||||
self.weights["required"] * cov_req
|
||||
+ self.weights["preferred"] * cov_pref
|
||||
+ self.weights["experience"] * exp_score
|
||||
+ self.weights["scenario"] * cov_scen
|
||||
), 2)
|
||||
|
||||
# 可迁移技能：与缺失必备技能同类别、且候选人已掌握的技能
|
||||
miss_cats = {self.category_of.get(s, "") for s in miss_req} - {""}
|
||||
transferable = sorted(s for s in have
|
||||
if self.category_of.get(s, "") in miss_cats
|
||||
and s not in job.required_skills)
|
||||
|
||||
path = topological_stages(miss_req + miss_pref, self.lexicon)
|
||||
return MatchResult(
|
||||
job_id=job.job_id,
|
||||
job_title=job.title,
|
||||
total_score=total,
|
||||
dimension_scores=dims,
|
||||
matched_required=m_req,
|
||||
missing_required=miss_req,
|
||||
matched_preferred=m_pref,
|
||||
missing_preferred=miss_pref,
|
||||
transferable_skills=transferable,
|
||||
learning_path=path,
|
||||
is_match=total >= self.threshold,
|
||||
)
|
||||
@@ -0,0 +1,151 @@
|
||||
"""简历解析：.txt 直接读，.docx 用 python-docx，.pdf 用 pypdf。
|
||||
|
||||
解析后做四件事：脱敏（姓名/手机/邮箱/住址打码）、技能抽取并归一化到词表、
|
||||
工作年限与场景识别、结构化信息抽取（性别/学历/住址/分节/证件照），
|
||||
供规范简历模板展示与导出。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from app.extract.base import SkillMatcher, load_lexicon
|
||||
from app.models import ResumeProfile
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_PHONE_RE = re.compile(r"1[3-9]\d{9}")
|
||||
_EMAIL_RE = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
_NAME_RE = re.compile(r"姓名[:：]\s*([一-龥]{2,4})")
|
||||
_YEARS_RE = re.compile(r"(\d+(?:\.\d+)?)\s*年(?:以上)?[^\n，。；]{0,6}?经验")
|
||||
_GENDER_RE = re.compile(r"性别[:：]?\s*(男|女)")
|
||||
_EDU_RE = re.compile(r"(博士后|博士|硕士|研究生|MBA|本科|大专|专科)")
|
||||
_ADDR_RE = re.compile(r"(?:家庭住址|现居(?:住)?(?:地|址)?|联系地址|通讯地址|籍贯)[:：]\s*([^\n，。；]{2,30})")
|
||||
|
||||
# 常见简历分节标题（整行匹配，可带冒号）
|
||||
_SECTION_TITLES = {
|
||||
"个人信息", "基本信息", "求职意向", "教育经历", "教育背景",
|
||||
"工作经历", "工作经验", "实习经历", "项目经历", "项目经验",
|
||||
"技能清单", "专业技能", "核心技能", "技能特长", "技能",
|
||||
"自我评价", "个人评价", "荣誉证书", "荣誉奖项", "证书",
|
||||
}
|
||||
|
||||
|
||||
def mask_text(text: str) -> str:
|
||||
"""脱敏：手机、邮箱、姓名、住址打码。"""
|
||||
text = _PHONE_RE.sub(lambda m: m.group(0)[:3] + "****" + m.group(0)[-2:], text)
|
||||
text = _EMAIL_RE.sub(lambda m: m.group(0)[0] + "***@" + m.group(0).split("@")[1], text)
|
||||
text = _NAME_RE.sub(lambda m: "姓名：" + m.group(1)[0] + "*" * (len(m.group(1)) - 1), text)
|
||||
text = _ADDR_RE.sub(lambda m: m.group(0)[:2] + m.group(1)[:6] + "***"
|
||||
if len(m.group(1)) > 6 else m.group(0), text)
|
||||
return text
|
||||
|
||||
|
||||
def split_sections(text: str) -> list[dict]:
|
||||
"""按常见标题把简历正文分节，返回 [{title, text}]；无标题时整体归入"简历全文"。"""
|
||||
sections: list[dict] = []
|
||||
cur_title, cur_lines = "基本信息", []
|
||||
for ln in text.splitlines():
|
||||
key = ln.strip().rstrip(":：").strip()
|
||||
if key in _SECTION_TITLES:
|
||||
if "\n".join(cur_lines).strip():
|
||||
sections.append({"title": cur_title, "text": "\n".join(cur_lines).strip()})
|
||||
cur_title, cur_lines = key, []
|
||||
else:
|
||||
cur_lines.append(ln)
|
||||
if "\n".join(cur_lines).strip():
|
||||
sections.append({"title": cur_title, "text": "\n".join(cur_lines).strip()})
|
||||
if not sections and text.strip():
|
||||
sections = [{"title": "简历全文", "text": text.strip()}]
|
||||
return sections
|
||||
|
||||
|
||||
def extract_photo_b64(path: str | Path) -> str:
|
||||
"""从 docx/pdf 中提取第一张图片作为证件照，返回 data URI；无图片返回空串。"""
|
||||
path = Path(path)
|
||||
suffix = path.suffix.lower()
|
||||
try:
|
||||
if suffix == ".docx":
|
||||
import zipfile
|
||||
with zipfile.ZipFile(str(path)) as z:
|
||||
names = sorted(n for n in z.namelist()
|
||||
if n.startswith("word/media/")
|
||||
and n.lower().endswith((".png", ".jpg", ".jpeg")))
|
||||
if not names:
|
||||
return ""
|
||||
data = z.read(names[0])
|
||||
mime = "image/png" if names[0].lower().endswith(".png") else "image/jpeg"
|
||||
return f"data:{mime};base64,{base64.b64encode(data).decode()}"
|
||||
if suffix == ".pdf":
|
||||
from pypdf import PdfReader
|
||||
reader = PdfReader(str(path))
|
||||
for page in reader.pages[:2]:
|
||||
for img in getattr(page, "images", []):
|
||||
mime = "image/png" if img.name.lower().endswith(".png") else "image/jpeg"
|
||||
return f"data:{mime};base64,{base64.b64encode(img.data).decode()}"
|
||||
except Exception as exc: # 照片是增强信息，提取失败不阻断解析
|
||||
logger.warning("简历照片提取失败（已跳过）: %s", exc)
|
||||
return ""
|
||||
|
||||
|
||||
def read_resume_file(path: str | Path) -> str:
|
||||
"""按扩展名读取简历文件文本。"""
|
||||
path = Path(path)
|
||||
suffix = path.suffix.lower()
|
||||
if suffix == ".txt":
|
||||
return path.read_text(encoding="utf-8")
|
||||
if suffix == ".docx":
|
||||
from docx import Document # python-docx，requirements 已声明
|
||||
doc = Document(str(path))
|
||||
return "\n".join(p.text for p in doc.paragraphs)
|
||||
if suffix == ".pdf":
|
||||
try:
|
||||
from pypdf import PdfReader
|
||||
except ImportError as exc:
|
||||
raise RuntimeError(
|
||||
"解析 .pdf 需要可选依赖 pypdf，请先执行：pip install pypdf"
|
||||
) from exc
|
||||
reader = PdfReader(str(path))
|
||||
return "\n".join(page.extract_text() or "" for page in reader.pages)
|
||||
raise ValueError(f"不支持的简历格式：{suffix}（支持 .txt/.docx/.pdf）")
|
||||
|
||||
|
||||
def _mask_addr(addr: str) -> str:
|
||||
"""住址脱敏：保留前 6 字（大致到区/县），其余打码。"""
|
||||
return addr[:6] + "***" if len(addr) > 6 else addr
|
||||
|
||||
|
||||
def parse_resume(text: str, matcher: SkillMatcher | None = None,
|
||||
photo_b64: str = "") -> ResumeProfile:
|
||||
"""从简历文本解析脱敏画像（含结构化基本信息与分节）。"""
|
||||
matcher = matcher or SkillMatcher(load_lexicon())
|
||||
name_m = _NAME_RE.search(text)
|
||||
phone_m = _PHONE_RE.search(text)
|
||||
email_m = _EMAIL_RE.search(text)
|
||||
years_m = _YEARS_RE.search(text)
|
||||
gender_m = _GENDER_RE.search(text)
|
||||
edu_m = _EDU_RE.search(text)
|
||||
addr_m = _ADDR_RE.search(text)
|
||||
masked = mask_text(text)
|
||||
return ResumeProfile(
|
||||
name=(name_m.group(1)[0] + "*" * (len(name_m.group(1)) - 1)) if name_m else "",
|
||||
phone=(phone_m.group(0)[:3] + "****" + phone_m.group(0)[-2:]) if phone_m else "",
|
||||
email=(email_m.group(0)[0] + "***@" + email_m.group(0).split("@")[1]) if email_m else "",
|
||||
years=float(years_m.group(1)) if years_m else 0.0,
|
||||
skills=matcher.find_names(text),
|
||||
scenarios=matcher.find_scenarios(text),
|
||||
raw_masked_text=masked,
|
||||
gender=gender_m.group(1) if gender_m else "",
|
||||
education=edu_m.group(1) if edu_m else "",
|
||||
address=_mask_addr(addr_m.group(1).strip()) if addr_m else "",
|
||||
photo_b64=photo_b64,
|
||||
sections=split_sections(masked),
|
||||
)
|
||||
|
||||
|
||||
def parse_resume_file(path: str | Path, matcher: SkillMatcher | None = None) -> ResumeProfile:
|
||||
"""读取文件并解析（含证件照提取）。"""
|
||||
return parse_resume(read_resume_file(path), matcher,
|
||||
photo_b64=extract_photo_b64(path))
|
||||
@@ -0,0 +1,133 @@
|
||||
"""规范简历 DOCX 导出：照片 + 基本信息表 + 分节正文（关键词高亮）+ 可选匹配诊断。
|
||||
|
||||
完全离线，仅依赖 python-docx。返回 bytes，由 API 层以文件流下发。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import base64
|
||||
import io
|
||||
import re
|
||||
|
||||
from docx import Document
|
||||
from docx.enum.text import WD_ALIGN_PARAGRAPH
|
||||
from docx.oxml.ns import qn
|
||||
from docx.shared import Cm, Pt, RGBColor
|
||||
|
||||
from app.models import ResumeProfile
|
||||
|
||||
_GREEN = RGBColor(0x00, 0xB4, 0x2A) # 命中必备
|
||||
_BLUE = RGBColor(0x2F, 0x54, 0xEB) # 命中加分
|
||||
_GRAY = RGBColor(0x4E, 0x59, 0x69) # 其他技能
|
||||
_DARK = RGBColor(0x1D, 0x21, 0x29)
|
||||
|
||||
|
||||
def _set_cn_font(doc: Document) -> None:
|
||||
st = doc.styles["Normal"]
|
||||
st.font.name = "Times New Roman"
|
||||
st.font.size = Pt(10.5)
|
||||
st.element.rPr.rFonts.set(qn("w:eastAsia"), "宋体")
|
||||
|
||||
|
||||
def _highlight_runs(paragraph, text: str, cls_of: dict[str, RGBColor]) -> None:
|
||||
"""按技能清单把 text 切段写入段落，命中技能加粗上色。"""
|
||||
if not cls_of:
|
||||
paragraph.add_run(text)
|
||||
return
|
||||
names = sorted(cls_of, key=len, reverse=True)
|
||||
pat = re.compile("|".join(re.escape(n) for n in names), re.IGNORECASE)
|
||||
pos = 0
|
||||
for m in pat.finditer(text):
|
||||
if m.start() > pos:
|
||||
paragraph.add_run(text[pos:m.start()])
|
||||
hit = m.group(0)
|
||||
run = paragraph.add_run(hit)
|
||||
run.bold = True
|
||||
run.font.color.rgb = cls_of.get(hit.lower(), _DARK)
|
||||
pos = m.end()
|
||||
if pos < len(text):
|
||||
paragraph.add_run(text[pos:])
|
||||
|
||||
|
||||
def build_resume_docx(profile: ResumeProfile, match: dict | None = None) -> bytes:
|
||||
"""把脱敏简历画像渲染为规范 DOCX，返回文件 bytes。"""
|
||||
doc = Document()
|
||||
_set_cn_font(doc)
|
||||
|
||||
title = doc.add_paragraph()
|
||||
title.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
||||
trun = title.add_run("个 人 简 历")
|
||||
trun.bold = True
|
||||
trun.font.size = Pt(20)
|
||||
sub = doc.add_paragraph()
|
||||
sub.alignment = WD_ALIGN_PARAGRAPH.CENTER
|
||||
srun = sub.add_run("（CareerNexus 职联 · 结构化解析导出，已自动脱敏）")
|
||||
srun.font.size = Pt(9)
|
||||
srun.font.color.rgb = _GRAY
|
||||
|
||||
# ---- 头部：左照片 + 右基本信息表 ----
|
||||
head = doc.add_table(rows=1, cols=2)
|
||||
head.style = "Table Grid"
|
||||
head.columns[0].width = Cm(3.4)
|
||||
head.columns[1].width = Cm(13.4)
|
||||
photo_cell, info_cell = head.cell(0, 0), head.cell(0, 1)
|
||||
if profile.photo_b64:
|
||||
try:
|
||||
raw = base64.b64decode(profile.photo_b64.split(",", 1)[1])
|
||||
photo_cell.paragraphs[0].add_run().add_picture(io.BytesIO(raw), width=Cm(3.0))
|
||||
except Exception:
|
||||
photo_cell.paragraphs[0].add_run("（照片解析失败）")
|
||||
else:
|
||||
photo_cell.paragraphs[0].add_run("（未提供照片）")
|
||||
|
||||
info_items = [
|
||||
("姓名", profile.name or "—"), ("性别", profile.gender or "—"),
|
||||
("学历", profile.education or "—"), ("工作年限", f"{profile.years:g} 年" if profile.years else "—"),
|
||||
("手机", profile.phone or "—"), ("邮箱", profile.email or "—"),
|
||||
("家庭住址", profile.address or "—"),
|
||||
]
|
||||
info_table = info_cell.add_table(rows=(len(info_items) + 1) // 2, cols=2)
|
||||
for i, (k, v) in enumerate(info_items):
|
||||
cell = info_table.cell(i // 2, i % 2)
|
||||
p = cell.paragraphs[0]
|
||||
kr = p.add_run(f"{k}：")
|
||||
kr.bold = True
|
||||
p.add_run(v)
|
||||
|
||||
# 技能高亮类别（小写键）
|
||||
cls_of: dict[str, RGBColor] = {}
|
||||
if match:
|
||||
for s in match.get("matched_required", []):
|
||||
cls_of[s.lower()] = _GREEN
|
||||
for s in match.get("matched_preferred", []):
|
||||
cls_of[s.lower()] = _BLUE
|
||||
for s in profile.skills:
|
||||
cls_of.setdefault(s.lower(), _GRAY)
|
||||
|
||||
# ---- 分节正文（关键词标注） ----
|
||||
for sec in profile.sections or [{"title": "简历全文", "text": profile.raw_masked_text}]:
|
||||
h = doc.add_paragraph()
|
||||
hrun = h.add_run(f"▍{sec['title']}")
|
||||
hrun.bold = True
|
||||
hrun.font.size = Pt(13)
|
||||
hrun.font.color.rgb = _BLUE
|
||||
_highlight_runs(doc.add_paragraph(), sec["text"], cls_of)
|
||||
|
||||
# ---- 匹配诊断摘要（可选） ----
|
||||
if match and match.get("job_title"):
|
||||
h = doc.add_paragraph()
|
||||
hrun = h.add_run("▍人岗匹配诊断（CareerNexus）")
|
||||
hrun.bold = True
|
||||
hrun.font.size = Pt(13)
|
||||
hrun.font.color.rgb = _BLUE
|
||||
for line in [
|
||||
f"目标岗位：{match['job_title']} 综合匹配分：{match.get('total_score', 0)}",
|
||||
f"命中必备技能：{'、'.join(match.get('matched_required', [])) or '无'}",
|
||||
f"命中加分技能：{'、'.join(match.get('matched_preferred', [])) or '无'}",
|
||||
f"缺失技能：{'、'.join((match.get('missing_required', [])
|
||||
+ match.get('missing_preferred', []))) or '无'}",
|
||||
]:
|
||||
doc.add_paragraph(line)
|
||||
|
||||
buf = io.BytesIO()
|
||||
doc.save(buf)
|
||||
return buf.getvalue()
|
||||
@@ -0,0 +1,163 @@
|
||||
"""语义向量通道（可选增强）：本地 BGE-M3 权重 或 联网 Embedding API 双模式。
|
||||
|
||||
- provider=local：sentence-transformers 加载本地权重（如 models/bge-m3）；
|
||||
- provider=api：OpenAI 兼容 /embeddings 接口（默认阿里 DashScope 兼容模式，
|
||||
模型 text-embedding-v4，填 API Key 即可，无需下载权重）；
|
||||
- 任一模式就绪即启用语义召回；均不可用时回退词表归一化，行为与无此模块一致；
|
||||
- 依赖与权重均为可选安装，不进入 requirements（"零重依赖、离线优先"约束）。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import logging
|
||||
from pathlib import Path
|
||||
|
||||
import httpx
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# BGE-M3 目录的就绪判定：配置文件 + 任一权重文件 + 分词器
|
||||
_REQUIRED_CONFIG = "config.json"
|
||||
_WEIGHT_FILES = ("model.safetensors", "pytorch_model.bin")
|
||||
_TOKENIZER_FILES = ("tokenizer.json", "vocab.txt")
|
||||
|
||||
DEFAULT_API_BASE = "https://dashscope.aliyuncs.com/compatible-mode/v1"
|
||||
DEFAULT_API_MODEL = "text-embedding-v4"
|
||||
|
||||
_model = None # 懒加载的本地 SentenceTransformer 实例
|
||||
_model_path = ""
|
||||
_skill_emb = None # 词表技能向量缓存
|
||||
_skill_names: list[str] = []
|
||||
_emb_cache_key = "" # 缓存对应的通道标识（provider+路径/模型）
|
||||
|
||||
|
||||
def deps_installed() -> bool:
|
||||
"""sentence-transformers 是否已安装。"""
|
||||
return importlib.util.find_spec("sentence_transformers") is not None
|
||||
|
||||
|
||||
def resolve_weights_path(path: str | Path) -> Path:
|
||||
"""权重路径解析：相对路径相对工程根目录解析，绝对路径原样返回（兼容既有配置）。"""
|
||||
from app.config import ROOT
|
||||
p = Path(str(path or ""))
|
||||
return p if p.is_absolute() else ROOT / p
|
||||
|
||||
|
||||
def weights_present(path: str | Path) -> bool:
|
||||
"""权重目录是否包含 BGE-M3 所需文件。"""
|
||||
p = Path(str(path or ""))
|
||||
if not p.is_dir() or not (p / _REQUIRED_CONFIG).exists():
|
||||
return False
|
||||
if not any((p / f).exists() for f in _WEIGHT_FILES):
|
||||
return False
|
||||
return any((p / f).exists() for f in _TOKENIZER_FILES)
|
||||
|
||||
|
||||
def status(cfg: dict) -> dict:
|
||||
"""语义通道状态：供 /api/health 与 /api/config 展示。"""
|
||||
provider = str(cfg.get("embed_provider") or "local").strip() or "local"
|
||||
path = str(cfg.get("embed_model_path") or "").strip()
|
||||
api_key = str(cfg.get("embed_api_key") or "").strip()
|
||||
present = weights_present(resolve_weights_path(path)) if path else False
|
||||
deps = deps_installed()
|
||||
local_ready = bool(path and present and deps)
|
||||
api_ready = bool(api_key)
|
||||
ready = api_ready if provider == "api" else local_ready
|
||||
return {
|
||||
"provider": provider,
|
||||
"configured": bool(path) if provider == "local" else api_ready,
|
||||
"path": path,
|
||||
"weights_present": present,
|
||||
"deps_installed": deps,
|
||||
"local_ready": local_ready,
|
||||
"api_key_masked": ("••••" + api_key[-4:]) if api_key else "",
|
||||
"api_base_url": str(cfg.get("embed_api_base_url") or DEFAULT_API_BASE),
|
||||
"api_model": str(cfg.get("embed_api_model") or DEFAULT_API_MODEL),
|
||||
"ready": bool(ready),
|
||||
}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 本地模式
|
||||
def _load_local(path: str):
|
||||
"""懒加载本地模型；不可用时返回 None。"""
|
||||
global _model, _model_path
|
||||
if _model is not None and _model_path == path:
|
||||
return _model
|
||||
try:
|
||||
from sentence_transformers import SentenceTransformer # 可选依赖
|
||||
_model = SentenceTransformer(path)
|
||||
_model_path = path
|
||||
logger.info("语义向量模型已加载（本地）: %s", path)
|
||||
except Exception as exc: # noqa: BLE001 - 模型加载失败必须可回退
|
||||
logger.warning("本地语义模型加载失败，回退词表归一化: %s", exc)
|
||||
_model = None
|
||||
return None
|
||||
return _model
|
||||
|
||||
|
||||
def _embed_local(texts: list[str], path: str):
|
||||
model = _load_local(path)
|
||||
if model is None:
|
||||
return None
|
||||
return model.encode(texts, normalize_embeddings=True)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 联网模式
|
||||
def _embed_api(texts: list[str], cfg: dict):
|
||||
"""调用 OpenAI 兼容 embeddings 接口（DashScope 等），返回向量列表。"""
|
||||
st = status(cfg)
|
||||
try:
|
||||
resp = httpx.post(
|
||||
f"{st['api_base_url'].rstrip('/')}/embeddings",
|
||||
headers={"Authorization": f"Bearer {cfg.get('embed_api_key')}"},
|
||||
json={"model": st["api_model"], "input": texts},
|
||||
timeout=30,
|
||||
)
|
||||
resp.raise_for_status()
|
||||
data = resp.json()["data"]
|
||||
data.sort(key=lambda d: d["index"])
|
||||
import math
|
||||
vecs = []
|
||||
for d in data:
|
||||
v = d["embedding"]
|
||||
norm = math.sqrt(sum(x * x for x in v)) or 1.0
|
||||
vecs.append([x / norm for x in v])
|
||||
return vecs
|
||||
except Exception as exc: # noqa: BLE001 - 网络/鉴权失败必须可回退
|
||||
logger.warning("Embedding API 调用失败，回退词表归一化: %s", exc)
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------- 统一入口
|
||||
def _embed(texts: list[str], cfg: dict):
|
||||
"""按 provider 取向量（已归一化）；失败返回 None。"""
|
||||
provider = str(cfg.get("embed_provider") or "local").strip() or "local"
|
||||
if provider == "api":
|
||||
return _embed_api(texts, cfg)
|
||||
return _embed_local(texts, str(resolve_weights_path(
|
||||
str(cfg.get("embed_model_path") or "").strip())))
|
||||
|
||||
|
||||
def most_similar(name: str, cfg: dict, lexicon: dict,
|
||||
threshold: float = 0.62) -> tuple[str, float] | None:
|
||||
"""返回词表中与 name 语义最近的技能 (标准名, 相似度)；未过阈值或不可用返回 None。"""
|
||||
global _skill_emb, _skill_names, _emb_cache_key
|
||||
st = status(cfg)
|
||||
if not st["ready"]:
|
||||
return None
|
||||
cache_key = f"{st['provider']}|{st['path'] or st['api_model']}"
|
||||
if _skill_emb is None or cache_key != _emb_cache_key:
|
||||
_skill_names = [s["name"] for s in lexicon["skills"]]
|
||||
_skill_emb = _embed(_skill_names, cfg)
|
||||
if _skill_emb is None:
|
||||
return None
|
||||
_emb_cache_key = cache_key
|
||||
vecs = _embed([name], cfg)
|
||||
if vecs is None:
|
||||
return None
|
||||
vec = vecs[0]
|
||||
scores = [sum(a * b for a, b in zip(row, vec)) for row in _skill_emb]
|
||||
best_i = max(range(len(scores)), key=lambda i: scores[i])
|
||||
if scores[best_i] >= threshold:
|
||||
return _skill_names[best_i], round(scores[best_i], 4)
|
||||
return None
|
||||
+139
@@ -0,0 +1,139 @@
|
||||
"""CareerNexus 全链路数据模型（pydantic v2）。
|
||||
|
||||
覆盖：JD 原始记录、结构化抽取结果、岗位定义、图谱观察、
|
||||
变更事件、人岗匹配结果、简历画像与人机协同审核记录。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from typing import Literal, Optional
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class JDRecord(BaseModel):
|
||||
"""一条招聘 JD 原始记录。source_id 由接入层分配（文件名#行号），不可变。"""
|
||||
|
||||
jid: str
|
||||
job_title: str
|
||||
company: str
|
||||
city: str = ""
|
||||
salary: str = ""
|
||||
industry: str = "" # 招聘平台标注的所属行业
|
||||
publish_date: date
|
||||
source: str = ""
|
||||
url: str = ""
|
||||
text: str
|
||||
source_id: str = "" # 形如 batch_2023.jsonl#17
|
||||
weight: float = 1.0 # 清洗阶段赋予的时间衰减权重
|
||||
is_duplicate: bool = False # 近重复（抄袭）降权标记
|
||||
dup_of: Optional[str] = None # 指向被保留的最早来源 jid
|
||||
is_noise: bool = False # 噪声（广告/无关条款）标记
|
||||
|
||||
|
||||
class ExtractionResult(BaseModel):
|
||||
"""从单条 JD 文本抽取出的结构化岗位定义。"""
|
||||
|
||||
jid: str = ""
|
||||
job_title: str = ""
|
||||
responsibilities: list[str] = Field(default_factory=list)
|
||||
required_skills: list[str] = Field(default_factory=list)
|
||||
preferred_skills: list[str] = Field(default_factory=list)
|
||||
scenarios: list[str] = Field(default_factory=list)
|
||||
evidence: dict[str, str] = Field(default_factory=dict) # 字段 -> 证据片段
|
||||
confidence: float = 0.0
|
||||
extractor: str = "offline"
|
||||
prompt_version: str = "offline-v1"
|
||||
|
||||
|
||||
class JobDefinition(BaseModel):
|
||||
"""聚合后的岗位定义（图谱中的 Job 节点属性）。"""
|
||||
|
||||
job_id: str
|
||||
title: str
|
||||
category: str = ""
|
||||
industries: list[str] = Field(default_factory=list) # 归属行业（1-2 个）
|
||||
required_skills: list[str] = Field(default_factory=list)
|
||||
preferred_skills: list[str] = Field(default_factory=list)
|
||||
responsibilities: list[str] = Field(default_factory=list)
|
||||
scenarios: list[str] = Field(default_factory=list)
|
||||
expected_years: int = 3
|
||||
status: Literal["active", "pending", "rejected"] = "active"
|
||||
source_count: int = 0 # 支撑 JD 数
|
||||
company_count: int = 0 # 独立企业数
|
||||
# 候选新岗位的支撑证据（新岗位发现时填写）：支撑 JD 的 source_id 与出现季度窗口，
|
||||
# 审核入库时用于生成真实的 REQUIRES 区间、来源列表与演化事件。
|
||||
support_source_ids: list[str] = Field(default_factory=list)
|
||||
support_windows: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class GraphObservation(BaseModel):
|
||||
"""一次岗位-技能关系观察（带时间窗，追加而不覆盖）。"""
|
||||
|
||||
job_id: str
|
||||
skill: str
|
||||
importance: Literal["required", "preferred"]
|
||||
weight: float = 1.0
|
||||
window: str = "" # 季度窗口，如 2022Q1
|
||||
source_ids: list[str] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ChangeEvent(BaseModel):
|
||||
"""岗位能力演化事件：新增 / 增强 / 减弱 / 退出。"""
|
||||
|
||||
event_id: str
|
||||
job_id: str
|
||||
skill: str
|
||||
change_type: Literal["新增", "增强", "减弱", "退出"]
|
||||
window: str
|
||||
prev_weight: float = 0.0
|
||||
curr_weight: float = 0.0
|
||||
source_ids: list[str] = Field(default_factory=list)
|
||||
description: str = ""
|
||||
created_at: str = ""
|
||||
|
||||
|
||||
class ResumeProfile(BaseModel):
|
||||
"""脱敏后的简历画像。"""
|
||||
|
||||
name: str = ""
|
||||
phone: str = ""
|
||||
email: str = ""
|
||||
years: float = 0.0
|
||||
skills: list[str] = Field(default_factory=list)
|
||||
scenarios: list[str] = Field(default_factory=list)
|
||||
raw_masked_text: str = ""
|
||||
# 结构化信息（规范简历模板用）
|
||||
gender: str = "" # 性别（不脱敏，人口统计学字段）
|
||||
education: str = "" # 最高学历（本科/硕士/博士…）
|
||||
address: str = "" # 家庭住址/现居地（脱敏到城市）
|
||||
photo_b64: str = "" # 证件照 base64（data URI，无照片为空）
|
||||
sections: list[dict] = Field(default_factory=list) # [{title, text}] 分节正文
|
||||
|
||||
|
||||
class MatchResult(BaseModel):
|
||||
"""人岗匹配结果：总分、四维分、命中/缺失/可迁移技能与学习路径。"""
|
||||
|
||||
job_id: str
|
||||
job_title: str = ""
|
||||
total_score: float = 0.0
|
||||
dimension_scores: dict[str, float] = Field(default_factory=dict)
|
||||
matched_required: list[str] = Field(default_factory=list)
|
||||
missing_required: list[str] = Field(default_factory=list)
|
||||
matched_preferred: list[str] = Field(default_factory=list)
|
||||
missing_preferred: list[str] = Field(default_factory=list)
|
||||
transferable_skills: list[str] = Field(default_factory=list)
|
||||
learning_path: list[dict] = Field(default_factory=list)
|
||||
is_match: bool = False
|
||||
|
||||
|
||||
class ReviewRecord(BaseModel):
|
||||
"""人机协同审核留痕。"""
|
||||
|
||||
candidate_id: str
|
||||
action: Literal["accept", "modify", "reject"]
|
||||
operator: str
|
||||
reason: str = ""
|
||||
before: dict = Field(default_factory=dict)
|
||||
after: dict = Field(default_factory=dict)
|
||||
time: str = ""
|
||||
@@ -0,0 +1,130 @@
|
||||
"""人工状态覆写层：审核入库岗位与节点编辑跨图谱重建持久化。
|
||||
|
||||
- 人工审核通过的岗位、节点编辑先写入 SQLite 状态库 ``job_overrides`` 表
|
||||
（见 app/storage/db.py），同时应用到当前图谱；
|
||||
- pipeline 全量重建图谱后调用 :func:`replay_overrides` 按序重放，
|
||||
人工入库岗位强制纳入正式图谱（不受"2023 年前出现"规则限制），
|
||||
节点编辑同样保留。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from app.graph.tkg import LEVEL_TO_YEARS, TemporalKG, job_level, new_event_id, now_str, quarter_of
|
||||
from app.models import ChangeEvent, GraphObservation, JobDefinition
|
||||
from app.storage import db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FALLBACK_SOURCE = "人工审核入库" # 无支撑证据的旧候选入库时的来源标记
|
||||
|
||||
|
||||
def normalize_job_id(job: JobDefinition) -> str:
|
||||
"""候选转正：cand_X 统一归一为 job_<岗位名>，保证全图只有一个岗位节点。"""
|
||||
if job.job_id.startswith("cand_"):
|
||||
return f"job_{job.title}"
|
||||
return job.job_id
|
||||
|
||||
|
||||
def apply_accepted_job(kg: TemporalKG, job: JobDefinition) -> None:
|
||||
"""把审核通过的岗位写入图谱：真实区间/来源的 REQUIRES 边、场景/行业边与"新增"演化事件。
|
||||
|
||||
区间自首个支持窗口起；来源取候选簇真实支撑 JD 的 source_id 列表
|
||||
（旧候选缺支撑证据时回退到"人工审核入库"标记，窗口取当前季度）。
|
||||
调用前须先把 job_id 归一化（见 normalize_job_id）；若图中残留旧候选节点则清理。
|
||||
"""
|
||||
job.status = "active"
|
||||
first_window = min(job.support_windows) if job.support_windows else quarter_of(datetime.now())
|
||||
sources = list(job.support_source_ids) or [FALLBACK_SOURCE]
|
||||
# 清理可能残留的旧候选节点（如 cand_X），避免一个岗位两个节点
|
||||
cand_id = f"cand_{job.title}"
|
||||
if cand_id != job.job_id and kg.g.has_node(cand_id):
|
||||
kg.g.remove_node(cand_id)
|
||||
kg.jobs.pop(cand_id, None)
|
||||
kg.add_job(job)
|
||||
required = set(job.required_skills)
|
||||
for skill in job.required_skills + job.preferred_skills:
|
||||
importance = "required" if skill in required else "preferred"
|
||||
kg.observe(GraphObservation(
|
||||
job_id=job.job_id, skill=skill, importance=importance,
|
||||
weight=1.0, window=first_window, source_ids=sources))
|
||||
kg.add_event(ChangeEvent(
|
||||
event_id=new_event_id(), job_id=job.job_id, skill=skill,
|
||||
change_type="新增", window=first_window,
|
||||
prev_weight=0.0, curr_weight=1.0, source_ids=sources[:10],
|
||||
description=(f"{first_window}，岗位【{job.title}】技能【{skill}】出现新增能力要求"
|
||||
f"（人工审核入库；数据源：招聘 JD，支撑样本 {len(sources)} 条）"),
|
||||
created_at=now_str()))
|
||||
for sc in job.scenarios:
|
||||
kg.add_scenario(sc)
|
||||
kg.g.add_edge(job.job_id, f"场景::{sc}", rel="APPLIES_TO")
|
||||
for ind in job.industries:
|
||||
kg.add_industry(ind)
|
||||
kg.g.add_edge(job.job_id, f"行业::{ind}", rel="BELONGS_TO")
|
||||
logger.info("岗位入库：%s（窗口 %s 起，%d 个来源）", job.title, first_window, len(sources))
|
||||
|
||||
|
||||
def apply_node_updates(kg: TemporalKG, node_id: str, updates: dict) -> None:
|
||||
"""把节点编辑应用到图谱；Job 节点同步 kg.jobs 的 JobDefinition（title/category/级别）。"""
|
||||
if node_id not in kg.g:
|
||||
return
|
||||
kg.g.nodes[node_id].update(updates)
|
||||
job = kg.jobs.get(node_id)
|
||||
if job is None:
|
||||
return
|
||||
if "title" in updates:
|
||||
job.title = str(updates["title"])
|
||||
if "category" in updates:
|
||||
job.category = str(updates["category"])
|
||||
if "level" in updates and updates["level"] in LEVEL_TO_YEARS:
|
||||
# 级别由期望年限推出：人工改级别时同步年限，保证级别筛选与详情面板一致
|
||||
job.expected_years = LEVEL_TO_YEARS[updates["level"]]
|
||||
kg.g.nodes[node_id]["expected_years"] = job.expected_years
|
||||
|
||||
|
||||
# ---------- 覆写记录（job_overrides 表） ----------
|
||||
def record_job_accept(db_path: str | Path, job: JobDefinition, candidate_id: str = "") -> None:
|
||||
"""审核通过：把岗位定义（含支撑证据）写入覆写层，供重建后重放。
|
||||
|
||||
``candidate_id`` 为转正前的候选 id（cand_X），重放时据此把候选从待审核区清除。
|
||||
"""
|
||||
db.save_override(db_path, job.job_id, {
|
||||
"kind": "job_accept", "candidate_id": candidate_id or job.job_id,
|
||||
"job": job.model_dump()})
|
||||
|
||||
|
||||
def record_node_update(db_path: str | Path, node_id: str, updates: dict) -> None:
|
||||
"""节点编辑：与已有覆写合并后写入（同一节点多次编辑累积重放）。"""
|
||||
existing = db.get_override(db_path, node_id)
|
||||
merged = dict(existing.get("updates", {})) if existing and existing.get("kind") == "node_update" else {}
|
||||
merged.update(updates)
|
||||
db.save_override(db_path, node_id,
|
||||
{"kind": "node_update", "node_id": node_id, "updates": merged})
|
||||
|
||||
|
||||
def replay_overrides(kg: TemporalKG, db_path: str | Path) -> dict:
|
||||
"""pipeline 重建后重放全部人工覆写：入库岗位强制纳入，节点编辑重新应用。"""
|
||||
replayed = {"jobs": 0, "node_updates": 0, "job_ids": []}
|
||||
try:
|
||||
overrides = db.list_overrides(db_path)
|
||||
except Exception as exc: # noqa: BLE001 - 状态库不可用不应拖垮重建
|
||||
logger.warning("读取人工覆写失败，跳过重放: %s", exc)
|
||||
return replayed
|
||||
for ov in overrides:
|
||||
kind = ov.get("kind")
|
||||
if kind == "job_accept":
|
||||
job = JobDefinition(**ov["job"])
|
||||
job.job_id = normalize_job_id(job)
|
||||
apply_accepted_job(kg, job)
|
||||
replayed["jobs"] += 1
|
||||
replayed["job_ids"].append(job.job_id)
|
||||
db.delete_candidate(db_path, ov.get("candidate_id") or job.job_id)
|
||||
elif kind == "node_update":
|
||||
apply_node_updates(kg, ov.get("node_id", ""), ov.get("updates", {}))
|
||||
replayed["node_updates"] += 1
|
||||
if overrides:
|
||||
logger.info("人工状态重放完成：%d 个人工岗位 / %d 条节点编辑",
|
||||
replayed["jobs"], replayed["node_updates"])
|
||||
return replayed
|
||||
@@ -0,0 +1,179 @@
|
||||
"""端到端编排：导入 → 清洗 → 抽取 → 图谱写入 → 演化分析 → 新岗位发现。
|
||||
|
||||
供 CLI（scripts/run_demo.py）与 API（/api/jd/import）复用，
|
||||
返回各阶段统计。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from app.cleaning.decay import apply_decay
|
||||
from app.cleaning.dedup import find_near_duplicates
|
||||
from app.cleaning.noise import filter_noise
|
||||
from app.config import ROOT, load_config, resolve
|
||||
from app.discover.cluster import discover_new_jobs, save_pending
|
||||
from app.evolution.analyzer import analyze, build_observations
|
||||
from app.extract.base import load_lexicon
|
||||
from app.extract.llm import get_extractor
|
||||
from app.graph.tkg import TemporalKG
|
||||
from app.ingest.loader import load_jd_dir
|
||||
from app.models import ExtractionResult, JDRecord, JobDefinition
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 既有岗位注册表的数据截止年：早于该年出现的岗位视为“既有岗位”，
|
||||
# 之后才出现的岗位交给“新岗位发现”链路（待审核后才入正式图谱）。
|
||||
KNOWN_JOB_BEFORE_YEAR = 2023
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
||||
)
|
||||
|
||||
|
||||
def job_id_of(title: str) -> str:
|
||||
return f"job_{title}"
|
||||
|
||||
|
||||
def build_job_definitions(
|
||||
records: list[JDRecord],
|
||||
extractions: dict[str, ExtractionResult],
|
||||
known_titles: set[str],
|
||||
lexicon: dict,
|
||||
) -> dict[str, JobDefinition]:
|
||||
"""按标题聚合近三年抽取结果，生成既有岗位定义（频次阈值投票，多源一致才入选）。
|
||||
|
||||
岗位定义刻画“当前能力要求”，因此只用最近 3 个年份的 JD 聚合，
|
||||
历史技能由时序图谱与演化事件表达，不进当前定义。
|
||||
"""
|
||||
jobs_meta = {j["title"]: j for j in lexicon.get("jobs", [])}
|
||||
max_year = max((r.publish_date.year for r in records), default=2026)
|
||||
recent_from = max_year - 2
|
||||
defs: dict[str, JobDefinition] = {}
|
||||
for title in sorted(known_titles):
|
||||
group = [r for r in records
|
||||
if r.jid in extractions and extractions[r.jid].job_title == title
|
||||
and not r.is_noise and not r.is_duplicate
|
||||
and r.publish_date.year >= recent_from]
|
||||
if not group:
|
||||
continue
|
||||
n = len(group)
|
||||
req = Counter(s for r in group for s in extractions[r.jid].required_skills)
|
||||
pref = Counter(s for r in group for s in extractions[r.jid].preferred_skills)
|
||||
duty = Counter(d for r in group for d in extractions[r.jid].responsibilities)
|
||||
scen = Counter(s for r in group for s in extractions[r.jid].scenarios)
|
||||
meta = jobs_meta.get(title, {})
|
||||
required = [s for s, c in req.most_common() if c >= max(2, n * 0.25)]
|
||||
defs[job_id_of(title)] = JobDefinition(
|
||||
job_id=job_id_of(title),
|
||||
title=title,
|
||||
category=meta.get("category", ""),
|
||||
industries=list(meta.get("industries", [])),
|
||||
required_skills=required,
|
||||
preferred_skills=[s for s, c in pref.most_common()
|
||||
if c >= max(2, n * 0.15) and s not in required],
|
||||
responsibilities=[d for d, _ in duty.most_common(5)],
|
||||
scenarios=[s for s, _ in scen.most_common(5)],
|
||||
expected_years=int(meta.get("expected_years", 3)),
|
||||
status="active",
|
||||
source_count=n,
|
||||
company_count=len({r.company for r in group}),
|
||||
)
|
||||
return defs
|
||||
|
||||
|
||||
def run_pipeline(config: dict | None = None) -> dict:
|
||||
"""执行全链路，返回统计信息。"""
|
||||
cfg = config or load_config()
|
||||
stats: dict = {"stages": {}}
|
||||
|
||||
# 1. 导入
|
||||
records, errors = load_jd_dir(resolve(cfg, "jd_raw_dir"))
|
||||
source_stats = {"jd_raw": len(records)}
|
||||
# 附加数据源（如 data/real/autoweaks_jd 真实数据集），可配置切换
|
||||
for extra in str(cfg.get("jd_extra_dirs") or "").split(","):
|
||||
extra = extra.strip()
|
||||
if not extra:
|
||||
continue
|
||||
extra_path = Path(extra) if Path(extra).is_absolute() else ROOT / extra
|
||||
if not extra_path.exists():
|
||||
logger.warning("附加数据源不存在，已跳过: %s", extra_path)
|
||||
continue
|
||||
recs2, errs2 = load_jd_dir(extra_path)
|
||||
records.extend(recs2)
|
||||
errors.extend(errs2)
|
||||
source_stats[extra_path.name] = len(recs2)
|
||||
logger.info("附加数据源 %s: %d 条 JD", extra_path.name, len(recs2))
|
||||
stats["stages"]["ingest"] = {"records": len(records), "bad_lines": len(errors),
|
||||
"sources": source_stats}
|
||||
|
||||
# 2. 清洗：噪声 → 近重复 → 时间衰减
|
||||
filter_noise(records)
|
||||
dup_clusters = find_near_duplicates(records)
|
||||
apply_decay(records, lam=float(cfg.get("decay_lambda", 0.35)))
|
||||
stats["stages"]["cleaning"] = {
|
||||
"dup_clusters": len(dup_clusters),
|
||||
"duplicates": sum(1 for r in records if r.is_duplicate),
|
||||
"noise": sum(1 for r in records if r.is_noise),
|
||||
}
|
||||
|
||||
# 3. 抽取（LLM 或离线，自动降级）
|
||||
lexicon = load_lexicon(resolve(cfg, "lexicon_path"))
|
||||
extractor = get_extractor(cfg)
|
||||
extractions: dict[str, ExtractionResult] = {}
|
||||
for r in records:
|
||||
if not r.is_noise:
|
||||
extractions[r.jid] = extractor.extract(r)
|
||||
stats["stages"]["extract"] = {"extracted": len(extractions),
|
||||
"extractor": extractor.name}
|
||||
|
||||
# 4. 图谱写入：既有岗位定义 + 观察流
|
||||
first_years: dict[str, int] = {}
|
||||
for r in records:
|
||||
if r.jid in extractions:
|
||||
t = extractions[r.jid].job_title
|
||||
first_years[t] = min(first_years.get(t, 9999), r.publish_date.year)
|
||||
known_titles = {t for t, y in first_years.items() if y < KNOWN_JOB_BEFORE_YEAR}
|
||||
|
||||
kg = TemporalKG()
|
||||
defs = build_job_definitions(records, extractions, known_titles, lexicon)
|
||||
for d in defs.values():
|
||||
kg.add_job(d)
|
||||
for sc in d.scenarios:
|
||||
kg.add_scenario(sc)
|
||||
kg.g.add_edge(d.job_id, f"场景::{sc}", rel="APPLIES_TO")
|
||||
for ind in d.industries:
|
||||
kg.add_industry(ind)
|
||||
kg.g.add_edge(d.job_id, f"行业::{ind}", rel="BELONGS_TO")
|
||||
|
||||
job_id_of_title = {d.title: d.job_id for d in defs.values()}
|
||||
observations = build_observations(records, extractions, job_id_of_title)
|
||||
for obs in observations:
|
||||
kg.observe(obs)
|
||||
stats["stages"]["graph"] = kg.stats()
|
||||
|
||||
# 5. 演化分析（事件追加进图谱）
|
||||
events = analyze(kg, observations=observations)
|
||||
stats["stages"]["evolution"] = {"events": len(events)}
|
||||
|
||||
# 6. 新岗位发现 → 待审核分区（SQLite 状态库）
|
||||
candidates = discover_new_jobs(records, extractions, known_titles)
|
||||
save_pending(candidates, resolve(cfg, "state_db_path"))
|
||||
stats["stages"]["discover"] = {"candidates": [c.title for c in candidates]}
|
||||
|
||||
# 7. 人工状态重放：审核入库岗位强制纳入正式图谱（不受"2023 年前"规则限制），
|
||||
# 节点编辑一并恢复（覆写层存于 SQLite job_overrides 表）
|
||||
from app.services.overrides import replay_overrides
|
||||
stats["stages"]["overrides"] = replay_overrides(kg, resolve(cfg, "state_db_path"))
|
||||
|
||||
kg.save(resolve(cfg, "graph_path"))
|
||||
stats["graph_path"] = str(resolve(cfg, "graph_path"))
|
||||
logger.info("pipeline 完成：%s", stats["stages"])
|
||||
return stats
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import json
|
||||
print(json.dumps(run_pipeline(), ensure_ascii=False, indent=2))
|
||||
@@ -0,0 +1,360 @@
|
||||
"""SQLite 应用状态库：审核留痕 / 待审核候选 / 异步任务（默认启用，零新依赖）。
|
||||
|
||||
- 数据库文件默认 ``data/careernexus.db``（配置键 ``state_db_path``），仅用标准库 sqlite3；
|
||||
- WAL 模式、外键开；``init_schema()`` 幂等建表，重复调用无副作用；
|
||||
- 一次性迁移：打开连接时若旧文件存在、对应表为空且尚未迁移过，把
|
||||
``data/review/audit.jsonl`` 与 ``data/review/pending.json`` 导入
|
||||
（原文件保留不删除，迁移标记写入 meta 表并记日志）。
|
||||
|
||||
留痕存取约定：审核记录（candidate_id/action）把 action 存进 kind 列、
|
||||
candidate_id 存进 node_id 列；节点编辑记录（kind=node_update）原样存取。
|
||||
``get_audit`` 按此约定还原出与旧 audit.jsonl 行完全一致的字典结构。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import sqlite3
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SCHEMA = """\
|
||||
CREATE TABLE IF NOT EXISTS audit_log (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
ts TEXT,
|
||||
node_id TEXT,
|
||||
kind TEXT,
|
||||
operator TEXT,
|
||||
reason TEXT,
|
||||
before_json TEXT,
|
||||
after_json TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS review_candidates (
|
||||
job_id TEXT PRIMARY KEY,
|
||||
payload_json TEXT,
|
||||
updated_ts TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS tasks (
|
||||
task_id TEXT PRIMARY KEY,
|
||||
kind TEXT,
|
||||
status TEXT,
|
||||
payload_json TEXT,
|
||||
created_ts TEXT,
|
||||
updated_ts TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS job_overrides (
|
||||
name TEXT PRIMARY KEY,
|
||||
payload_json TEXT,
|
||||
updated_ts TEXT
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS meta (
|
||||
key TEXT PRIMARY KEY,
|
||||
value TEXT
|
||||
);
|
||||
"""
|
||||
|
||||
REVIEW_ACTIONS = {"accept", "modify", "reject"}
|
||||
|
||||
|
||||
def _now() -> str:
|
||||
return datetime.now().strftime("%Y-%m-%d %H:%M:%S")
|
||||
|
||||
|
||||
# ---------- 连接与建表 ----------
|
||||
def init_schema(conn: sqlite3.Connection) -> None:
|
||||
"""幂等建表（CREATE TABLE IF NOT EXISTS），可反复调用。"""
|
||||
conn.executescript(SCHEMA)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def connect(
|
||||
db_path: str | Path,
|
||||
legacy_audit: str | Path | None = None,
|
||||
legacy_pending: str | Path | None = None,
|
||||
) -> sqlite3.Connection:
|
||||
"""打开（必要时创建）状态库：WAL + 外键 + 幂等建表 + 一次性旧文件迁移。"""
|
||||
path = Path(db_path)
|
||||
if str(path.parent) not in ("", "."):
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(str(path))
|
||||
conn.row_factory = sqlite3.Row
|
||||
conn.execute("PRAGMA journal_mode=WAL")
|
||||
conn.execute("PRAGMA foreign_keys=ON")
|
||||
init_schema(conn)
|
||||
migrate_legacy(conn, legacy_audit=legacy_audit, legacy_pending=legacy_pending)
|
||||
return conn
|
||||
|
||||
|
||||
# ---------- 一次性迁移（旧 JSONL/JSON -> SQLite，原文件保留） ----------
|
||||
def _migration_done(conn: sqlite3.Connection, key: str) -> bool:
|
||||
row = conn.execute("SELECT value FROM meta WHERE key = ?", (key,)).fetchone()
|
||||
return row is not None
|
||||
|
||||
|
||||
def _mark_migrated(conn: sqlite3.Connection, key: str) -> None:
|
||||
conn.execute("INSERT OR REPLACE INTO meta(key, value) VALUES (?, ?)", (key, _now()))
|
||||
|
||||
|
||||
def _table_empty(conn: sqlite3.Connection, table: str) -> bool:
|
||||
return conn.execute(f"SELECT COUNT(*) FROM {table}").fetchone()[0] == 0
|
||||
|
||||
|
||||
def migrate_legacy(
|
||||
conn: sqlite3.Connection,
|
||||
legacy_audit: str | Path | None = None,
|
||||
legacy_pending: str | Path | None = None,
|
||||
) -> None:
|
||||
"""把旧状态文件导入 SQLite：仅当对应表为空且未迁移过时执行，原文件保留。"""
|
||||
if legacy_audit is not None:
|
||||
_migrate_audit(conn, Path(legacy_audit))
|
||||
if legacy_pending is not None:
|
||||
_migrate_pending(conn, Path(legacy_pending))
|
||||
conn.commit()
|
||||
|
||||
|
||||
def _migrate_audit(conn: sqlite3.Connection, path: Path) -> None:
|
||||
key = "migrated_audit"
|
||||
if _migration_done(conn, key):
|
||||
return
|
||||
_mark_migrated(conn, key)
|
||||
if not path.exists() or not _table_empty(conn, "audit_log"):
|
||||
return
|
||||
n = 0
|
||||
for line in path.read_text(encoding="utf-8").splitlines():
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
record = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
_insert_audit(conn, record)
|
||||
n += 1
|
||||
logger.info("状态迁移：%s -> audit_log（%d 条，原文件保留）", path, n)
|
||||
|
||||
|
||||
def _migrate_pending(conn: sqlite3.Connection, path: Path) -> None:
|
||||
key = "migrated_pending"
|
||||
if _migration_done(conn, key):
|
||||
return
|
||||
_mark_migrated(conn, key)
|
||||
if not path.exists() or not _table_empty(conn, "review_candidates"):
|
||||
return
|
||||
try:
|
||||
candidates = json.loads(path.read_text(encoding="utf-8"))
|
||||
except json.JSONDecodeError:
|
||||
return
|
||||
for payload in candidates:
|
||||
_upsert_candidate(conn, payload)
|
||||
logger.info("状态迁移：%s -> review_candidates（%d 条，原文件保留）", path, len(candidates))
|
||||
|
||||
|
||||
# ---------- 审核留痕 ----------
|
||||
def _insert_audit(conn: sqlite3.Connection, record: dict) -> None:
|
||||
kind = record.get("kind") or record.get("action") or ""
|
||||
node_id = record.get("node_id") or record.get("candidate_id") or ""
|
||||
conn.execute(
|
||||
"INSERT INTO audit_log(ts, node_id, kind, operator, reason, before_json, after_json)"
|
||||
" VALUES (?, ?, ?, ?, ?, ?, ?)",
|
||||
(
|
||||
record.get("time") or _now(),
|
||||
node_id,
|
||||
kind,
|
||||
record.get("operator", ""),
|
||||
record.get("reason", ""),
|
||||
json.dumps(record.get("before") or {}, ensure_ascii=False),
|
||||
json.dumps(record.get("after") or {}, ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def append_audit(db_path: str | Path, record: dict) -> None:
|
||||
"""追加一条留痕（审核记录或 kind=node_update 的节点编辑记录）。"""
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
_insert_audit(conn, record)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _audit_row_to_record(row: sqlite3.Row) -> dict:
|
||||
"""把 audit_log 行还原为旧 audit.jsonl 行的字典结构（/api/audit 兼容）。"""
|
||||
before = json.loads(row["before_json"] or "{}")
|
||||
after = json.loads(row["after_json"] or "{}")
|
||||
if row["kind"] in REVIEW_ACTIONS:
|
||||
return {"candidate_id": row["node_id"], "action": row["kind"],
|
||||
"operator": row["operator"], "reason": row["reason"],
|
||||
"before": before, "after": after, "time": row["ts"]}
|
||||
return {"kind": row["kind"], "node_id": row["node_id"],
|
||||
"operator": row["operator"], "reason": row["reason"],
|
||||
"before": before, "after": after, "time": row["ts"]}
|
||||
|
||||
|
||||
def get_audit(db_path: str | Path, node_id: str | None = None, limit: int = 50) -> list[dict]:
|
||||
"""读取留痕（新→旧排序），可按节点/候选 id 过滤。"""
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
if node_id:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM audit_log WHERE node_id = ? ORDER BY id DESC LIMIT ?",
|
||||
(node_id, limit)).fetchall()
|
||||
else:
|
||||
rows = conn.execute(
|
||||
"SELECT * FROM audit_log ORDER BY id DESC LIMIT ?", (limit,)).fetchall()
|
||||
return [_audit_row_to_record(r) for r in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------- 待审核候选 ----------
|
||||
def _upsert_candidate(conn: sqlite3.Connection, payload: dict) -> None:
|
||||
conn.execute(
|
||||
"INSERT INTO review_candidates(job_id, payload_json, updated_ts) VALUES (?, ?, ?)"
|
||||
" ON CONFLICT(job_id) DO UPDATE SET payload_json = excluded.payload_json,"
|
||||
" updated_ts = excluded.updated_ts",
|
||||
(payload["job_id"], json.dumps(payload, ensure_ascii=False), _now()))
|
||||
|
||||
|
||||
def save_candidate(db_path: str | Path, payload: dict) -> None:
|
||||
"""新增或更新一个待审核候选（按 job_id upsert）。"""
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
_upsert_candidate(conn, payload)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_candidate(db_path: str | Path, job_id: str) -> dict | None:
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT payload_json FROM review_candidates WHERE job_id = ?",
|
||||
(job_id,)).fetchone()
|
||||
return json.loads(row["payload_json"]) if row else None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def list_candidates(db_path: str | Path) -> list[dict]:
|
||||
"""全部待审核候选（按写入顺序）。"""
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT payload_json FROM review_candidates ORDER BY rowid").fetchall()
|
||||
return [json.loads(r["payload_json"]) for r in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def delete_candidate(db_path: str | Path, job_id: str) -> None:
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
conn.execute("DELETE FROM review_candidates WHERE job_id = ?", (job_id,))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------- 异步任务 ----------
|
||||
def upsert_task(
|
||||
db_path: str | Path,
|
||||
task_id: str,
|
||||
kind: str,
|
||||
status: str,
|
||||
payload: dict,
|
||||
created_ts: str | None = None,
|
||||
) -> None:
|
||||
"""登记或更新任务状态（payload 为内存中任务 dict 的 JSON 快照）。"""
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
conn.execute(
|
||||
"INSERT INTO tasks(task_id, kind, status, payload_json, created_ts, updated_ts)"
|
||||
" VALUES (?, ?, ?, ?, ?, ?)"
|
||||
" ON CONFLICT(task_id) DO UPDATE SET status = excluded.status,"
|
||||
" payload_json = excluded.payload_json, updated_ts = excluded.updated_ts",
|
||||
(task_id, kind, status, json.dumps(payload, ensure_ascii=False),
|
||||
created_ts or _now(), _now()))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def _task_row(row: sqlite3.Row) -> dict:
|
||||
return {"task_id": row["task_id"], "kind": row["kind"], "status": row["status"],
|
||||
"payload": json.loads(row["payload_json"] or "{}"),
|
||||
"created_ts": row["created_ts"], "updated_ts": row["updated_ts"]}
|
||||
|
||||
|
||||
def get_task(db_path: str | Path, task_id: str) -> dict | None:
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
row = conn.execute("SELECT * FROM tasks WHERE task_id = ?", (task_id,)).fetchone()
|
||||
return _task_row(row) if row else None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def list_tasks(db_path: str | Path) -> list[dict]:
|
||||
"""全部任务（按创建时间升序），供重启后恢复内存任务表。"""
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
rows = conn.execute("SELECT * FROM tasks ORDER BY created_ts, rowid").fetchall()
|
||||
return [_task_row(r) for r in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------- 人工状态覆写层（跨图谱重建持久化） ----------
|
||||
def _upsert_override(conn: sqlite3.Connection, name: str, payload: dict) -> None:
|
||||
conn.execute(
|
||||
"INSERT INTO job_overrides(name, payload_json, updated_ts) VALUES (?, ?, ?)"
|
||||
" ON CONFLICT(name) DO UPDATE SET payload_json = excluded.payload_json,"
|
||||
" updated_ts = excluded.updated_ts",
|
||||
(name, json.dumps(payload, ensure_ascii=False), _now()))
|
||||
|
||||
|
||||
def save_override(db_path: str | Path, name: str, payload: dict) -> None:
|
||||
"""新增或更新一条人工覆写（按 name upsert）。
|
||||
|
||||
payload 约定：{"kind": "job_accept", "job": <JobDefinition dump>}
|
||||
或 {"kind": "node_update", "node_id": ..., "updates": {...}}。
|
||||
"""
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
_upsert_override(conn, name, payload)
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def get_override(db_path: str | Path, name: str) -> dict | None:
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
row = conn.execute(
|
||||
"SELECT payload_json FROM job_overrides WHERE name = ?", (name,)).fetchone()
|
||||
return json.loads(row["payload_json"]) if row else None
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def list_overrides(db_path: str | Path) -> list[dict]:
|
||||
"""全部人工覆写（按写入顺序），pipeline 重建后按序重放。"""
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
rows = conn.execute(
|
||||
"SELECT payload_json FROM job_overrides ORDER BY rowid").fetchall()
|
||||
return [json.loads(r["payload_json"]) for r in rows]
|
||||
finally:
|
||||
conn.close()
|
||||
|
||||
|
||||
def delete_override(db_path: str | Path, name: str) -> None:
|
||||
conn = connect(db_path)
|
||||
try:
|
||||
conn.execute("DELETE FROM job_overrides WHERE name = ?", (name,))
|
||||
conn.commit()
|
||||
finally:
|
||||
conn.close()
|
||||
Reference in New Issue
Block a user