CareerNexus 职联：岗位能力图谱构建与动态演化分析平台（挑战杯 XH-202621）
- 全链路：多源JD接入/清洗交叉验证/结构化抽取(星火+离线双通道)/时序知识图谱/新岗位发现/演化分析/人岗匹配/学习路径 - WebUI 五视图：驾驶舱/图谱工作台/岗位演化/匹配(单人+批量)/学习路径 - 设置中心(config.json)/Token鉴权/SQLite状态库/语义向量通道(BGE-M3本地或联网API) - 71 个 pytest 用例；数据目录仅保留结构(.gitkeep)，数据由 scripts/generate_dataset.py 复现
This commit is contained in:
RoyirinRosmontis
@@ -0,0 +1,130 @@
|
||||
"""人工状态覆写层：审核入库岗位与节点编辑跨图谱重建持久化。
|
||||
|
||||
- 人工审核通过的岗位、节点编辑先写入 SQLite 状态库 ``job_overrides`` 表
|
||||
（见 app/storage/db.py），同时应用到当前图谱；
|
||||
- pipeline 全量重建图谱后调用 :func:`replay_overrides` 按序重放，
|
||||
人工入库岗位强制纳入正式图谱（不受"2023 年前出现"规则限制），
|
||||
节点编辑同样保留。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
from app.graph.tkg import LEVEL_TO_YEARS, TemporalKG, job_level, new_event_id, now_str, quarter_of
|
||||
from app.models import ChangeEvent, GraphObservation, JobDefinition
|
||||
from app.storage import db
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
FALLBACK_SOURCE = "人工审核入库" # 无支撑证据的旧候选入库时的来源标记
|
||||
|
||||
|
||||
def normalize_job_id(job: JobDefinition) -> str:
|
||||
"""候选转正：cand_X 统一归一为 job_<岗位名>，保证全图只有一个岗位节点。"""
|
||||
if job.job_id.startswith("cand_"):
|
||||
return f"job_{job.title}"
|
||||
return job.job_id
|
||||
|
||||
|
||||
def apply_accepted_job(kg: TemporalKG, job: JobDefinition) -> None:
|
||||
"""把审核通过的岗位写入图谱：真实区间/来源的 REQUIRES 边、场景/行业边与"新增"演化事件。
|
||||
|
||||
区间自首个支持窗口起；来源取候选簇真实支撑 JD 的 source_id 列表
|
||||
（旧候选缺支撑证据时回退到"人工审核入库"标记，窗口取当前季度）。
|
||||
调用前须先把 job_id 归一化（见 normalize_job_id）；若图中残留旧候选节点则清理。
|
||||
"""
|
||||
job.status = "active"
|
||||
first_window = min(job.support_windows) if job.support_windows else quarter_of(datetime.now())
|
||||
sources = list(job.support_source_ids) or [FALLBACK_SOURCE]
|
||||
# 清理可能残留的旧候选节点（如 cand_X），避免一个岗位两个节点
|
||||
cand_id = f"cand_{job.title}"
|
||||
if cand_id != job.job_id and kg.g.has_node(cand_id):
|
||||
kg.g.remove_node(cand_id)
|
||||
kg.jobs.pop(cand_id, None)
|
||||
kg.add_job(job)
|
||||
required = set(job.required_skills)
|
||||
for skill in job.required_skills + job.preferred_skills:
|
||||
importance = "required" if skill in required else "preferred"
|
||||
kg.observe(GraphObservation(
|
||||
job_id=job.job_id, skill=skill, importance=importance,
|
||||
weight=1.0, window=first_window, source_ids=sources))
|
||||
kg.add_event(ChangeEvent(
|
||||
event_id=new_event_id(), job_id=job.job_id, skill=skill,
|
||||
change_type="新增", window=first_window,
|
||||
prev_weight=0.0, curr_weight=1.0, source_ids=sources[:10],
|
||||
description=(f"{first_window}，岗位【{job.title}】技能【{skill}】出现新增能力要求"
|
||||
f"（人工审核入库；数据源：招聘 JD，支撑样本 {len(sources)} 条）"),
|
||||
created_at=now_str()))
|
||||
for sc in job.scenarios:
|
||||
kg.add_scenario(sc)
|
||||
kg.g.add_edge(job.job_id, f"场景::{sc}", rel="APPLIES_TO")
|
||||
for ind in job.industries:
|
||||
kg.add_industry(ind)
|
||||
kg.g.add_edge(job.job_id, f"行业::{ind}", rel="BELONGS_TO")
|
||||
logger.info("岗位入库：%s（窗口 %s 起，%d 个来源）", job.title, first_window, len(sources))
|
||||
|
||||
|
||||
def apply_node_updates(kg: TemporalKG, node_id: str, updates: dict) -> None:
|
||||
"""把节点编辑应用到图谱；Job 节点同步 kg.jobs 的 JobDefinition（title/category/级别）。"""
|
||||
if node_id not in kg.g:
|
||||
return
|
||||
kg.g.nodes[node_id].update(updates)
|
||||
job = kg.jobs.get(node_id)
|
||||
if job is None:
|
||||
return
|
||||
if "title" in updates:
|
||||
job.title = str(updates["title"])
|
||||
if "category" in updates:
|
||||
job.category = str(updates["category"])
|
||||
if "level" in updates and updates["level"] in LEVEL_TO_YEARS:
|
||||
# 级别由期望年限推出：人工改级别时同步年限，保证级别筛选与详情面板一致
|
||||
job.expected_years = LEVEL_TO_YEARS[updates["level"]]
|
||||
kg.g.nodes[node_id]["expected_years"] = job.expected_years
|
||||
|
||||
|
||||
# ---------- 覆写记录（job_overrides 表） ----------
|
||||
def record_job_accept(db_path: str | Path, job: JobDefinition, candidate_id: str = "") -> None:
|
||||
"""审核通过：把岗位定义（含支撑证据）写入覆写层，供重建后重放。
|
||||
|
||||
``candidate_id`` 为转正前的候选 id（cand_X），重放时据此把候选从待审核区清除。
|
||||
"""
|
||||
db.save_override(db_path, job.job_id, {
|
||||
"kind": "job_accept", "candidate_id": candidate_id or job.job_id,
|
||||
"job": job.model_dump()})
|
||||
|
||||
|
||||
def record_node_update(db_path: str | Path, node_id: str, updates: dict) -> None:
|
||||
"""节点编辑：与已有覆写合并后写入（同一节点多次编辑累积重放）。"""
|
||||
existing = db.get_override(db_path, node_id)
|
||||
merged = dict(existing.get("updates", {})) if existing and existing.get("kind") == "node_update" else {}
|
||||
merged.update(updates)
|
||||
db.save_override(db_path, node_id,
|
||||
{"kind": "node_update", "node_id": node_id, "updates": merged})
|
||||
|
||||
|
||||
def replay_overrides(kg: TemporalKG, db_path: str | Path) -> dict:
|
||||
"""pipeline 重建后重放全部人工覆写：入库岗位强制纳入，节点编辑重新应用。"""
|
||||
replayed = {"jobs": 0, "node_updates": 0, "job_ids": []}
|
||||
try:
|
||||
overrides = db.list_overrides(db_path)
|
||||
except Exception as exc: # noqa: BLE001 - 状态库不可用不应拖垮重建
|
||||
logger.warning("读取人工覆写失败，跳过重放: %s", exc)
|
||||
return replayed
|
||||
for ov in overrides:
|
||||
kind = ov.get("kind")
|
||||
if kind == "job_accept":
|
||||
job = JobDefinition(**ov["job"])
|
||||
job.job_id = normalize_job_id(job)
|
||||
apply_accepted_job(kg, job)
|
||||
replayed["jobs"] += 1
|
||||
replayed["job_ids"].append(job.job_id)
|
||||
db.delete_candidate(db_path, ov.get("candidate_id") or job.job_id)
|
||||
elif kind == "node_update":
|
||||
apply_node_updates(kg, ov.get("node_id", ""), ov.get("updates", {}))
|
||||
replayed["node_updates"] += 1
|
||||
if overrides:
|
||||
logger.info("人工状态重放完成：%d 个人工岗位 / %d 条节点编辑",
|
||||
replayed["jobs"], replayed["node_updates"])
|
||||
return replayed
|
||||
@@ -0,0 +1,179 @@
|
||||
"""端到端编排：导入 → 清洗 → 抽取 → 图谱写入 → 演化分析 → 新岗位发现。
|
||||
|
||||
供 CLI（scripts/run_demo.py）与 API（/api/jd/import）复用，
|
||||
返回各阶段统计。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from app.cleaning.decay import apply_decay
|
||||
from app.cleaning.dedup import find_near_duplicates
|
||||
from app.cleaning.noise import filter_noise
|
||||
from app.config import ROOT, load_config, resolve
|
||||
from app.discover.cluster import discover_new_jobs, save_pending
|
||||
from app.evolution.analyzer import analyze, build_observations
|
||||
from app.extract.base import load_lexicon
|
||||
from app.extract.llm import get_extractor
|
||||
from app.graph.tkg import TemporalKG
|
||||
from app.ingest.loader import load_jd_dir
|
||||
from app.models import ExtractionResult, JDRecord, JobDefinition
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 既有岗位注册表的数据截止年：早于该年出现的岗位视为“既有岗位”，
|
||||
# 之后才出现的岗位交给“新岗位发现”链路（待审核后才入正式图谱）。
|
||||
KNOWN_JOB_BEFORE_YEAR = 2023
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
||||
)
|
||||
|
||||
|
||||
def job_id_of(title: str) -> str:
|
||||
return f"job_{title}"
|
||||
|
||||
|
||||
def build_job_definitions(
|
||||
records: list[JDRecord],
|
||||
extractions: dict[str, ExtractionResult],
|
||||
known_titles: set[str],
|
||||
lexicon: dict,
|
||||
) -> dict[str, JobDefinition]:
|
||||
"""按标题聚合近三年抽取结果，生成既有岗位定义（频次阈值投票，多源一致才入选）。
|
||||
|
||||
岗位定义刻画“当前能力要求”，因此只用最近 3 个年份的 JD 聚合，
|
||||
历史技能由时序图谱与演化事件表达，不进当前定义。
|
||||
"""
|
||||
jobs_meta = {j["title"]: j for j in lexicon.get("jobs", [])}
|
||||
max_year = max((r.publish_date.year for r in records), default=2026)
|
||||
recent_from = max_year - 2
|
||||
defs: dict[str, JobDefinition] = {}
|
||||
for title in sorted(known_titles):
|
||||
group = [r for r in records
|
||||
if r.jid in extractions and extractions[r.jid].job_title == title
|
||||
and not r.is_noise and not r.is_duplicate
|
||||
and r.publish_date.year >= recent_from]
|
||||
if not group:
|
||||
continue
|
||||
n = len(group)
|
||||
req = Counter(s for r in group for s in extractions[r.jid].required_skills)
|
||||
pref = Counter(s for r in group for s in extractions[r.jid].preferred_skills)
|
||||
duty = Counter(d for r in group for d in extractions[r.jid].responsibilities)
|
||||
scen = Counter(s for r in group for s in extractions[r.jid].scenarios)
|
||||
meta = jobs_meta.get(title, {})
|
||||
required = [s for s, c in req.most_common() if c >= max(2, n * 0.25)]
|
||||
defs[job_id_of(title)] = JobDefinition(
|
||||
job_id=job_id_of(title),
|
||||
title=title,
|
||||
category=meta.get("category", ""),
|
||||
industries=list(meta.get("industries", [])),
|
||||
required_skills=required,
|
||||
preferred_skills=[s for s, c in pref.most_common()
|
||||
if c >= max(2, n * 0.15) and s not in required],
|
||||
responsibilities=[d for d, _ in duty.most_common(5)],
|
||||
scenarios=[s for s, _ in scen.most_common(5)],
|
||||
expected_years=int(meta.get("expected_years", 3)),
|
||||
status="active",
|
||||
source_count=n,
|
||||
company_count=len({r.company for r in group}),
|
||||
)
|
||||
return defs
|
||||
|
||||
|
||||
def run_pipeline(config: dict | None = None) -> dict:
|
||||
"""执行全链路，返回统计信息。"""
|
||||
cfg = config or load_config()
|
||||
stats: dict = {"stages": {}}
|
||||
|
||||
# 1. 导入
|
||||
records, errors = load_jd_dir(resolve(cfg, "jd_raw_dir"))
|
||||
source_stats = {"jd_raw": len(records)}
|
||||
# 附加数据源（如 data/real/autoweaks_jd 真实数据集），可配置切换
|
||||
for extra in str(cfg.get("jd_extra_dirs") or "").split(","):
|
||||
extra = extra.strip()
|
||||
if not extra:
|
||||
continue
|
||||
extra_path = Path(extra) if Path(extra).is_absolute() else ROOT / extra
|
||||
if not extra_path.exists():
|
||||
logger.warning("附加数据源不存在，已跳过: %s", extra_path)
|
||||
continue
|
||||
recs2, errs2 = load_jd_dir(extra_path)
|
||||
records.extend(recs2)
|
||||
errors.extend(errs2)
|
||||
source_stats[extra_path.name] = len(recs2)
|
||||
logger.info("附加数据源 %s: %d 条 JD", extra_path.name, len(recs2))
|
||||
stats["stages"]["ingest"] = {"records": len(records), "bad_lines": len(errors),
|
||||
"sources": source_stats}
|
||||
|
||||
# 2. 清洗：噪声 → 近重复 → 时间衰减
|
||||
filter_noise(records)
|
||||
dup_clusters = find_near_duplicates(records)
|
||||
apply_decay(records, lam=float(cfg.get("decay_lambda", 0.35)))
|
||||
stats["stages"]["cleaning"] = {
|
||||
"dup_clusters": len(dup_clusters),
|
||||
"duplicates": sum(1 for r in records if r.is_duplicate),
|
||||
"noise": sum(1 for r in records if r.is_noise),
|
||||
}
|
||||
|
||||
# 3. 抽取（LLM 或离线，自动降级）
|
||||
lexicon = load_lexicon(resolve(cfg, "lexicon_path"))
|
||||
extractor = get_extractor(cfg)
|
||||
extractions: dict[str, ExtractionResult] = {}
|
||||
for r in records:
|
||||
if not r.is_noise:
|
||||
extractions[r.jid] = extractor.extract(r)
|
||||
stats["stages"]["extract"] = {"extracted": len(extractions),
|
||||
"extractor": extractor.name}
|
||||
|
||||
# 4. 图谱写入：既有岗位定义 + 观察流
|
||||
first_years: dict[str, int] = {}
|
||||
for r in records:
|
||||
if r.jid in extractions:
|
||||
t = extractions[r.jid].job_title
|
||||
first_years[t] = min(first_years.get(t, 9999), r.publish_date.year)
|
||||
known_titles = {t for t, y in first_years.items() if y < KNOWN_JOB_BEFORE_YEAR}
|
||||
|
||||
kg = TemporalKG()
|
||||
defs = build_job_definitions(records, extractions, known_titles, lexicon)
|
||||
for d in defs.values():
|
||||
kg.add_job(d)
|
||||
for sc in d.scenarios:
|
||||
kg.add_scenario(sc)
|
||||
kg.g.add_edge(d.job_id, f"场景::{sc}", rel="APPLIES_TO")
|
||||
for ind in d.industries:
|
||||
kg.add_industry(ind)
|
||||
kg.g.add_edge(d.job_id, f"行业::{ind}", rel="BELONGS_TO")
|
||||
|
||||
job_id_of_title = {d.title: d.job_id for d in defs.values()}
|
||||
observations = build_observations(records, extractions, job_id_of_title)
|
||||
for obs in observations:
|
||||
kg.observe(obs)
|
||||
stats["stages"]["graph"] = kg.stats()
|
||||
|
||||
# 5. 演化分析（事件追加进图谱）
|
||||
events = analyze(kg, observations=observations)
|
||||
stats["stages"]["evolution"] = {"events": len(events)}
|
||||
|
||||
# 6. 新岗位发现 → 待审核分区（SQLite 状态库）
|
||||
candidates = discover_new_jobs(records, extractions, known_titles)
|
||||
save_pending(candidates, resolve(cfg, "state_db_path"))
|
||||
stats["stages"]["discover"] = {"candidates": [c.title for c in candidates]}
|
||||
|
||||
# 7. 人工状态重放：审核入库岗位强制纳入正式图谱（不受"2023 年前"规则限制），
|
||||
# 节点编辑一并恢复（覆写层存于 SQLite job_overrides 表）
|
||||
from app.services.overrides import replay_overrides
|
||||
stats["stages"]["overrides"] = replay_overrides(kg, resolve(cfg, "state_db_path"))
|
||||
|
||||
kg.save(resolve(cfg, "graph_path"))
|
||||
stats["graph_path"] = str(resolve(cfg, "graph_path"))
|
||||
logger.info("pipeline 完成：%s", stats["stages"])
|
||||
return stats
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import json
|
||||
print(json.dumps(run_pipeline(), ensure_ascii=False, indent=2))
|
||||
Reference in New Issue
Block a user