CareerNexus 职联：岗位能力图谱构建与动态演化分析平台（挑战杯 XH-202621）
- 全链路：多源JD接入/清洗交叉验证/结构化抽取(星火+离线双通道)/时序知识图谱/新岗位发现/演化分析/人岗匹配/学习路径 - WebUI 五视图：驾驶舱/图谱工作台/岗位演化/匹配(单人+批量)/学习路径 - 设置中心(config.json)/Token鉴权/SQLite状态库/语义向量通道(BGE-M3本地或联网API) - 71 个 pytest 用例；数据目录仅保留结构(.gitkeep)，数据由 scripts/generate_dataset.py 复现
This commit is contained in:
RoyirinRosmontis
@@ -0,0 +1,179 @@
|
||||
"""端到端编排：导入 → 清洗 → 抽取 → 图谱写入 → 演化分析 → 新岗位发现。
|
||||
|
||||
供 CLI（scripts/run_demo.py）与 API（/api/jd/import）复用，
|
||||
返回各阶段统计。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
from app.cleaning.decay import apply_decay
|
||||
from app.cleaning.dedup import find_near_duplicates
|
||||
from app.cleaning.noise import filter_noise
|
||||
from app.config import ROOT, load_config, resolve
|
||||
from app.discover.cluster import discover_new_jobs, save_pending
|
||||
from app.evolution.analyzer import analyze, build_observations
|
||||
from app.extract.base import load_lexicon
|
||||
from app.extract.llm import get_extractor
|
||||
from app.graph.tkg import TemporalKG
|
||||
from app.ingest.loader import load_jd_dir
|
||||
from app.models import ExtractionResult, JDRecord, JobDefinition
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 既有岗位注册表的数据截止年：早于该年出现的岗位视为“既有岗位”，
|
||||
# 之后才出现的岗位交给“新岗位发现”链路（待审核后才入正式图谱）。
|
||||
KNOWN_JOB_BEFORE_YEAR = 2023
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
||||
)
|
||||
|
||||
|
||||
def job_id_of(title: str) -> str:
|
||||
return f"job_{title}"
|
||||
|
||||
|
||||
def build_job_definitions(
|
||||
records: list[JDRecord],
|
||||
extractions: dict[str, ExtractionResult],
|
||||
known_titles: set[str],
|
||||
lexicon: dict,
|
||||
) -> dict[str, JobDefinition]:
|
||||
"""按标题聚合近三年抽取结果，生成既有岗位定义（频次阈值投票，多源一致才入选）。
|
||||
|
||||
岗位定义刻画“当前能力要求”，因此只用最近 3 个年份的 JD 聚合，
|
||||
历史技能由时序图谱与演化事件表达，不进当前定义。
|
||||
"""
|
||||
jobs_meta = {j["title"]: j for j in lexicon.get("jobs", [])}
|
||||
max_year = max((r.publish_date.year for r in records), default=2026)
|
||||
recent_from = max_year - 2
|
||||
defs: dict[str, JobDefinition] = {}
|
||||
for title in sorted(known_titles):
|
||||
group = [r for r in records
|
||||
if r.jid in extractions and extractions[r.jid].job_title == title
|
||||
and not r.is_noise and not r.is_duplicate
|
||||
and r.publish_date.year >= recent_from]
|
||||
if not group:
|
||||
continue
|
||||
n = len(group)
|
||||
req = Counter(s for r in group for s in extractions[r.jid].required_skills)
|
||||
pref = Counter(s for r in group for s in extractions[r.jid].preferred_skills)
|
||||
duty = Counter(d for r in group for d in extractions[r.jid].responsibilities)
|
||||
scen = Counter(s for r in group for s in extractions[r.jid].scenarios)
|
||||
meta = jobs_meta.get(title, {})
|
||||
required = [s for s, c in req.most_common() if c >= max(2, n * 0.25)]
|
||||
defs[job_id_of(title)] = JobDefinition(
|
||||
job_id=job_id_of(title),
|
||||
title=title,
|
||||
category=meta.get("category", ""),
|
||||
industries=list(meta.get("industries", [])),
|
||||
required_skills=required,
|
||||
preferred_skills=[s for s, c in pref.most_common()
|
||||
if c >= max(2, n * 0.15) and s not in required],
|
||||
responsibilities=[d for d, _ in duty.most_common(5)],
|
||||
scenarios=[s for s, _ in scen.most_common(5)],
|
||||
expected_years=int(meta.get("expected_years", 3)),
|
||||
status="active",
|
||||
source_count=n,
|
||||
company_count=len({r.company for r in group}),
|
||||
)
|
||||
return defs
|
||||
|
||||
|
||||
def run_pipeline(config: dict | None = None) -> dict:
|
||||
"""执行全链路，返回统计信息。"""
|
||||
cfg = config or load_config()
|
||||
stats: dict = {"stages": {}}
|
||||
|
||||
# 1. 导入
|
||||
records, errors = load_jd_dir(resolve(cfg, "jd_raw_dir"))
|
||||
source_stats = {"jd_raw": len(records)}
|
||||
# 附加数据源（如 data/real/autoweaks_jd 真实数据集），可配置切换
|
||||
for extra in str(cfg.get("jd_extra_dirs") or "").split(","):
|
||||
extra = extra.strip()
|
||||
if not extra:
|
||||
continue
|
||||
extra_path = Path(extra) if Path(extra).is_absolute() else ROOT / extra
|
||||
if not extra_path.exists():
|
||||
logger.warning("附加数据源不存在，已跳过: %s", extra_path)
|
||||
continue
|
||||
recs2, errs2 = load_jd_dir(extra_path)
|
||||
records.extend(recs2)
|
||||
errors.extend(errs2)
|
||||
source_stats[extra_path.name] = len(recs2)
|
||||
logger.info("附加数据源 %s: %d 条 JD", extra_path.name, len(recs2))
|
||||
stats["stages"]["ingest"] = {"records": len(records), "bad_lines": len(errors),
|
||||
"sources": source_stats}
|
||||
|
||||
# 2. 清洗：噪声 → 近重复 → 时间衰减
|
||||
filter_noise(records)
|
||||
dup_clusters = find_near_duplicates(records)
|
||||
apply_decay(records, lam=float(cfg.get("decay_lambda", 0.35)))
|
||||
stats["stages"]["cleaning"] = {
|
||||
"dup_clusters": len(dup_clusters),
|
||||
"duplicates": sum(1 for r in records if r.is_duplicate),
|
||||
"noise": sum(1 for r in records if r.is_noise),
|
||||
}
|
||||
|
||||
# 3. 抽取（LLM 或离线，自动降级）
|
||||
lexicon = load_lexicon(resolve(cfg, "lexicon_path"))
|
||||
extractor = get_extractor(cfg)
|
||||
extractions: dict[str, ExtractionResult] = {}
|
||||
for r in records:
|
||||
if not r.is_noise:
|
||||
extractions[r.jid] = extractor.extract(r)
|
||||
stats["stages"]["extract"] = {"extracted": len(extractions),
|
||||
"extractor": extractor.name}
|
||||
|
||||
# 4. 图谱写入：既有岗位定义 + 观察流
|
||||
first_years: dict[str, int] = {}
|
||||
for r in records:
|
||||
if r.jid in extractions:
|
||||
t = extractions[r.jid].job_title
|
||||
first_years[t] = min(first_years.get(t, 9999), r.publish_date.year)
|
||||
known_titles = {t for t, y in first_years.items() if y < KNOWN_JOB_BEFORE_YEAR}
|
||||
|
||||
kg = TemporalKG()
|
||||
defs = build_job_definitions(records, extractions, known_titles, lexicon)
|
||||
for d in defs.values():
|
||||
kg.add_job(d)
|
||||
for sc in d.scenarios:
|
||||
kg.add_scenario(sc)
|
||||
kg.g.add_edge(d.job_id, f"场景::{sc}", rel="APPLIES_TO")
|
||||
for ind in d.industries:
|
||||
kg.add_industry(ind)
|
||||
kg.g.add_edge(d.job_id, f"行业::{ind}", rel="BELONGS_TO")
|
||||
|
||||
job_id_of_title = {d.title: d.job_id for d in defs.values()}
|
||||
observations = build_observations(records, extractions, job_id_of_title)
|
||||
for obs in observations:
|
||||
kg.observe(obs)
|
||||
stats["stages"]["graph"] = kg.stats()
|
||||
|
||||
# 5. 演化分析（事件追加进图谱）
|
||||
events = analyze(kg, observations=observations)
|
||||
stats["stages"]["evolution"] = {"events": len(events)}
|
||||
|
||||
# 6. 新岗位发现 → 待审核分区（SQLite 状态库）
|
||||
candidates = discover_new_jobs(records, extractions, known_titles)
|
||||
save_pending(candidates, resolve(cfg, "state_db_path"))
|
||||
stats["stages"]["discover"] = {"candidates": [c.title for c in candidates]}
|
||||
|
||||
# 7. 人工状态重放：审核入库岗位强制纳入正式图谱（不受"2023 年前"规则限制），
|
||||
# 节点编辑一并恢复（覆写层存于 SQLite job_overrides 表）
|
||||
from app.services.overrides import replay_overrides
|
||||
stats["stages"]["overrides"] = replay_overrides(kg, resolve(cfg, "state_db_path"))
|
||||
|
||||
kg.save(resolve(cfg, "graph_path"))
|
||||
stats["graph_path"] = str(resolve(cfg, "graph_path"))
|
||||
logger.info("pipeline 完成：%s", stats["stages"])
|
||||
return stats
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import json
|
||||
print(json.dumps(run_pipeline(), ensure_ascii=False, indent=2))
|
||||
Reference in New Issue
Block a user