CareerNexus 职联：岗位能力图谱构建与动态演化分析平台（挑战杯 XH-202621）

- 全链路：多源JD接入/清洗交叉验证/结构化抽取(星火+离线双通道)/时序知识图谱/新岗位发现/演化分析/人岗匹配/学习路径 - WebUI 五视图：驾驶舱/图谱工作台/岗位演化/匹配(单人+批量)/学习路径 - 设置中心(config.json)/Token鉴权/SQLite状态库/语义向量通道(BGE-M3本地或联网API) - 71 个 pytest 用例；数据目录仅保留结构(.gitkeep)，数据由 scripts/generate_dataset.py 复现
2026-08-03 13:15:35 +08:00
commit a10442d78c
78 changed files with 12860 additions and 0 deletions
@@ -0,0 +1,251 @@
+"""指标复现脚本：真实“从文本抽取 → 与 gold 比对”流程，不硬编码任何答案。
+
+三项硬指标（合成测试集）：
+1. JD 解析准确率   —— 对 data/testset/jd_testset.jsonl 重新抽取，与 gold_answers.json
+   算字段级 P/R/F1（必备技能/加分技能/职责/场景 micro-F1 + 岗位名称准确率）；
+2. 简历提取准确率 —— 对 data/resumes/*.txt 解析技能，与 gold.json 算 F1；
+3. 人岗匹配准确率 —— 对 data/testset/match_gold.json 标注对跑匹配引擎，
+   判定“命中预期匹配/不匹配”的比例。
+
+两个真实数据维度（data/real/，与合成集分开报告、不混算）：
+4. 真实 JD（AutoWeakS 初标+抽检 silver gold）—— 对 jd_sample.jsonl 重新抽取，
+   与 gold_answers.json 算技能/场景字段级 F1；
+5. 真实简历（AIStudio CC0）—— 对 50 份简历解析，姓名/电话命中率 + 技能 F1。
+
+输出 reports/metrics.json 与控制台报告。
+"""
+from __future__ import annotations
+
+import json
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+
+from app.extract.offline import OfflineExtractor  # noqa: E402
+from app.extract.base import SkillMatcher, load_lexicon  # noqa: E402
+from app.graph.tkg import TemporalKG  # noqa: E402
+from app.ingest.loader import load_jd_file  # noqa: E402
+from app.match.engine import MatchEngine  # noqa: E402
+from app.match.resume import parse_resume  # noqa: E402
+from app.models import JDRecord  # noqa: E402
+
+
+def prf(tp: int, pred: int, gold: int) -> tuple[float, float, float]:
+    p = tp / pred if pred else 0.0
+    r = tp / gold if gold else 0.0
+    f1 = 2 * p * r / (p + r) if p + r else 0.0
+    return round(p, 4), round(r, 4), round(f1, 4)
+
+
+def eval_jd_extraction() -> dict:
+    """JD 解析：从文本重新抽取，与标准答案做字段级比对。"""
+    records, errors = load_jd_file(ROOT / "data" / "testset" / "jd_testset.jsonl")
+    gold = json.loads((ROOT / "data" / "testset" / "gold_answers.json").read_text(encoding="utf-8"))
+    extractor = OfflineExtractor(load_lexicon())
+
+    fields = ["required_skills", "preferred_skills", "responsibilities", "scenarios"]
+    tp = {f: 0 for f in fields}
+    pred = {f: 0 for f in fields}
+    gold_n = {f: 0 for f in fields}
+    title_hit = 0
+    for rec in records:
+        g = gold[rec.jid]
+        ext = extractor.extract(rec)
+        title_hit += int(ext.job_title == g["job_title"])
+        for f in fields:
+            got, want = set(getattr(ext, f)), set(g[f])
+            tp[f] += len(got & want)
+            pred[f] += len(got)
+            gold_n[f] += len(want)
+
+    detail = {}
+    f1s = []
+    for f in fields:
+        p, r, f1 = prf(tp[f], pred[f], gold_n[f])
+        detail[f] = {"precision": p, "recall": r, "f1": f1}
+        f1s.append(f1)
+    title_acc = round(title_hit / len(records), 4)
+    # 综合准确率：技能字段权重更高（技能是岗位图谱的核心）
+    accuracy = round(0.6 * (f1s[0] + f1s[1]) / 2 + 0.2 * f1s[2] + 0.1 * f1s[3] + 0.1 * title_acc, 4)
+    return {"accuracy": accuracy, "title_accuracy": title_acc, "fields": detail,
+            "samples": len(records), "bad_lines": len(errors)}
+
+
+def eval_resume() -> dict:
+    """简历技能提取准确率（逐份 F1 取平均）。"""
+    gold = json.loads((ROOT / "data" / "resumes" / "gold.json").read_text(encoding="utf-8"))
+    matcher = SkillMatcher(load_lexicon())
+    f1s, detail = [], {}
+    for name, g in gold.items():
+        text = (ROOT / "data" / "resumes" / name).read_text(encoding="utf-8")
+        profile = parse_resume(text, matcher)
+        got, want = set(profile.skills), set(g["skills"])
+        p, r, f1 = prf(len(got & want), len(got), len(want))
+        f1s.append(f1)
+        detail[name] = {"precision": p, "recall": r, "f1": f1,
+                        "extra": sorted(got - want), "missing": sorted(want - got)}
+    return {"accuracy": round(sum(f1s) / len(f1s), 4), "detail": detail}
+
+
+def eval_match() -> dict:
+    """人岗匹配准确率：命中预期匹配/不匹配。"""
+    graph_path = ROOT / "data" / "graph.json"
+    if not graph_path.exists():
+        from app.services.pipeline import run_pipeline
+        run_pipeline()
+    kg = TemporalKG.load(graph_path)
+    by_title = {j.title: j for j in kg.jobs.values()}
+    pairs = json.loads((ROOT / "data" / "testset" / "match_gold.json").read_text(encoding="utf-8"))
+    engine = MatchEngine(load_lexicon())
+    from app.models import ResumeProfile
+    hit, detail = 0, []
+    for pair in pairs:
+        job = by_title.get(pair["job_title"])
+        if job is None:
+            continue
+        profile = ResumeProfile(skills=pair["skills"], years=pair["years"],
+                                scenarios=pair["scenarios"])
+        result = engine.match(profile, job)
+        ok = result.is_match == pair["expected_match"]
+        hit += int(ok)
+        detail.append({"job": pair["job_title"], "expected": pair["expected_match"],
+                       "score": result.total_score, "ok": ok})
+    return {"accuracy": round(hit / len(pairs), 4), "pairs": len(pairs), "detail": detail}
+
+
+def eval_real_jd() -> dict:
+    """真实 JD（AutoWeakS）：从文本重新抽取，与 silver gold（初标+抽检）字段级比对。
+
+    真实 JD 是自由文本、无编号版面，gold 的 responsibilities/preferred_skills 为空，
+    口径 = 全文词表技能（required_skills）+ 应用场景 + 启发式岗位名，见 data/real/README.md。
+    """
+    records, errors = load_jd_file(ROOT / "data" / "real" / "autoweaks_jd" / "jd_sample.jsonl")
+    gold = json.loads(
+        (ROOT / "data" / "real" / "autoweaks_jd" / "gold_answers.json").read_text(encoding="utf-8"))
+    extractor = OfflineExtractor(load_lexicon())
+
+    fields = ["required_skills", "preferred_skills", "scenarios"]
+    tp = {f: 0 for f in fields}
+    pred = {f: 0 for f in fields}
+    gold_n = {f: 0 for f in fields}
+    title_hit = 0
+    for rec in records:
+        g = gold[rec.jid]
+        ext = extractor.extract(rec)
+        title_hit += int(ext.job_title == g["job_title"])
+        for f in fields:
+            got, want = set(getattr(ext, f)), set(g[f])
+            tp[f] += len(got & want)
+            pred[f] += len(got)
+            gold_n[f] += len(want)
+
+    detail, active_f1 = {}, []
+    for f in fields:
+        if not gold_n[f] and not pred[f]:
+            detail[f] = None      # 该字段在真实语料中无标注口径，不参与
+            continue
+        p, r, f1 = prf(tp[f], pred[f], gold_n[f])
+        detail[f] = {"precision": p, "recall": r, "f1": f1}
+        active_f1.append((f, f1))
+    weights = {"required_skills": 0.7, "preferred_skills": 0.2, "scenarios": 0.1}
+    wsum = sum(weights[f] for f, _ in active_f1) or 1.0
+    accuracy = round(sum(weights[f] * f1 for f, f1 in active_f1) / wsum, 4)
+    return {"accuracy": accuracy,
+            "title_accuracy": round(title_hit / len(records), 4),
+            "fields": detail, "samples": len(records), "bad_lines": len(errors)}
+
+
+def eval_real_resume() -> dict:
+    """真实简历（AIStudio CC0）：姓名/电话命中率（脱敏形态比对）+ 技能 F1。"""
+    gold = json.loads(
+        (ROOT / "data" / "real" / "aistudio_resumes" / "gold.json").read_text(encoding="utf-8"))
+    matcher = SkillMatcher(load_lexicon())
+    f1s, name_hit, phone_hit, phone_total = [], 0, 0, 0
+    empty_skills = 0
+    for name, g in gold.items():
+        text = (ROOT / "data" / "real" / "aistudio_resumes" / name).read_text(encoding="utf-8")
+        profile = parse_resume(text, matcher)
+        # gold 存明文，解析结果是脱敏形态（姓氏+* / 138****5678），按脱敏形态比对
+        want_name = g["name"][0] + "*" * (len(g["name"]) - 1)
+        name_hit += int(profile.name == want_name)
+        if g.get("phone"):
+            phone_total += 1
+            want_phone = g["phone"][:3] + "****" + g["phone"][-2:]
+            phone_hit += int(profile.phone == want_phone)
+        got, want = set(profile.skills), set(g["skills"])
+        if not got and not want:
+            empty_skills += 1
+            f1s.append(1.0)      # 双方均无词表技能 = 完全一致（真实简历大量非技术内容）
+            continue
+        _, _, f1 = prf(len(got & want), len(got), len(want))
+        f1s.append(f1)
+    n = len(gold)
+    return {"accuracy": round(sum(f1s) / n, 4),
+            "name_accuracy": round(name_hit / n, 4),
+            "phone_accuracy": round(phone_hit / phone_total, 4) if phone_total else 0.0,
+            "samples": n, "phone_samples": phone_total,
+            "empty_skill_resumes": empty_skills}
+
+
+def main() -> None:
+    jd = eval_jd_extraction()
+    resume = eval_resume()
+    match = eval_match()
+    real_jd = eval_real_jd()
+    real_resume = eval_real_resume()
+    metrics = {
+        "jd_extraction_accuracy": jd["accuracy"],
+        "resume_extraction_accuracy": resume["accuracy"],
+        "match_accuracy": match["accuracy"],
+        "jd_detail": jd,
+        "resume_detail": resume["detail"],
+        "match_detail": match["detail"],
+        "real_jd": real_jd,
+        "real_resume": real_resume,
+    }
+    out = ROOT / "reports"
+    out.mkdir(exist_ok=True)
+    (out / "metrics.json").write_text(
+        json.dumps(metrics, ensure_ascii=False, indent=1), encoding="utf-8")
+
+    print("=" * 56)
+    print("CareerNexus 指标复现报告")
+    print("=" * 56)
+    print(f"JD 解析准确率    : {jd['accuracy']:.2%}  (样本 {jd['samples']} 条)")
+    for f, d in jd["fields"].items():
+        print(f"  - {f:<18} P={d['precision']:.2%} R={d['recall']:.2%} F1={d['f1']:.2%}")
+    print(f"  - 岗位名称准确率     {jd['title_accuracy']:.2%}")
+    print(f"简历提取准确率  : {resume['accuracy']:.2%}")
+    for name, d in resume["detail"].items():
+        print(f"  - {name:<22} F1={d['f1']:.2%}")
+    print(f"人岗匹配准确率  : {match['accuracy']:.2%}  (标注对 {match['pairs']} 对)")
+    print("-" * 56)
+    print("真实数据维度（data/real/，与合成集分开报告）")
+    print(f"真实 JD（AutoWeakS 初标+抽检）: 综合 F1 {real_jd['accuracy']:.2%}"
+          f"  (样本 {real_jd['samples']} 条)")
+    for f, d in real_jd["fields"].items():
+        if d is None:
+            print(f"  - {f:<18} (真实语料无标注口径，不参与)")
+        else:
+            print(f"  - {f:<18} P={d['precision']:.2%} R={d['recall']:.2%} F1={d['f1']:.2%}")
+    print(f"  - 岗位名称准确率     {real_jd['title_accuracy']:.2%} (启发式标注)")
+    print(f"真实简历（AIStudio CC0）    : 技能 F1 {real_resume['accuracy']:.2%}"
+          f"  姓名命中 {real_resume['name_accuracy']:.2%}"
+          f"  电话命中 {real_resume['phone_accuracy']:.2%}  ({real_resume['samples']} 份)")
+    print(f"\n指标已写入 {out / 'metrics.json'}")
+    target = 0.90
+    for name, v in [("JD解析", jd["accuracy"]), ("简历提取", resume["accuracy"]),
+                    ("人岗匹配", match["accuracy"])]:
+        status = "达标" if v >= target else "未达标"
+        print(f"[{status}] {name} {v:.2%} (目标 ≥{target:.0%})")
+    real_targets = [("真实JD技能F1", real_jd["fields"]["required_skills"]["f1"], 0.80),
+                    ("真实姓名命中", real_resume["name_accuracy"], 1.00)]
+    for name, v, t in real_targets:
+        status = "达标" if v >= t else "未达标"
+        print(f"[{status}] {name} {v:.2%} (目标 ≥{t:.0%})")
+
+
+if __name__ == "__main__":
+    main()