"""指标复现脚本：真实“从文本抽取 → 与 gold 比对”流程，不硬编码任何答案。 三项硬指标（合成测试集）： 1. JD 解析准确率 —— 对 data/testset/jd_testset.jsonl 重新抽取，与 gold_answers.json 算字段级 P/R/F1（必备技能/加分技能/职责/场景 micro-F1 + 岗位名称准确率）； 2. 简历提取准确率 —— 对 data/resumes/*.txt 解析技能，与 gold.json 算 F1； 3. 人岗匹配准确率 —— 对 data/testset/match_gold.json 标注对跑匹配引擎， 判定“命中预期匹配/不匹配”的比例。 两个真实数据维度（data/real/，与合成集分开报告、不混算）： 4. 真实 JD（AutoWeakS 初标+抽检 silver gold）—— 对 jd_sample.jsonl 重新抽取， 与 gold_answers.json 算技能/场景字段级 F1；另按 jd_eval_500.jsonl / gold_answers_500.json 报告 500 条分层口径（real_jd_500，包含原 200 条）； 5. 真实简历（AIStudio CC0）—— 对 200 份简历解析，姓名/电话命中率 + 技能 F1。 输出 reports/metrics.json 与控制台报告。 """ from __future__ import annotations import json import sys from pathlib import Path ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) from app.extract.offline import OfflineExtractor # noqa: E402 from app.extract.base import SkillMatcher, load_lexicon # noqa: E402 from app.graph.tkg import TemporalKG # noqa: E402 from app.ingest.loader import load_jd_file # noqa: E402 from app.match.engine import MatchEngine # noqa: E402 from app.match.resume import parse_resume # noqa: E402 from app.models import JDRecord # noqa: E402 def prf(tp: int, pred: int, gold: int) -> tuple[float, float, float]: p = tp / pred if pred else 0.0 r = tp / gold if gold else 0.0 f1 = 2 * p * r / (p + r) if p + r else 0.0 return round(p, 4), round(r, 4), round(f1, 4) def eval_jd_extraction() -> dict: """JD 解析：从文本重新抽取，与标准答案做字段级比对。""" records, errors = load_jd_file(ROOT / "data" / "testset" / "jd_testset.jsonl") gold = json.loads((ROOT / "data" / "testset" / "gold_answers.json").read_text(encoding="utf-8")) extractor = OfflineExtractor(load_lexicon()) fields = ["required_skills", "preferred_skills", "responsibilities", "scenarios"] tp = {f: 0 for f in fields} pred = {f: 0 for f in fields} gold_n = {f: 0 for f in fields} title_hit = 0 for rec in records: g = gold[rec.jid] ext = extractor.extract(rec) title_hit += int(ext.job_title == g["job_title"]) for f in fields: got, want = set(getattr(ext, f)), set(g[f]) tp[f] += len(got & want) pred[f] += len(got) gold_n[f] += len(want) detail = {} f1s = [] for f in fields: p, r, f1 = prf(tp[f], pred[f], gold_n[f]) detail[f] = {"precision": p, "recall": r, "f1": f1} f1s.append(f1) title_acc = round(title_hit / len(records), 4) # 综合准确率：技能字段权重更高（技能是岗位图谱的核心） accuracy = round(0.6 * (f1s[0] + f1s[1]) / 2 + 0.2 * f1s[2] + 0.1 * f1s[3] + 0.1 * title_acc, 4) return {"accuracy": accuracy, "title_accuracy": title_acc, "fields": detail, "samples": len(records), "bad_lines": len(errors)} def eval_resume() -> dict: """简历技能提取准确率（逐份 F1 取平均）。""" gold = json.loads((ROOT / "data" / "resumes" / "gold.json").read_text(encoding="utf-8")) matcher = SkillMatcher(load_lexicon()) f1s, detail = [], {} for name, g in gold.items(): text = (ROOT / "data" / "resumes" / name).read_text(encoding="utf-8") profile = parse_resume(text, matcher) got, want = set(profile.skills), set(g["skills"]) p, r, f1 = prf(len(got & want), len(got), len(want)) f1s.append(f1) detail[name] = {"precision": p, "recall": r, "f1": f1, "extra": sorted(got - want), "missing": sorted(want - got)} return {"accuracy": round(sum(f1s) / len(f1s), 4), "detail": detail} def eval_match() -> dict: """人岗匹配准确率：命中预期匹配/不匹配。""" graph_path = ROOT / "data" / "graph.json" if not graph_path.exists(): from app.services.pipeline import run_pipeline run_pipeline() kg = TemporalKG.load(graph_path) by_title = {j.title: j for j in kg.jobs.values()} pairs = json.loads((ROOT / "data" / "testset" / "match_gold.json").read_text(encoding="utf-8")) engine = MatchEngine(load_lexicon()) from app.models import ResumeProfile hit, detail = 0, [] for pair in pairs: job = by_title.get(pair["job_title"]) if job is None: continue profile = ResumeProfile(skills=pair["skills"], years=pair["years"], scenarios=pair["scenarios"]) result = engine.match(profile, job) ok = result.is_match == pair["expected_match"] hit += int(ok) detail.append({"job": pair["job_title"], "expected": pair["expected_match"], "score": result.total_score, "ok": ok}) return {"accuracy": round(hit / len(pairs), 4), "pairs": len(pairs), "detail": detail} def _eval_real_jd(records_path: Path, gold_path: Path) -> dict: """真实 JD（AutoWeakS）：从文本重新抽取，与 silver gold（初标+抽检）字段级比对。 真实 JD 是自由文本、无编号版面，gold 的 responsibilities/preferred_skills 为空， 口径 = 全文词表技能（required_skills）+ 应用场景 + 启发式岗位名，见 data/real/README.md。 """ records, errors = load_jd_file(records_path) gold = json.loads(gold_path.read_text(encoding="utf-8")) extractor = OfflineExtractor(load_lexicon()) fields = ["required_skills", "preferred_skills", "scenarios"] tp = {f: 0 for f in fields} pred = {f: 0 for f in fields} gold_n = {f: 0 for f in fields} title_hit = 0 for rec in records: g = gold[rec.jid] ext = extractor.extract(rec) title_hit += int(ext.job_title == g["job_title"]) for f in fields: got, want = set(getattr(ext, f)), set(g[f]) tp[f] += len(got & want) pred[f] += len(got) gold_n[f] += len(want) detail, active_f1 = {}, [] for f in fields: # 真实 JD 的 gold 口径：responsibilities/preferred_skills 为空即不参与评分 # （即使抽取器从"优先考虑"字样产出加分候选，也不计入，口径见 data/real/README.md） if not gold_n[f]: detail[f] = None # 该字段在真实语料中无标注口径，不参与 continue p, r, f1 = prf(tp[f], pred[f], gold_n[f]) detail[f] = {"precision": p, "recall": r, "f1": f1} active_f1.append((f, f1)) weights = {"required_skills": 0.7, "preferred_skills": 0.2, "scenarios": 0.1} wsum = sum(weights[f] for f, _ in active_f1) or 1.0 accuracy = round(sum(weights[f] * f1 for f, f1 in active_f1) / wsum, 4) return {"accuracy": accuracy, "title_accuracy": round(title_hit / len(records), 4), "fields": detail, "samples": len(records), "bad_lines": len(errors)} def eval_real_jd() -> dict: """真实 JD 200 条口径（历史基线）。""" return _eval_real_jd( ROOT / "data" / "real" / "autoweaks_jd" / "jd_sample.jsonl", ROOT / "data" / "real" / "autoweaks_jd" / "gold_answers.json") def eval_real_jd_500() -> dict: """真实 JD 500 条分层口径（含原 200 条，见 scripts/expand_real_dataset.py）。""" return _eval_real_jd( ROOT / "data" / "real" / "autoweaks_jd" / "jd_eval_500.jsonl", ROOT / "data" / "real" / "autoweaks_jd" / "gold_answers_500.json") def eval_real_resume() -> dict: """真实简历（AIStudio CC0）：姓名/电话命中率（脱敏形态比对）+ 技能 F1。""" gold = json.loads( (ROOT / "data" / "real" / "aistudio_resumes" / "gold.json").read_text(encoding="utf-8")) matcher = SkillMatcher(load_lexicon()) f1s, name_hit, phone_hit, phone_total = [], 0, 0, 0 empty_skills = 0 for name, g in gold.items(): text = (ROOT / "data" / "real" / "aistudio_resumes" / name).read_text(encoding="utf-8") profile = parse_resume(text, matcher) # gold 存明文，解析结果是脱敏形态（姓氏+* / 138****5678），按脱敏形态比对 want_name = g["name"][0] + "*" * (len(g["name"]) - 1) name_hit += int(profile.name == want_name) if g.get("phone"): phone_total += 1 want_phone = g["phone"][:3] + "****" + g["phone"][-2:] phone_hit += int(profile.phone == want_phone) got, want = set(profile.skills), set(g["skills"]) if not got and not want: empty_skills += 1 f1s.append(1.0) # 双方均无词表技能 = 完全一致（真实简历大量非技术内容） continue _, _, f1 = prf(len(got & want), len(got), len(want)) f1s.append(f1) n = len(gold) return {"accuracy": round(sum(f1s) / n, 4), "name_accuracy": round(name_hit / n, 4), "phone_accuracy": round(phone_hit / phone_total, 4) if phone_total else 0.0, "samples": n, "phone_samples": phone_total, "empty_skill_resumes": empty_skills} def main() -> None: jd = eval_jd_extraction() resume = eval_resume() match = eval_match() real_jd = eval_real_jd() real_jd_500 = eval_real_jd_500() real_resume = eval_real_resume() metrics = { "jd_extraction_accuracy": jd["accuracy"], "resume_extraction_accuracy": resume["accuracy"], "match_accuracy": match["accuracy"], "jd_detail": jd, "resume_detail": resume["detail"], "match_detail": match["detail"], "real_jd": real_jd, "real_jd_500": real_jd_500, "real_resume": real_resume, } out = ROOT / "reports" out.mkdir(exist_ok=True) (out / "metrics.json").write_text( json.dumps(metrics, ensure_ascii=False, indent=1), encoding="utf-8") print("=" * 56) print("CareerNexus 指标复现报告") print("=" * 56) print(f"JD 解析准确率 : {jd['accuracy']:.2%} (样本 {jd['samples']} 条)") for f, d in jd["fields"].items(): print(f" - {f:<18} P={d['precision']:.2%} R={d['recall']:.2%} F1={d['f1']:.2%}") print(f" - 岗位名称准确率 {jd['title_accuracy']:.2%}") print(f"简历提取准确率 : {resume['accuracy']:.2%}") for name, d in resume["detail"].items(): print(f" - {name:<22} F1={d['f1']:.2%}") print(f"人岗匹配准确率 : {match['accuracy']:.2%} (标注对 {match['pairs']} 对)") print("-" * 56) print("真实数据维度（data/real/，与合成集分开报告）") print(f"真实 JD（AutoWeakS 初标+抽检）: 综合 F1 {real_jd['accuracy']:.2%}" f" (样本 {real_jd['samples']} 条)") print(f"真实 JD（500 条分层） : 综合 F1 {real_jd_500['accuracy']:.2%}" f" (样本 {real_jd_500['samples']} 条)") for f, d in real_jd_500["fields"].items(): if d is None: print(f" - {f:<18} (真实语料无标注口径，不参与)") else: print(f" - {f:<18} P={d['precision']:.2%} R={d['recall']:.2%} F1={d['f1']:.2%}") for f, d in real_jd["fields"].items(): if d is None: print(f" - {f:<18} (真实语料无标注口径，不参与)") else: print(f" - {f:<18} P={d['precision']:.2%} R={d['recall']:.2%} F1={d['f1']:.2%}") print(f" - 岗位名称准确率 {real_jd['title_accuracy']:.2%} (启发式标注)") print(f"真实简历（AIStudio CC0） : 技能 F1 {real_resume['accuracy']:.2%}" f" 姓名命中 {real_resume['name_accuracy']:.2%}" f" 电话命中 {real_resume['phone_accuracy']:.2%} ({real_resume['samples']} 份)") print(f"\n指标已写入 {out / 'metrics.json'}") target = 0.90 for name, v in [("JD解析", jd["accuracy"]), ("简历提取", resume["accuracy"]), ("人岗匹配", match["accuracy"])]: status = "达标" if v >= target else "未达标" print(f"[{status}] {name} {v:.2%} (目标 ≥{target:.0%})") real_targets = [("真实JD技能F1", real_jd["fields"]["required_skills"]["f1"], 0.80), ("真实姓名命中", real_resume["name_accuracy"], 1.00)] for name, v, t in real_targets: status = "达标" if v >= t else "未达标" print(f"[{status}] {name} {v:.2%} (目标 ≥{t:.0%})") if __name__ == "__main__": main()