RoyirinRosmontis
1722c00249
- scripts/expand_real_dataset.py：AutoWeakS 全量 2215 条 + 500 条分层评测集 + 简历扩量（seed=2026 可复现） - pipeline 被动语料机制（passive_sources）：真实无日期 JD 参与接入/清洗/抽取验证（2915 条命中率 99.8%），不进图谱/演化/发现 - 新岗位发现第四指标'差异度'与双年窗口增长率口径 - 评测新增 real_jd_500 维度（99.91%）；测试会话隔离 jd_extra_dirs 防止本机配置拖慢套件
277 lines
12 KiB
Python
277 lines
12 KiB
Python
"""指标复现脚本：真实“从文本抽取 → 与 gold 比对”流程，不硬编码任何答案。
|
||
|
||
三项硬指标（合成测试集）：
|
||
1. JD 解析准确率 —— 对 data/testset/jd_testset.jsonl 重新抽取，与 gold_answers.json
|
||
算字段级 P/R/F1（必备技能/加分技能/职责/场景 micro-F1 + 岗位名称准确率）；
|
||
2. 简历提取准确率 —— 对 data/resumes/*.txt 解析技能，与 gold.json 算 F1；
|
||
3. 人岗匹配准确率 —— 对 data/testset/match_gold.json 标注对跑匹配引擎，
|
||
判定“命中预期匹配/不匹配”的比例。
|
||
|
||
两个真实数据维度（data/real/，与合成集分开报告、不混算）：
|
||
4. 真实 JD（AutoWeakS 初标+抽检 silver gold）—— 对 jd_sample.jsonl 重新抽取，
|
||
与 gold_answers.json 算技能/场景字段级 F1；另按 jd_eval_500.jsonl /
|
||
gold_answers_500.json 报告 500 条分层口径（real_jd_500，包含原 200 条）；
|
||
5. 真实简历（AIStudio CC0）—— 对 200 份简历解析，姓名/电话命中率 + 技能 F1。
|
||
|
||
输出 reports/metrics.json 与控制台报告。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import json
|
||
import sys
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parents[1]
|
||
sys.path.insert(0, str(ROOT))
|
||
|
||
from app.extract.offline import OfflineExtractor # noqa: E402
|
||
from app.extract.base import SkillMatcher, load_lexicon # noqa: E402
|
||
from app.graph.tkg import TemporalKG # noqa: E402
|
||
from app.ingest.loader import load_jd_file # noqa: E402
|
||
from app.match.engine import MatchEngine # noqa: E402
|
||
from app.match.resume import parse_resume # noqa: E402
|
||
from app.models import JDRecord # noqa: E402
|
||
|
||
|
||
def prf(tp: int, pred: int, gold: int) -> tuple[float, float, float]:
|
||
p = tp / pred if pred else 0.0
|
||
r = tp / gold if gold else 0.0
|
||
f1 = 2 * p * r / (p + r) if p + r else 0.0
|
||
return round(p, 4), round(r, 4), round(f1, 4)
|
||
|
||
|
||
def eval_jd_extraction() -> dict:
|
||
"""JD 解析：从文本重新抽取，与标准答案做字段级比对。"""
|
||
records, errors = load_jd_file(ROOT / "data" / "testset" / "jd_testset.jsonl")
|
||
gold = json.loads((ROOT / "data" / "testset" / "gold_answers.json").read_text(encoding="utf-8"))
|
||
extractor = OfflineExtractor(load_lexicon())
|
||
|
||
fields = ["required_skills", "preferred_skills", "responsibilities", "scenarios"]
|
||
tp = {f: 0 for f in fields}
|
||
pred = {f: 0 for f in fields}
|
||
gold_n = {f: 0 for f in fields}
|
||
title_hit = 0
|
||
for rec in records:
|
||
g = gold[rec.jid]
|
||
ext = extractor.extract(rec)
|
||
title_hit += int(ext.job_title == g["job_title"])
|
||
for f in fields:
|
||
got, want = set(getattr(ext, f)), set(g[f])
|
||
tp[f] += len(got & want)
|
||
pred[f] += len(got)
|
||
gold_n[f] += len(want)
|
||
|
||
detail = {}
|
||
f1s = []
|
||
for f in fields:
|
||
p, r, f1 = prf(tp[f], pred[f], gold_n[f])
|
||
detail[f] = {"precision": p, "recall": r, "f1": f1}
|
||
f1s.append(f1)
|
||
title_acc = round(title_hit / len(records), 4)
|
||
# 综合准确率：技能字段权重更高（技能是岗位图谱的核心）
|
||
accuracy = round(0.6 * (f1s[0] + f1s[1]) / 2 + 0.2 * f1s[2] + 0.1 * f1s[3] + 0.1 * title_acc, 4)
|
||
return {"accuracy": accuracy, "title_accuracy": title_acc, "fields": detail,
|
||
"samples": len(records), "bad_lines": len(errors)}
|
||
|
||
|
||
def eval_resume() -> dict:
|
||
"""简历技能提取准确率（逐份 F1 取平均）。"""
|
||
gold = json.loads((ROOT / "data" / "resumes" / "gold.json").read_text(encoding="utf-8"))
|
||
matcher = SkillMatcher(load_lexicon())
|
||
f1s, detail = [], {}
|
||
for name, g in gold.items():
|
||
text = (ROOT / "data" / "resumes" / name).read_text(encoding="utf-8")
|
||
profile = parse_resume(text, matcher)
|
||
got, want = set(profile.skills), set(g["skills"])
|
||
p, r, f1 = prf(len(got & want), len(got), len(want))
|
||
f1s.append(f1)
|
||
detail[name] = {"precision": p, "recall": r, "f1": f1,
|
||
"extra": sorted(got - want), "missing": sorted(want - got)}
|
||
return {"accuracy": round(sum(f1s) / len(f1s), 4), "detail": detail}
|
||
|
||
|
||
def eval_match() -> dict:
|
||
"""人岗匹配准确率：命中预期匹配/不匹配。"""
|
||
graph_path = ROOT / "data" / "graph.json"
|
||
if not graph_path.exists():
|
||
from app.services.pipeline import run_pipeline
|
||
run_pipeline()
|
||
kg = TemporalKG.load(graph_path)
|
||
by_title = {j.title: j for j in kg.jobs.values()}
|
||
pairs = json.loads((ROOT / "data" / "testset" / "match_gold.json").read_text(encoding="utf-8"))
|
||
engine = MatchEngine(load_lexicon())
|
||
from app.models import ResumeProfile
|
||
hit, detail = 0, []
|
||
for pair in pairs:
|
||
job = by_title.get(pair["job_title"])
|
||
if job is None:
|
||
continue
|
||
profile = ResumeProfile(skills=pair["skills"], years=pair["years"],
|
||
scenarios=pair["scenarios"])
|
||
result = engine.match(profile, job)
|
||
ok = result.is_match == pair["expected_match"]
|
||
hit += int(ok)
|
||
detail.append({"job": pair["job_title"], "expected": pair["expected_match"],
|
||
"score": result.total_score, "ok": ok})
|
||
return {"accuracy": round(hit / len(pairs), 4), "pairs": len(pairs), "detail": detail}
|
||
|
||
|
||
def _eval_real_jd(records_path: Path, gold_path: Path) -> dict:
|
||
"""真实 JD（AutoWeakS）：从文本重新抽取，与 silver gold（初标+抽检）字段级比对。
|
||
|
||
真实 JD 是自由文本、无编号版面，gold 的 responsibilities/preferred_skills 为空，
|
||
口径 = 全文词表技能（required_skills）+ 应用场景 + 启发式岗位名，见 data/real/README.md。
|
||
"""
|
||
records, errors = load_jd_file(records_path)
|
||
gold = json.loads(gold_path.read_text(encoding="utf-8"))
|
||
extractor = OfflineExtractor(load_lexicon())
|
||
|
||
fields = ["required_skills", "preferred_skills", "scenarios"]
|
||
tp = {f: 0 for f in fields}
|
||
pred = {f: 0 for f in fields}
|
||
gold_n = {f: 0 for f in fields}
|
||
title_hit = 0
|
||
for rec in records:
|
||
g = gold[rec.jid]
|
||
ext = extractor.extract(rec)
|
||
title_hit += int(ext.job_title == g["job_title"])
|
||
for f in fields:
|
||
got, want = set(getattr(ext, f)), set(g[f])
|
||
tp[f] += len(got & want)
|
||
pred[f] += len(got)
|
||
gold_n[f] += len(want)
|
||
|
||
detail, active_f1 = {}, []
|
||
for f in fields:
|
||
# 真实 JD 的 gold 口径：responsibilities/preferred_skills 为空即不参与评分
|
||
# （即使抽取器从"优先考虑"字样产出加分候选，也不计入，口径见 data/real/README.md）
|
||
if not gold_n[f]:
|
||
detail[f] = None # 该字段在真实语料中无标注口径，不参与
|
||
continue
|
||
p, r, f1 = prf(tp[f], pred[f], gold_n[f])
|
||
detail[f] = {"precision": p, "recall": r, "f1": f1}
|
||
active_f1.append((f, f1))
|
||
weights = {"required_skills": 0.7, "preferred_skills": 0.2, "scenarios": 0.1}
|
||
wsum = sum(weights[f] for f, _ in active_f1) or 1.0
|
||
accuracy = round(sum(weights[f] * f1 for f, f1 in active_f1) / wsum, 4)
|
||
return {"accuracy": accuracy,
|
||
"title_accuracy": round(title_hit / len(records), 4),
|
||
"fields": detail, "samples": len(records), "bad_lines": len(errors)}
|
||
|
||
|
||
def eval_real_jd() -> dict:
|
||
"""真实 JD 200 条口径（历史基线）。"""
|
||
return _eval_real_jd(
|
||
ROOT / "data" / "real" / "autoweaks_jd" / "jd_sample.jsonl",
|
||
ROOT / "data" / "real" / "autoweaks_jd" / "gold_answers.json")
|
||
|
||
|
||
def eval_real_jd_500() -> dict:
|
||
"""真实 JD 500 条分层口径（含原 200 条，见 scripts/expand_real_dataset.py）。"""
|
||
return _eval_real_jd(
|
||
ROOT / "data" / "real" / "autoweaks_jd" / "jd_eval_500.jsonl",
|
||
ROOT / "data" / "real" / "autoweaks_jd" / "gold_answers_500.json")
|
||
|
||
|
||
def eval_real_resume() -> dict:
|
||
"""真实简历（AIStudio CC0）：姓名/电话命中率（脱敏形态比对）+ 技能 F1。"""
|
||
gold = json.loads(
|
||
(ROOT / "data" / "real" / "aistudio_resumes" / "gold.json").read_text(encoding="utf-8"))
|
||
matcher = SkillMatcher(load_lexicon())
|
||
f1s, name_hit, phone_hit, phone_total = [], 0, 0, 0
|
||
empty_skills = 0
|
||
for name, g in gold.items():
|
||
text = (ROOT / "data" / "real" / "aistudio_resumes" / name).read_text(encoding="utf-8")
|
||
profile = parse_resume(text, matcher)
|
||
# gold 存明文，解析结果是脱敏形态（姓氏+* / 138****5678），按脱敏形态比对
|
||
want_name = g["name"][0] + "*" * (len(g["name"]) - 1)
|
||
name_hit += int(profile.name == want_name)
|
||
if g.get("phone"):
|
||
phone_total += 1
|
||
want_phone = g["phone"][:3] + "****" + g["phone"][-2:]
|
||
phone_hit += int(profile.phone == want_phone)
|
||
got, want = set(profile.skills), set(g["skills"])
|
||
if not got and not want:
|
||
empty_skills += 1
|
||
f1s.append(1.0) # 双方均无词表技能 = 完全一致（真实简历大量非技术内容）
|
||
continue
|
||
_, _, f1 = prf(len(got & want), len(got), len(want))
|
||
f1s.append(f1)
|
||
n = len(gold)
|
||
return {"accuracy": round(sum(f1s) / n, 4),
|
||
"name_accuracy": round(name_hit / n, 4),
|
||
"phone_accuracy": round(phone_hit / phone_total, 4) if phone_total else 0.0,
|
||
"samples": n, "phone_samples": phone_total,
|
||
"empty_skill_resumes": empty_skills}
|
||
|
||
|
||
def main() -> None:
|
||
jd = eval_jd_extraction()
|
||
resume = eval_resume()
|
||
match = eval_match()
|
||
real_jd = eval_real_jd()
|
||
real_jd_500 = eval_real_jd_500()
|
||
real_resume = eval_real_resume()
|
||
metrics = {
|
||
"jd_extraction_accuracy": jd["accuracy"],
|
||
"resume_extraction_accuracy": resume["accuracy"],
|
||
"match_accuracy": match["accuracy"],
|
||
"jd_detail": jd,
|
||
"resume_detail": resume["detail"],
|
||
"match_detail": match["detail"],
|
||
"real_jd": real_jd,
|
||
"real_jd_500": real_jd_500,
|
||
"real_resume": real_resume,
|
||
}
|
||
out = ROOT / "reports"
|
||
out.mkdir(exist_ok=True)
|
||
(out / "metrics.json").write_text(
|
||
json.dumps(metrics, ensure_ascii=False, indent=1), encoding="utf-8")
|
||
|
||
print("=" * 56)
|
||
print("CareerNexus 指标复现报告")
|
||
print("=" * 56)
|
||
print(f"JD 解析准确率 : {jd['accuracy']:.2%} (样本 {jd['samples']} 条)")
|
||
for f, d in jd["fields"].items():
|
||
print(f" - {f:<18} P={d['precision']:.2%} R={d['recall']:.2%} F1={d['f1']:.2%}")
|
||
print(f" - 岗位名称准确率 {jd['title_accuracy']:.2%}")
|
||
print(f"简历提取准确率 : {resume['accuracy']:.2%}")
|
||
for name, d in resume["detail"].items():
|
||
print(f" - {name:<22} F1={d['f1']:.2%}")
|
||
print(f"人岗匹配准确率 : {match['accuracy']:.2%} (标注对 {match['pairs']} 对)")
|
||
print("-" * 56)
|
||
print("真实数据维度（data/real/，与合成集分开报告）")
|
||
print(f"真实 JD（AutoWeakS 初标+抽检）: 综合 F1 {real_jd['accuracy']:.2%}"
|
||
f" (样本 {real_jd['samples']} 条)")
|
||
print(f"真实 JD（500 条分层） : 综合 F1 {real_jd_500['accuracy']:.2%}"
|
||
f" (样本 {real_jd_500['samples']} 条)")
|
||
for f, d in real_jd_500["fields"].items():
|
||
if d is None:
|
||
print(f" - {f:<18} (真实语料无标注口径，不参与)")
|
||
else:
|
||
print(f" - {f:<18} P={d['precision']:.2%} R={d['recall']:.2%} F1={d['f1']:.2%}")
|
||
for f, d in real_jd["fields"].items():
|
||
if d is None:
|
||
print(f" - {f:<18} (真实语料无标注口径，不参与)")
|
||
else:
|
||
print(f" - {f:<18} P={d['precision']:.2%} R={d['recall']:.2%} F1={d['f1']:.2%}")
|
||
print(f" - 岗位名称准确率 {real_jd['title_accuracy']:.2%} (启发式标注)")
|
||
print(f"真实简历（AIStudio CC0） : 技能 F1 {real_resume['accuracy']:.2%}"
|
||
f" 姓名命中 {real_resume['name_accuracy']:.2%}"
|
||
f" 电话命中 {real_resume['phone_accuracy']:.2%} ({real_resume['samples']} 份)")
|
||
print(f"\n指标已写入 {out / 'metrics.json'}")
|
||
target = 0.90
|
||
for name, v in [("JD解析", jd["accuracy"]), ("简历提取", resume["accuracy"]),
|
||
("人岗匹配", match["accuracy"])]:
|
||
status = "达标" if v >= target else "未达标"
|
||
print(f"[{status}] {name} {v:.2%} (目标 ≥{target:.0%})")
|
||
real_targets = [("真实JD技能F1", real_jd["fields"]["required_skills"]["f1"], 0.80),
|
||
("真实姓名命中", real_resume["name_accuracy"], 1.00)]
|
||
for name, v, t in real_targets:
|
||
status = "达标" if v >= t else "未达标"
|
||
print(f"[{status}] {name} {v:.2%} (目标 ≥{t:.0%})")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|