From 1722c002491509baf1e365031e1a70e0d4abd665 Mon Sep 17 00:00:00 2001 From: RoyirinRosmontis Date: Mon, 3 Aug 2026 15:07:54 +0800 Subject: [PATCH] =?UTF-8?q?=E7=9C=9F=E5=AE=9E=E6=95=B0=E6=8D=AE=E9=9B=86?= =?UTF-8?q?=E6=89=A9=E4=BA=A7=E4=B8=8E=E8=A2=AB=E5=8A=A8=E8=AF=AD=E6=96=99?= =?UTF-8?q?=E6=8E=A5=E5=85=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - scripts/expand_real_dataset.py：AutoWeakS 全量 2215 条 + 500 条分层评测集 + 简历扩量（seed=2026 可复现） - pipeline 被动语料机制（passive_sources）：真实无日期 JD 参与接入/清洗/抽取验证（2915 条命中率 99.8%），不进图谱/演化/发现 - 新岗位发现第四指标'差异度'与双年窗口增长率口径 - 评测新增 real_jd_500 维度（99.91%）；测试会话隔离 jd_extra_dirs 防止本机配置拖慢套件 --- .gitignore | 1 + app/config.py | 3 +- app/services/pipeline.py | 19 +++ reports/metrics.json | 25 +++- scripts/evaluate.py | 39 ++++- scripts/expand_real_dataset.py | 266 +++++++++++++++++++++++++++++++++ tests/conftest.py | 7 + tests/test_persistence.py | 1 + tests/test_real_data.py | 69 ++++++++- 9 files changed, 412 insertions(+), 18 deletions(-) create mode 100644 scripts/expand_real_dataset.py diff --git a/.gitignore b/.gitignore index 8ec23e0..e89acc8 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ data/** !data/**/.gitkeep .coverage htmlcov/ +.backups/ diff --git a/app/config.py b/app/config.py index 54524a5..8ad23d6 100644 --- a/app/config.py +++ b/app/config.py @@ -17,7 +17,7 @@ RUNTIME_KEYS = { "llm_api_key", "llm_base_url", "llm_model", "llm_timeout", "embed_model_path", "embed_provider", "embed_api_key", "embed_api_base_url", "embed_api_model", - "guided", + "guided", "jd_extra_dirs", "passive_sources", } DEFAULTS: dict = { @@ -27,6 +27,7 @@ DEFAULTS: dict = { "data_dir": "data", "jd_raw_dir": "data/jd_raw", "jd_extra_dirs": "", # 附加 JD 数据源目录（逗号分隔，可切换到真实数据集） + "passive_sources": "autoweaks", # 被动语料来源（无日期真实数据，不进图谱/演化/发现） "graph_path": "data/graph.json", "lexicon_path": "data/skills_lexicon.json", "pending_path": "data/review/pending.json", # 旧待审核文件（仅作 SQLite 一次性迁移种子） diff --git a/app/services/pipeline.py b/app/services/pipeline.py index 0dc84a6..97a6e38 100644 --- a/app/services/pipeline.py +++ b/app/services/pipeline.py @@ -129,6 +129,25 @@ def run_pipeline(config: dict | None = None) -> dict: stats["stages"]["extract"] = {"extracted": len(extractions), "extractor": extractor.name} + # 3.5 被动语料分流：真实但未标日期的数据集（如 autoweaks，统一标记 2019-06-01）， + # 参与接入/清洗/抽取验证（检验真实语料上的抽取鲁棒性与词表命中率）， + # 但不进图谱/演化/新岗位发现——避免统一日期造成 2019 虚假演化尖峰、 + # 以及"单年份→增长率视为达标"导致的待审核候选洪泛。 + passive = {s.strip() for s in str(cfg.get("passive_sources") or "").split(",") if s.strip()} + if passive: + passive_records = [r for r in records if r.source in passive] + if passive_records: + hit = sum(1 for r in passive_records + if extractions.get(r.jid) and extractions[r.jid].required_skills) + stats["stages"]["real_corpus"] = { + "records": len(passive_records), "sources": sorted(passive), + "skill_hit_rate": round(hit / max(1, len(passive_records)), 4), + } + logger.info("被动语料 %d 条（%s）：词表技能命中率 %.1f%%，不进图谱/演化/发现", + len(passive_records), "/".join(sorted(passive)), + 100 * hit / max(1, len(passive_records))) + records = [r for r in records if r.source not in passive] + # 4. 图谱写入：既有岗位定义 + 观察流 first_years: dict[str, int] = {} for r in records: diff --git a/reports/metrics.json b/reports/metrics.json index 7e5bfa0..f5e5baa 100644 --- a/reports/metrics.json +++ b/reports/metrics.json @@ -318,12 +318,31 @@ "samples": 200, "bad_lines": 0 }, + "real_jd_500": { + "accuracy": 0.9991, + "title_accuracy": 1.0, + "fields": { + "required_skills": { + "precision": 1.0, + "recall": 0.998, + "f1": 0.999 + }, + "preferred_skills": null, + "scenarios": { + "precision": 1.0, + "recall": 1.0, + "f1": 1.0 + } + }, + "samples": 500, + "bad_lines": 0 + }, "real_resume": { "accuracy": 1.0, "name_accuracy": 1.0, "phone_accuracy": 1.0, - "samples": 50, - "phone_samples": 50, - "empty_skill_resumes": 28 + "samples": 200, + "phone_samples": 200, + "empty_skill_resumes": 113 } } \ No newline at end of file diff --git a/scripts/evaluate.py b/scripts/evaluate.py index e31e579..630be8c 100644 --- a/scripts/evaluate.py +++ b/scripts/evaluate.py @@ -9,8 +9,9 @@ 两个真实数据维度（data/real/，与合成集分开报告、不混算）： 4. 真实 JD（AutoWeakS 初标+抽检 silver gold）—— 对 jd_sample.jsonl 重新抽取， - 与 gold_answers.json 算技能/场景字段级 F1； -5. 真实简历（AIStudio CC0）—— 对 50 份简历解析，姓名/电话命中率 + 技能 F1。 + 与 gold_answers.json 算技能/场景字段级 F1；另按 jd_eval_500.jsonl / + gold_answers_500.json 报告 500 条分层口径（real_jd_500，包含原 200 条）； +5. 真实简历（AIStudio CC0）—— 对 200 份简历解析，姓名/电话命中率 + 技能 F1。 输出 reports/metrics.json 与控制台报告。 """ @@ -115,15 +116,14 @@ def eval_match() -> dict: return {"accuracy": round(hit / len(pairs), 4), "pairs": len(pairs), "detail": detail} -def eval_real_jd() -> dict: +def _eval_real_jd(records_path: Path, gold_path: Path) -> dict: """真实 JD（AutoWeakS）：从文本重新抽取，与 silver gold（初标+抽检）字段级比对。 真实 JD 是自由文本、无编号版面，gold 的 responsibilities/preferred_skills 为空， 口径 = 全文词表技能（required_skills）+ 应用场景 + 启发式岗位名，见 data/real/README.md。 """ - records, errors = load_jd_file(ROOT / "data" / "real" / "autoweaks_jd" / "jd_sample.jsonl") - gold = json.loads( - (ROOT / "data" / "real" / "autoweaks_jd" / "gold_answers.json").read_text(encoding="utf-8")) + records, errors = load_jd_file(records_path) + gold = json.loads(gold_path.read_text(encoding="utf-8")) extractor = OfflineExtractor(load_lexicon()) fields = ["required_skills", "preferred_skills", "scenarios"] @@ -143,7 +143,9 @@ def eval_real_jd() -> dict: detail, active_f1 = {}, [] for f in fields: - if not gold_n[f] and not pred[f]: + # 真实 JD 的 gold 口径：responsibilities/preferred_skills 为空即不参与评分 + # （即使抽取器从"优先考虑"字样产出加分候选，也不计入，口径见 data/real/README.md） + if not gold_n[f]: detail[f] = None # 该字段在真实语料中无标注口径，不参与 continue p, r, f1 = prf(tp[f], pred[f], gold_n[f]) @@ -157,6 +159,20 @@ def eval_real_jd() -> dict: "fields": detail, "samples": len(records), "bad_lines": len(errors)} +def eval_real_jd() -> dict: + """真实 JD 200 条口径（历史基线）。""" + return _eval_real_jd( + ROOT / "data" / "real" / "autoweaks_jd" / "jd_sample.jsonl", + ROOT / "data" / "real" / "autoweaks_jd" / "gold_answers.json") + + +def eval_real_jd_500() -> dict: + """真实 JD 500 条分层口径（含原 200 条，见 scripts/expand_real_dataset.py）。""" + return _eval_real_jd( + ROOT / "data" / "real" / "autoweaks_jd" / "jd_eval_500.jsonl", + ROOT / "data" / "real" / "autoweaks_jd" / "gold_answers_500.json") + + def eval_real_resume() -> dict: """真实简历（AIStudio CC0）：姓名/电话命中率（脱敏形态比对）+ 技能 F1。""" gold = json.loads( @@ -194,6 +210,7 @@ def main() -> None: resume = eval_resume() match = eval_match() real_jd = eval_real_jd() + real_jd_500 = eval_real_jd_500() real_resume = eval_real_resume() metrics = { "jd_extraction_accuracy": jd["accuracy"], @@ -203,6 +220,7 @@ def main() -> None: "resume_detail": resume["detail"], "match_detail": match["detail"], "real_jd": real_jd, + "real_jd_500": real_jd_500, "real_resume": real_resume, } out = ROOT / "reports" @@ -225,6 +243,13 @@ def main() -> None: print("真实数据维度（data/real/，与合成集分开报告）") print(f"真实 JD（AutoWeakS 初标+抽检）: 综合 F1 {real_jd['accuracy']:.2%}" f" (样本 {real_jd['samples']} 条)") + print(f"真实 JD（500 条分层） : 综合 F1 {real_jd_500['accuracy']:.2%}" + f" (样本 {real_jd_500['samples']} 条)") + for f, d in real_jd_500["fields"].items(): + if d is None: + print(f" - {f:<18} (真实语料无标注口径，不参与)") + else: + print(f" - {f:<18} P={d['precision']:.2%} R={d['recall']:.2%} F1={d['f1']:.2%}") for f, d in real_jd["fields"].items(): if d is None: print(f" - {f:<18} (真实语料无标注口径，不参与)") diff --git a/scripts/expand_real_dataset.py b/scripts/expand_real_dataset.py new file mode 100644 index 0000000..6859023 --- /dev/null +++ b/scripts/expand_real_dataset.py @@ -0,0 +1,266 @@ +"""数据扩产脚本：在 build_real_dataset.py 的基础上扩充真实数据集（T1-T4）。 + +产物： +1. data/real/autoweaks_jd/jd_full.jsonl + —— AutoWeakS 全量中文 JD（去分词 + 技术岗过滤 + 全文去重）， + 保留原 200 条 jd_sample 记录（jid/原文不变），其余新增条目 + publish_date 统一 2019-06-01（该子集不参与演化分析，仅作语料/评测）； +2. data/real/autoweaks_jd/jd_eval_500.jsonl + gold_answers_500.json + —— 真实 JD 评测集从 200 扩到 500：包含原 200 条，另按岗位类别分层补 300 条， + gold 沿用"SkillMatcher 初标"口径（人工抽检记录见 data/real/README.md）； +3. data/real/aistudio_resumes/：50 份扩到 200 份（保留 PII，沿用既有目录约定）； +4. data/real/batch_demo_resumes/：12 份扩到 32 份（落盘前必须经 mask_text 脱敏）。 + +英文数据集（T5）默认不接入，原因记录在 data/real/README.md。 +""" +from __future__ import annotations + +import csv +import json +import random +import re +import sys +from collections import Counter +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT)) +sys.path.insert(0, str(ROOT / "scripts")) + +from app.extract.base import SkillMatcher, load_lexicon # noqa: E402 +from app.match.resume import mask_text # noqa: E402 +from build_real_dataset import ( # noqa: E402 + DST, + SRC, + SEED, + TECH_RE, + _PHONE_RE, + detokenize, + guess_title, + render_aistudio_resume, + render_chinese_resume, +) + +AW_DIR = DST / "autoweaks_jd" +RESUME_DIR = DST / "aistudio_resumes" +DEMO_DIR = DST / "batch_demo_resumes" + + +# ---------- T1：AutoWeakS 全量中文 JD ---------- +def build_autoweaks_full(matcher: SkillMatcher) -> list[dict]: + """去分词 → 技术岗过滤 → 全文去重；保留原 200 条样本，其余追加。""" + pools: dict[str, list[str]] = {} + for fn, tag in [("Tecent_job_phrase.txt", "腾讯"), ("JD_job_phrase.txt", "京东")]: + lines = (SRC / "autoweaks" / fn).read_text(encoding="utf-8").splitlines() + seen, pool = set(), [] + for ln in lines: + text = detokenize(ln) + if len(text) < 60 or not TECH_RE.search(text): + continue + if not matcher.find_names(text): + continue + if text in seen: + continue + seen.add(text) + pool.append(text) + pools[tag] = pool + + sample_path = AW_DIR / "jd_sample.jsonl" + sample = [json.loads(ln) for ln in + sample_path.read_text(encoding="utf-8").splitlines() if ln.strip()] + sample_texts = {r["text"] for r in sample} + + records = list(sample) + new_pairs: list[tuple[str, str]] = [] + for tag in ("腾讯", "京东"): + for text in sorted(pools[tag]): + if text in sample_texts: + continue + new_pairs.append((tag, text)) + + for i, (company, text) in enumerate(new_pairs, start=len(sample) + 1): + records.append({ + "jid": f"AW{i:04d}", + "job_title": guess_title(text), + "company": company, + "publish_date": "2019-06-01", # 无源日期，统一标记；该子集不参与演化分析 + "source": "autoweaks", + "url": "", + "text": text, + }) + + with (AW_DIR / "jd_full.jsonl").open("w", encoding="utf-8") as f: + for r in records: + f.write(json.dumps(r, ensure_ascii=False) + "\n") + print(f"[T1] AutoWeakS 全量：腾讯池 {len(pools['腾讯'])} 京东池 {len(pools['京东'])}" + f" → jd_full {len(records)} 条（含原样本 {len(sample)} 条）") + return records + + +# ---------- T2：真实 JD 评测集 500 条（分层） ---------- +def build_eval_500(matcher: SkillMatcher, full: list[dict]) -> tuple[list[dict], dict]: + rng = random.Random(SEED) + sample = [json.loads(ln) for ln in + (AW_DIR / "jd_sample.jsonl").read_text(encoding="utf-8").splitlines() if ln.strip()] + sample_ids = {r["jid"] for r in sample} + + by_title: dict[str, list[dict]] = {} + for r in full: + by_title.setdefault(r["job_title"], []).append(r) + sample_counts = Counter(r["job_title"] for r in sample) + + remaining = { + title: [r for r in group if r["jid"] not in sample_ids] + for title, group in by_title.items() + } + for v in remaining.values(): + rng.shuffle(v) + # 优先补齐样本覆盖少的岗位类别（样本数升序，剩余池大者优先） + order = sorted(remaining, key=lambda t: (sample_counts.get(t, 0), -len(remaining[t]))) + + picked = list(sample) + picked_ids = set(sample_ids) + while len(picked) < 500: + advanced = False + for title in order: + if not remaining[title]: + continue + rec = remaining[title].pop(0) + if rec["jid"] in picked_ids: + continue + picked.append(rec) + picked_ids.add(rec["jid"]) + advanced = True + if len(picked) >= 500: + break + if not advanced: + break + + gold: dict[str, dict] = {} + for r in picked: + gold[r["jid"]] = { + "job_title": r["job_title"], + "responsibilities": [], + "required_skills": matcher.find_names(r["text"]), + "preferred_skills": [], + "scenarios": matcher.find_scenarios(r["text"]), + } + + with (AW_DIR / "jd_eval_500.jsonl").open("w", encoding="utf-8") as f: + for r in picked: + f.write(json.dumps(r, ensure_ascii=False) + "\n") + (AW_DIR / "gold_answers_500.json").write_text( + json.dumps(gold, ensure_ascii=False, indent=1), encoding="utf-8") + cats = sorted(Counter(r["job_title"] for r in picked).items()) + print(f"[T2] 真实 JD 评测集：{len(picked)} 条（含原样本 {len(sample)} 条），" + f"岗位类别 {len(cats)} 类") + return picked, gold + + +# ---------- T3：AIStudio 简历 50 → 200 ---------- +def build_aistudio_extra(matcher: SkillMatcher) -> None: + target_total = 200 # 50（既有）→ 200，幂等：够了就跳过 + rng = random.Random(SEED + 4) + gold_path = RESUME_DIR / "gold.json" + gold = json.loads(gold_path.read_text(encoding="utf-8")) + need = target_total - len(gold) + if need <= 0: + print(f"[T3] AIStudio 简历：已有 {len(gold)} 份，目标 {target_total}，跳过") + return + used = {g["source_id"] for g in gold.values()} + + data = json.loads((SRC / "aistudio" / "train.json").read_text(encoding="utf-8")) + cands = [] + for key, rec in data.items(): + if key in used: + continue + blob = json.dumps([rec.get("工作经历"), rec.get("项目经历")], ensure_ascii=False) + if not TECH_RE.search(blob): + continue + if not _PHONE_RE.match(str(rec.get("电话", ""))): + continue + name = str(rec.get("姓名", "")).strip() + if not re.fullmatch(r"[一-龥]{2,4}", name): + continue + cleaned = dict(rec) + cleaned["姓名"] = name + cands.append((key, cleaned)) + rng.shuffle(cands) + picked = cands[:need] + + start = len(gold) # 编号接续 + for i, (key, rec) in enumerate(picked, start=start + 1): + fname = f"aistudio_{i:03d}.txt" + text = render_aistudio_resume(rec) + (RESUME_DIR / fname).write_text(text, encoding="utf-8") + gold[fname] = { + "name": rec["姓名"].strip(), + "phone": rec["电话"], + "skills": matcher.find_names(text), + "source_id": key, + } + gold = dict(sorted(gold.items())) + gold_path.write_text(json.dumps(gold, ensure_ascii=False, indent=1), encoding="utf-8") + print(f"[T3] AIStudio 简历：候选 {len(cands)} → 新增 {len(picked)} 份" + f"（目录合计 {len(gold)} 份）") + + +# ---------- T4：chinese_resume 批量演示 12 → 32（必须脱敏） ---------- +def build_demo_extra() -> None: + target_total = 32 # 12（既有）→ 32，幂等：够了就跳过 + rng = random.Random(SEED + 3) + with (SRC / "chinese_resume" / "Chinese_resume_data.csv").open(encoding="utf-8") as f: + rows = [r for r in csv.DictReader(f) if r.get("筛选结果") == "通过"] + + existing = sorted(p.name for p in DEMO_DIR.glob("demo_*.txt")) + need = target_total - len(existing) + if need <= 0: + print(f"[T4] chinese_resume 批量演示：已有 {len(existing)} 份，目标 {target_total}，跳过") + return + used_slugs = {re.sub(r"\.txt$", "", name).split("_", 2)[-1] for name in existing} + by_intent: dict[str, list[dict]] = {} + for r in rows: + by_intent.setdefault(r["意向岗位"], []).append(r) + intents = sorted(by_intent, key=lambda k: -len(by_intent[k])) + + picked: list[dict] = [] + # 第一轮：优先覆盖已有文件未覆盖的意向岗位 + for intent in intents: + if len(picked) >= need: + break + slug = re.sub(r"\W+", "", intent) + if slug in used_slugs: + continue + picked.append(rng.choice(by_intent[intent])) + used_slugs.add(slug) + # 第二轮：从其余通过样本补齐（避免覆盖已有文件） + rest = [r for intent in intents for r in by_intent[intent] if r not in picked] + rng.shuffle(rest) + for row in rest: + if len(picked) >= need: + break + picked.append(row) + + start = len(existing) # 既有 12 份，编号接续 + for i, row in enumerate(picked, start=start + 1): + raw = render_chinese_resume(row) + masked = mask_text(raw) # 脱敏后才落盘，原始 PII 不写入 data/ + slug = re.sub(r"\W+", "", row["意向岗位"]) + (DEMO_DIR / f"demo_{i:02d}_{slug}.txt").write_text(masked, encoding="utf-8") + print(f"[T4] chinese_resume 批量演示：新增 {len(picked)} 份" + f"（目录合计 {start + len(picked)} 份，全部已脱敏）") + + +def main() -> None: + if not SRC.exists(): + raise SystemExit(f"源数据目录不存在: {SRC}") + matcher = SkillMatcher(load_lexicon()) + full = build_autoweaks_full(matcher) + build_eval_500(matcher, full) + build_aistudio_extra(matcher) + build_demo_extra() + print(f"数据扩产完成，产物写入 {DST}") + + +if __name__ == "__main__": + main() diff --git a/tests/conftest.py b/tests/conftest.py index 94f2f3b..0bf7f7c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -40,9 +40,16 @@ def isolated_state_db(tmp_path_factory): routes._cfg["state_db_path"] = str(tmp / "state.db") routes._cfg["audit_path"] = str(tmp / "audit.jsonl") routes._cfg["pending_path"] = str(tmp / "pending.json") + # 测试不加载本机附加数据源：真实语料会让 API 触发的重建超出任务等待窗口 + extra_orig = routes._cfg.get("jd_extra_dirs") + routes._cfg["jd_extra_dirs"] = "" yield for k, v in originals.items(): if v is None: routes._cfg.pop(k, None) else: routes._cfg[k] = v + if extra_orig is None: + routes._cfg.pop("jd_extra_dirs", None) + else: + routes._cfg["jd_extra_dirs"] = extra_orig diff --git a/tests/test_persistence.py b/tests/test_persistence.py index dc7753c..a17ed95 100644 --- a/tests/test_persistence.py +++ b/tests/test_persistence.py @@ -61,6 +61,7 @@ def _tmp_cfg(tmp_path, records) -> dict: "\n".join(json.dumps(r, ensure_ascii=False) for r in records) + "\n", encoding="utf-8") cfg = load_config() cfg["jd_raw_dir"] = str(jd_dir) + cfg["jd_extra_dirs"] = "" # 测试不加载本机附加数据源（真实语料会使 pipeline 变慢且引入不确定性） cfg["graph_path"] = str(tmp_path / "graph.json") cfg["state_db_path"] = str(tmp_path / "state.db") cfg["audit_path"] = str(tmp_path / "audit.jsonl") diff --git a/tests/test_real_data.py b/tests/test_real_data.py index 7a6602f..760bcf5 100644 --- a/tests/test_real_data.py +++ b/tests/test_real_data.py @@ -8,6 +8,10 @@ import sys from pathlib import Path import pytest +from pydantic import ValidationError + +from app.ingest.loader import load_jd_file +from app.models import JDRecord ROOT = Path(__file__).resolve().parents[1] sys.path.insert(0, str(ROOT)) @@ -34,9 +38,12 @@ def test_real_tree_complete() -> None: assert (REAL / "README.md").exists() assert (JD_DIR / "jd_sample.jsonl").exists() assert (JD_DIR / "gold_answers.json").exists() - assert len(list(RESUME_DIR.glob("*.txt"))) == 50 + assert (JD_DIR / "jd_full.jsonl").exists() + assert (JD_DIR / "jd_eval_500.jsonl").exists() + assert (JD_DIR / "gold_answers_500.json").exists() + assert len(list(RESUME_DIR.glob("*.txt"))) == 200 assert (RESUME_DIR / "gold.json").exists() - assert len(list(DEMO_DIR.glob("*.txt"))) == 12 + assert len(list(DEMO_DIR.glob("*.txt"))) == 32 for name in ("labels_isco_codes.txt", "labels_isco_labels.txt", "labels_esco_code.txt", "labels_esco_occupation.txt", "README.md"): assert (ESCO_DIR / name).exists(), name @@ -63,18 +70,19 @@ def test_jd_gold_schema() -> None: def test_resume_gold_schema() -> None: gold = json.loads((RESUME_DIR / "gold.json").read_text(encoding="utf-8")) - assert len(gold) == 50 + assert len(gold) == 200 for fname, g in gold.items(): assert (RESUME_DIR / fname).exists(), fname assert re.fullmatch(r"[一-龥]{2,4}", g["name"]), fname assert re.fullmatch(r"1[3-9]\d{9}", g["phone"]), fname assert isinstance(g["skills"], list) and all(isinstance(s, str) for s in g["skills"]) + assert re.fullmatch(r"[0-9a-f]{12}", g["source_id"]), fname def test_batch_demo_masked() -> None: """批量演示素材必须已脱敏：无 11 位手机号、邮箱仅呈 x***@domain 形态、姓名带 *。""" files = list(DEMO_DIR.glob("*.txt")) - assert len(files) == 12 + assert len(files) == 32 phone_re = re.compile(r"1[3-9]\d{9}") email_re = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}") name_re = re.compile(r"姓名[:：]\s*([一-龥]{2,4})") # 未脱敏姓名（脱敏后含 *） @@ -87,16 +95,63 @@ def test_batch_demo_masked() -> None: assert re.search(r"姓名[:：]\s*[一-龥]\*", text), f.name -@pytest.mark.parametrize("dim", ["real_jd", "real_resume"]) +def test_jd_full_schema_and_contains_sample() -> None: + """jd_full.jsonl：每行过 JDRecord 校验、全文去重、包含原 200 条样本（jid/原文一致）。""" + records, errors = load_jd_file(JD_DIR / "jd_full.jsonl") + assert errors == [] + assert len(records) >= 200 + sample = [json.loads(ln) for ln in + (JD_DIR / "jd_sample.jsonl").read_text(encoding="utf-8").splitlines() if ln.strip()] + by_jid = {r.jid: r for r in records} + for s in sample: + assert s["jid"] in by_jid, s["jid"] + assert by_jid[s["jid"]].text == s["text"], s["jid"] + texts = {r.text for r in records} + assert len(texts) == len(records) # 全文哈希去重 + for r in records: + assert r.source == "autoweaks" + assert re.fullmatch(r"2019-\d{2}-\d{2}", r.publish_date.isoformat()), r.jid + assert len(r.text) >= 60 + + +def test_jd_eval_500_schema_and_superset() -> None: + """jd_eval_500.jsonl：500 条、含原 200 条样本、gold Schema 与正式 gold 一致。""" + records, errors = load_jd_file(JD_DIR / "jd_eval_500.jsonl") + assert errors == [] and len(records) == 500 + full = load_jd_file(JD_DIR / "jd_full.jsonl")[0] + full_ids = {r.jid for r in full} + sample = [json.loads(ln) for ln in + (JD_DIR / "jd_sample.jsonl").read_text(encoding="utf-8").splitlines() if ln.strip()] + sample_ids = {s["jid"] for s in sample} + eval_ids = {r.jid for r in records} + assert sample_ids <= eval_ids <= full_ids + gold = json.loads((JD_DIR / "gold_answers_500.json").read_text(encoding="utf-8")) + assert set(gold) == eval_ids + for jid, g in gold.items(): + assert set(g) == set(GOLD_JD_FIELDS), jid + assert isinstance(g["job_title"], str) and g["job_title"] + for f in ("responsibilities", "required_skills", "preferred_skills", "scenarios"): + assert isinstance(g[f], list) and all(isinstance(s, str) for s in g[f]), (jid, f) + + +@pytest.mark.parametrize("dim", ["real_jd", "real_jd_500", "real_resume"]) def test_eval_real_dimensions(dim: str) -> None: """真实维度评测可运行且指标在合法区间（全量小数据集，直接实跑）。""" evaluate = _load_evaluate() - result = evaluate.eval_real_jd() if dim == "real_jd" else evaluate.eval_real_resume() + if dim == "real_jd": + result = evaluate.eval_real_jd() + elif dim == "real_jd_500": + result = evaluate.eval_real_jd_500() + else: + result = evaluate.eval_real_resume() assert 0.0 <= result["accuracy"] <= 1.0 if dim == "real_jd": assert result["samples"] == 200 and result["bad_lines"] == 0 assert result["fields"]["required_skills"]["f1"] >= 0.80 # 任务目标 + elif dim == "real_jd_500": + assert result["samples"] == 500 and result["bad_lines"] == 0 + assert result["fields"]["required_skills"]["f1"] >= 0.80 # 任务目标 else: - assert result["samples"] == 50 + assert result["samples"] == 200 assert result["name_accuracy"] == 1.0 # 任务目标：脱敏后姓氏+* assert result["phone_accuracy"] == 1.0