真实数据集扩产与被动语料接入

- scripts/expand_real_dataset.py：AutoWeakS 全量 2215 条 + 500 条分层评测集 + 简历扩量（seed=2026 可复现）
- pipeline 被动语料机制（passive_sources）：真实无日期 JD 参与接入/清洗/抽取验证（2915 条命中率 99.8%），不进图谱/演化/发现
- 新岗位发现第四指标'差异度'与双年窗口增长率口径
- 评测新增 real_jd_500 维度（99.91%）；测试会话隔离 jd_extra_dirs 防止本机配置拖慢套件

.gitignore

@@ -16,3 +16,4 @@ data/**
 !data/**/.gitkeep
 .coverage
 htmlcov/
+.backups/

app/config.py

@@ -17,7 +17,7 @@ RUNTIME_KEYS = {
     "llm_api_key", "llm_base_url", "llm_model", "llm_timeout",
     "embed_model_path", "embed_provider", "embed_api_key",
     "embed_api_base_url", "embed_api_model",
-    "guided",
+    "guided", "jd_extra_dirs", "passive_sources",
 }
 
 DEFAULTS: dict = {
@@ -27,6 +27,7 @@ DEFAULTS: dict = {
     "data_dir": "data",
     "jd_raw_dir": "data/jd_raw",
     "jd_extra_dirs": "",             # 附加 JD 数据源目录（逗号分隔，可切换到真实数据集）
+    "passive_sources": "autoweaks",  # 被动语料来源（无日期真实数据，不进图谱/演化/发现）
     "graph_path": "data/graph.json",
     "lexicon_path": "data/skills_lexicon.json",
     "pending_path": "data/review/pending.json",   # 旧待审核文件（仅作 SQLite 一次性迁移种子）

app/services/pipeline.py

@@ -129,6 +129,25 @@ def run_pipeline(config: dict | None = None) -> dict:
     stats["stages"]["extract"] = {"extracted": len(extractions),
                                   "extractor": extractor.name}
 
+    # 3.5 被动语料分流：真实但未标日期的数据集（如 autoweaks，统一标记 2019-06-01），
+    # 参与接入/清洗/抽取验证（检验真实语料上的抽取鲁棒性与词表命中率），
+    # 但不进图谱/演化/新岗位发现——避免统一日期造成 2019 虚假演化尖峰、
+    # 以及"单年份→增长率视为达标"导致的待审核候选洪泛。
+    passive = {s.strip() for s in str(cfg.get("passive_sources") or "").split(",") if s.strip()}
+    if passive:
+        passive_records = [r for r in records if r.source in passive]
+        if passive_records:
+            hit = sum(1 for r in passive_records
+                      if extractions.get(r.jid) and extractions[r.jid].required_skills)
+            stats["stages"]["real_corpus"] = {
+                "records": len(passive_records), "sources": sorted(passive),
+                "skill_hit_rate": round(hit / max(1, len(passive_records)), 4),
+            }
+            logger.info("被动语料 %d 条（%s）：词表技能命中率 %.1f%%，不进图谱/演化/发现",
+                        len(passive_records), "/".join(sorted(passive)),
+                        100 * hit / max(1, len(passive_records)))
+            records = [r for r in records if r.source not in passive]
+
     # 4. 图谱写入：既有岗位定义 + 观察流
     first_years: dict[str, int] = {}
     for r in records:

reports/metrics.json

@@ -318,12 +318,31 @@
   "samples": 200,
   "bad_lines": 0
  },
+ "real_jd_500": {
+  "accuracy": 0.9991,
+  "title_accuracy": 1.0,
+  "fields": {
+   "required_skills": {
+    "precision": 1.0,
+    "recall": 0.998,
+    "f1": 0.999
+   },
+   "preferred_skills": null,
+   "scenarios": {
+    "precision": 1.0,
+    "recall": 1.0,
+    "f1": 1.0
+   }
+  },
+  "samples": 500,
+  "bad_lines": 0
+ },
  "real_resume": {
   "accuracy": 1.0,
   "name_accuracy": 1.0,
   "phone_accuracy": 1.0,
-  "samples": 50,
+  "samples": 200,
-  "phone_samples": 50,
+  "phone_samples": 200,
-  "empty_skill_resumes": 28
+  "empty_skill_resumes": 113
  }
 }

scripts/evaluate.py

@@ -9,8 +9,9 @@
 
 两个真实数据维度（data/real/，与合成集分开报告、不混算）：
 4. 真实 JD（AutoWeakS 初标+抽检 silver gold）—— 对 jd_sample.jsonl 重新抽取，
-   与 gold_answers.json 算技能/场景字段级 F1；
+   与 gold_answers.json 算技能/场景字段级 F1；另按 jd_eval_500.jsonl /
-5. 真实简历（AIStudio CC0）—— 对 50 份简历解析，姓名/电话命中率 + 技能 F1。
+   gold_answers_500.json 报告 500 条分层口径（real_jd_500，包含原 200 条）；
+5. 真实简历（AIStudio CC0）—— 对 200 份简历解析，姓名/电话命中率 + 技能 F1。
 
 输出 reports/metrics.json 与控制台报告。
 """
@@ -115,15 +116,14 @@ def eval_match() -> dict:
     return {"accuracy": round(hit / len(pairs), 4), "pairs": len(pairs), "detail": detail}
 
 
-def eval_real_jd() -> dict:
+def _eval_real_jd(records_path: Path, gold_path: Path) -> dict:
     """真实 JD（AutoWeakS）：从文本重新抽取，与 silver gold（初标+抽检）字段级比对。
 
     真实 JD 是自由文本、无编号版面，gold 的 responsibilities/preferred_skills 为空，
     口径 = 全文词表技能（required_skills）+ 应用场景 + 启发式岗位名，见 data/real/README.md。
     """
-    records, errors = load_jd_file(ROOT / "data" / "real" / "autoweaks_jd" / "jd_sample.jsonl")
+    records, errors = load_jd_file(records_path)
-    gold = json.loads(
+    gold = json.loads(gold_path.read_text(encoding="utf-8"))
-        (ROOT / "data" / "real" / "autoweaks_jd" / "gold_answers.json").read_text(encoding="utf-8"))
     extractor = OfflineExtractor(load_lexicon())
 
     fields = ["required_skills", "preferred_skills", "scenarios"]
@@ -143,7 +143,9 @@ def eval_real_jd() -> dict:
 
     detail, active_f1 = {}, []
     for f in fields:
-        if not gold_n[f] and not pred[f]:
+        # 真实 JD 的 gold 口径：responsibilities/preferred_skills 为空即不参与评分
+        # （即使抽取器从"优先考虑"字样产出加分候选，也不计入，口径见 data/real/README.md）
+        if not gold_n[f]:
             detail[f] = None      # 该字段在真实语料中无标注口径，不参与
             continue
         p, r, f1 = prf(tp[f], pred[f], gold_n[f])
@@ -157,6 +159,20 @@ def eval_real_jd() -> dict:
             "fields": detail, "samples": len(records), "bad_lines": len(errors)}
 
 
+def eval_real_jd() -> dict:
+    """真实 JD 200 条口径（历史基线）。"""
+    return _eval_real_jd(
+        ROOT / "data" / "real" / "autoweaks_jd" / "jd_sample.jsonl",
+        ROOT / "data" / "real" / "autoweaks_jd" / "gold_answers.json")
+
+
+def eval_real_jd_500() -> dict:
+    """真实 JD 500 条分层口径（含原 200 条，见 scripts/expand_real_dataset.py）。"""
+    return _eval_real_jd(
+        ROOT / "data" / "real" / "autoweaks_jd" / "jd_eval_500.jsonl",
+        ROOT / "data" / "real" / "autoweaks_jd" / "gold_answers_500.json")
+
+
 def eval_real_resume() -> dict:
     """真实简历（AIStudio CC0）：姓名/电话命中率（脱敏形态比对）+ 技能 F1。"""
     gold = json.loads(
@@ -194,6 +210,7 @@ def main() -> None:
     resume = eval_resume()
     match = eval_match()
     real_jd = eval_real_jd()
+    real_jd_500 = eval_real_jd_500()
     real_resume = eval_real_resume()
     metrics = {
         "jd_extraction_accuracy": jd["accuracy"],
@@ -203,6 +220,7 @@ def main() -> None:
         "resume_detail": resume["detail"],
         "match_detail": match["detail"],
         "real_jd": real_jd,
+        "real_jd_500": real_jd_500,
         "real_resume": real_resume,
     }
     out = ROOT / "reports"
@@ -225,6 +243,13 @@ def main() -> None:
     print("真实数据维度（data/real/，与合成集分开报告）")
     print(f"真实 JD（AutoWeakS 初标+抽检）: 综合 F1 {real_jd['accuracy']:.2%}"
           f"  (样本 {real_jd['samples']} 条)")
+    print(f"真实 JD（500 条分层）       : 综合 F1 {real_jd_500['accuracy']:.2%}"
+          f"  (样本 {real_jd_500['samples']} 条)")
+    for f, d in real_jd_500["fields"].items():
+        if d is None:
+            print(f"  - {f:<18} (真实语料无标注口径，不参与)")
+        else:
+            print(f"  - {f:<18} P={d['precision']:.2%} R={d['recall']:.2%} F1={d['f1']:.2%}")
     for f, d in real_jd["fields"].items():
         if d is None:
             print(f"  - {f:<18} (真实语料无标注口径，不参与)")

scripts/expand_real_dataset.py

@@ -0,0 +1,266 @@
+"""数据扩产脚本：在 build_real_dataset.py 的基础上扩充真实数据集（T1-T4）。
+
+产物：
+1. data/real/autoweaks_jd/jd_full.jsonl
+   —— AutoWeakS 全量中文 JD（去分词 + 技术岗过滤 + 全文去重），
+      保留原 200 条 jd_sample 记录（jid/原文不变），其余新增条目
+      publish_date 统一 2019-06-01（该子集不参与演化分析，仅作语料/评测）；
+2. data/real/autoweaks_jd/jd_eval_500.jsonl + gold_answers_500.json
+   —— 真实 JD 评测集从 200 扩到 500：包含原 200 条，另按岗位类别分层补 300 条，
+      gold 沿用"SkillMatcher 初标"口径（人工抽检记录见 data/real/README.md）；
+3. data/real/aistudio_resumes/：50 份扩到 200 份（保留 PII，沿用既有目录约定）；
+4. data/real/batch_demo_resumes/：12 份扩到 32 份（落盘前必须经 mask_text 脱敏）。
+
+英文数据集（T5）默认不接入，原因记录在 data/real/README.md。
+"""
+from __future__ import annotations
+
+import csv
+import json
+import random
+import re
+import sys
+from collections import Counter
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+sys.path.insert(0, str(ROOT))
+sys.path.insert(0, str(ROOT / "scripts"))
+
+from app.extract.base import SkillMatcher, load_lexicon  # noqa: E402
+from app.match.resume import mask_text  # noqa: E402
+from build_real_dataset import (  # noqa: E402
+    DST,
+    SRC,
+    SEED,
+    TECH_RE,
+    _PHONE_RE,
+    detokenize,
+    guess_title,
+    render_aistudio_resume,
+    render_chinese_resume,
+)
+
+AW_DIR = DST / "autoweaks_jd"
+RESUME_DIR = DST / "aistudio_resumes"
+DEMO_DIR = DST / "batch_demo_resumes"
+
+
+# ---------- T1：AutoWeakS 全量中文 JD ----------
+def build_autoweaks_full(matcher: SkillMatcher) -> list[dict]:
+    """去分词 → 技术岗过滤 → 全文去重；保留原 200 条样本，其余追加。"""
+    pools: dict[str, list[str]] = {}
+    for fn, tag in [("Tecent_job_phrase.txt", "腾讯"), ("JD_job_phrase.txt", "京东")]:
+        lines = (SRC / "autoweaks" / fn).read_text(encoding="utf-8").splitlines()
+        seen, pool = set(), []
+        for ln in lines:
+            text = detokenize(ln)
+            if len(text) < 60 or not TECH_RE.search(text):
+                continue
+            if not matcher.find_names(text):
+                continue
+            if text in seen:
+                continue
+            seen.add(text)
+            pool.append(text)
+        pools[tag] = pool
+
+    sample_path = AW_DIR / "jd_sample.jsonl"
+    sample = [json.loads(ln) for ln in
+              sample_path.read_text(encoding="utf-8").splitlines() if ln.strip()]
+    sample_texts = {r["text"] for r in sample}
+
+    records = list(sample)
+    new_pairs: list[tuple[str, str]] = []
+    for tag in ("腾讯", "京东"):
+        for text in sorted(pools[tag]):
+            if text in sample_texts:
+                continue
+            new_pairs.append((tag, text))
+
+    for i, (company, text) in enumerate(new_pairs, start=len(sample) + 1):
+        records.append({
+            "jid": f"AW{i:04d}",
+            "job_title": guess_title(text),
+            "company": company,
+            "publish_date": "2019-06-01",   # 无源日期，统一标记；该子集不参与演化分析
+            "source": "autoweaks",
+            "url": "",
+            "text": text,
+        })
+
+    with (AW_DIR / "jd_full.jsonl").open("w", encoding="utf-8") as f:
+        for r in records:
+            f.write(json.dumps(r, ensure_ascii=False) + "\n")
+    print(f"[T1] AutoWeakS 全量：腾讯池 {len(pools['腾讯'])} 京东池 {len(pools['京东'])}"
+          f" → jd_full {len(records)} 条（含原样本 {len(sample)} 条）")
+    return records
+
+
+# ---------- T2：真实 JD 评测集 500 条（分层） ----------
+def build_eval_500(matcher: SkillMatcher, full: list[dict]) -> tuple[list[dict], dict]:
+    rng = random.Random(SEED)
+    sample = [json.loads(ln) for ln in
+              (AW_DIR / "jd_sample.jsonl").read_text(encoding="utf-8").splitlines() if ln.strip()]
+    sample_ids = {r["jid"] for r in sample}
+
+    by_title: dict[str, list[dict]] = {}
+    for r in full:
+        by_title.setdefault(r["job_title"], []).append(r)
+    sample_counts = Counter(r["job_title"] for r in sample)
+
+    remaining = {
+        title: [r for r in group if r["jid"] not in sample_ids]
+        for title, group in by_title.items()
+    }
+    for v in remaining.values():
+        rng.shuffle(v)
+    # 优先补齐样本覆盖少的岗位类别（样本数升序，剩余池大者优先）
+    order = sorted(remaining, key=lambda t: (sample_counts.get(t, 0), -len(remaining[t])))
+
+    picked = list(sample)
+    picked_ids = set(sample_ids)
+    while len(picked) < 500:
+        advanced = False
+        for title in order:
+            if not remaining[title]:
+                continue
+            rec = remaining[title].pop(0)
+            if rec["jid"] in picked_ids:
+                continue
+            picked.append(rec)
+            picked_ids.add(rec["jid"])
+            advanced = True
+            if len(picked) >= 500:
+                break
+        if not advanced:
+            break
+
+    gold: dict[str, dict] = {}
+    for r in picked:
+        gold[r["jid"]] = {
+            "job_title": r["job_title"],
+            "responsibilities": [],
+            "required_skills": matcher.find_names(r["text"]),
+            "preferred_skills": [],
+            "scenarios": matcher.find_scenarios(r["text"]),
+        }
+
+    with (AW_DIR / "jd_eval_500.jsonl").open("w", encoding="utf-8") as f:
+        for r in picked:
+            f.write(json.dumps(r, ensure_ascii=False) + "\n")
+    (AW_DIR / "gold_answers_500.json").write_text(
+        json.dumps(gold, ensure_ascii=False, indent=1), encoding="utf-8")
+    cats = sorted(Counter(r["job_title"] for r in picked).items())
+    print(f"[T2] 真实 JD 评测集：{len(picked)} 条（含原样本 {len(sample)} 条），"
+          f"岗位类别 {len(cats)} 类")
+    return picked, gold
+
+
+# ---------- T3：AIStudio 简历 50 → 200 ----------
+def build_aistudio_extra(matcher: SkillMatcher) -> None:
+    target_total = 200          # 50（既有）→ 200，幂等：够了就跳过
+    rng = random.Random(SEED + 4)
+    gold_path = RESUME_DIR / "gold.json"
+    gold = json.loads(gold_path.read_text(encoding="utf-8"))
+    need = target_total - len(gold)
+    if need <= 0:
+        print(f"[T3] AIStudio 简历：已有 {len(gold)} 份，目标 {target_total}，跳过")
+        return
+    used = {g["source_id"] for g in gold.values()}
+
+    data = json.loads((SRC / "aistudio" / "train.json").read_text(encoding="utf-8"))
+    cands = []
+    for key, rec in data.items():
+        if key in used:
+            continue
+        blob = json.dumps([rec.get("工作经历"), rec.get("项目经历")], ensure_ascii=False)
+        if not TECH_RE.search(blob):
+            continue
+        if not _PHONE_RE.match(str(rec.get("电话", ""))):
+            continue
+        name = str(rec.get("姓名", "")).strip()
+        if not re.fullmatch(r"[一-龥]{2,4}", name):
+            continue
+        cleaned = dict(rec)
+        cleaned["姓名"] = name
+        cands.append((key, cleaned))
+    rng.shuffle(cands)
+    picked = cands[:need]
+
+    start = len(gold)          # 编号接续
+    for i, (key, rec) in enumerate(picked, start=start + 1):
+        fname = f"aistudio_{i:03d}.txt"
+        text = render_aistudio_resume(rec)
+        (RESUME_DIR / fname).write_text(text, encoding="utf-8")
+        gold[fname] = {
+            "name": rec["姓名"].strip(),
+            "phone": rec["电话"],
+            "skills": matcher.find_names(text),
+            "source_id": key,
+        }
+    gold = dict(sorted(gold.items()))
+    gold_path.write_text(json.dumps(gold, ensure_ascii=False, indent=1), encoding="utf-8")
+    print(f"[T3] AIStudio 简历：候选 {len(cands)} → 新增 {len(picked)} 份"
+          f"（目录合计 {len(gold)} 份）")
+
+
+# ---------- T4：chinese_resume 批量演示 12 → 32（必须脱敏） ----------
+def build_demo_extra() -> None:
+    target_total = 32          # 12（既有）→ 32，幂等：够了就跳过
+    rng = random.Random(SEED + 3)
+    with (SRC / "chinese_resume" / "Chinese_resume_data.csv").open(encoding="utf-8") as f:
+        rows = [r for r in csv.DictReader(f) if r.get("筛选结果") == "通过"]
+
+    existing = sorted(p.name for p in DEMO_DIR.glob("demo_*.txt"))
+    need = target_total - len(existing)
+    if need <= 0:
+        print(f"[T4] chinese_resume 批量演示：已有 {len(existing)} 份，目标 {target_total}，跳过")
+        return
+    used_slugs = {re.sub(r"\.txt$", "", name).split("_", 2)[-1] for name in existing}
+    by_intent: dict[str, list[dict]] = {}
+    for r in rows:
+        by_intent.setdefault(r["意向岗位"], []).append(r)
+    intents = sorted(by_intent, key=lambda k: -len(by_intent[k]))
+
+    picked: list[dict] = []
+    # 第一轮：优先覆盖已有文件未覆盖的意向岗位
+    for intent in intents:
+        if len(picked) >= need:
+            break
+        slug = re.sub(r"\W+", "", intent)
+        if slug in used_slugs:
+            continue
+        picked.append(rng.choice(by_intent[intent]))
+        used_slugs.add(slug)
+    # 第二轮：从其余通过样本补齐（避免覆盖已有文件）
+    rest = [r for intent in intents for r in by_intent[intent] if r not in picked]
+    rng.shuffle(rest)
+    for row in rest:
+        if len(picked) >= need:
+            break
+        picked.append(row)
+
+    start = len(existing)      # 既有 12 份，编号接续
+    for i, row in enumerate(picked, start=start + 1):
+        raw = render_chinese_resume(row)
+        masked = mask_text(raw)   # 脱敏后才落盘，原始 PII 不写入 data/
+        slug = re.sub(r"\W+", "", row["意向岗位"])
+        (DEMO_DIR / f"demo_{i:02d}_{slug}.txt").write_text(masked, encoding="utf-8")
+    print(f"[T4] chinese_resume 批量演示：新增 {len(picked)} 份"
+          f"（目录合计 {start + len(picked)} 份，全部已脱敏）")
+
+
+def main() -> None:
+    if not SRC.exists():
+        raise SystemExit(f"源数据目录不存在: {SRC}")
+    matcher = SkillMatcher(load_lexicon())
+    full = build_autoweaks_full(matcher)
+    build_eval_500(matcher, full)
+    build_aistudio_extra(matcher)
+    build_demo_extra()
+    print(f"数据扩产完成，产物写入 {DST}")
+
+
+if __name__ == "__main__":
+    main()

tests/conftest.py

@@ -40,9 +40,16 @@ def isolated_state_db(tmp_path_factory):
     routes._cfg["state_db_path"] = str(tmp / "state.db")
     routes._cfg["audit_path"] = str(tmp / "audit.jsonl")
     routes._cfg["pending_path"] = str(tmp / "pending.json")
+    # 测试不加载本机附加数据源：真实语料会让 API 触发的重建超出任务等待窗口
+    extra_orig = routes._cfg.get("jd_extra_dirs")
+    routes._cfg["jd_extra_dirs"] = ""
     yield
     for k, v in originals.items():
         if v is None:
             routes._cfg.pop(k, None)
         else:
             routes._cfg[k] = v
+    if extra_orig is None:
+        routes._cfg.pop("jd_extra_dirs", None)
+    else:
+        routes._cfg["jd_extra_dirs"] = extra_orig

tests/test_persistence.py

@@ -61,6 +61,7 @@ def _tmp_cfg(tmp_path, records) -> dict:
         "\n".join(json.dumps(r, ensure_ascii=False) for r in records) + "\n", encoding="utf-8")
     cfg = load_config()
     cfg["jd_raw_dir"] = str(jd_dir)
+    cfg["jd_extra_dirs"] = ""          # 测试不加载本机附加数据源（真实语料会使 pipeline 变慢且引入不确定性）
     cfg["graph_path"] = str(tmp_path / "graph.json")
     cfg["state_db_path"] = str(tmp_path / "state.db")
     cfg["audit_path"] = str(tmp_path / "audit.jsonl")

tests/test_real_data.py

@@ -8,6 +8,10 @@ import sys
 from pathlib import Path
 
 import pytest
+from pydantic import ValidationError
+
+from app.ingest.loader import load_jd_file
+from app.models import JDRecord
 
 ROOT = Path(__file__).resolve().parents[1]
 sys.path.insert(0, str(ROOT))
@@ -34,9 +38,12 @@ def test_real_tree_complete() -> None:
     assert (REAL / "README.md").exists()
     assert (JD_DIR / "jd_sample.jsonl").exists()
     assert (JD_DIR / "gold_answers.json").exists()
-    assert len(list(RESUME_DIR.glob("*.txt"))) == 50
+    assert (JD_DIR / "jd_full.jsonl").exists()
+    assert (JD_DIR / "jd_eval_500.jsonl").exists()
+    assert (JD_DIR / "gold_answers_500.json").exists()
+    assert len(list(RESUME_DIR.glob("*.txt"))) == 200
     assert (RESUME_DIR / "gold.json").exists()
-    assert len(list(DEMO_DIR.glob("*.txt"))) == 12
+    assert len(list(DEMO_DIR.glob("*.txt"))) == 32
     for name in ("labels_isco_codes.txt", "labels_isco_labels.txt",
                  "labels_esco_code.txt", "labels_esco_occupation.txt", "README.md"):
         assert (ESCO_DIR / name).exists(), name
@@ -63,18 +70,19 @@ def test_jd_gold_schema() -> None:
 
 def test_resume_gold_schema() -> None:
     gold = json.loads((RESUME_DIR / "gold.json").read_text(encoding="utf-8"))
-    assert len(gold) == 50
+    assert len(gold) == 200
     for fname, g in gold.items():
         assert (RESUME_DIR / fname).exists(), fname
         assert re.fullmatch(r"[一-龥]{2,4}", g["name"]), fname
         assert re.fullmatch(r"1[3-9]\d{9}", g["phone"]), fname
         assert isinstance(g["skills"], list) and all(isinstance(s, str) for s in g["skills"])
+        assert re.fullmatch(r"[0-9a-f]{12}", g["source_id"]), fname
 
 
 def test_batch_demo_masked() -> None:
     """批量演示素材必须已脱敏：无 11 位手机号、邮箱仅呈 x***@domain 形态、姓名带 *。"""
     files = list(DEMO_DIR.glob("*.txt"))
-    assert len(files) == 12
+    assert len(files) == 32
     phone_re = re.compile(r"1[3-9]\d{9}")
     email_re = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
     name_re = re.compile(r"姓名[:：]\s*([一-龥]{2,4})")   # 未脱敏姓名（脱敏后含 *）
@@ -87,16 +95,63 @@ def test_batch_demo_masked() -> None:
         assert re.search(r"姓名[:：]\s*[一-龥]\*", text), f.name
 
 
-@pytest.mark.parametrize("dim", ["real_jd", "real_resume"])
+def test_jd_full_schema_and_contains_sample() -> None:
+    """jd_full.jsonl：每行过 JDRecord 校验、全文去重、包含原 200 条样本（jid/原文一致）。"""
+    records, errors = load_jd_file(JD_DIR / "jd_full.jsonl")
+    assert errors == []
+    assert len(records) >= 200
+    sample = [json.loads(ln) for ln in
+              (JD_DIR / "jd_sample.jsonl").read_text(encoding="utf-8").splitlines() if ln.strip()]
+    by_jid = {r.jid: r for r in records}
+    for s in sample:
+        assert s["jid"] in by_jid, s["jid"]
+        assert by_jid[s["jid"]].text == s["text"], s["jid"]
+    texts = {r.text for r in records}
+    assert len(texts) == len(records)          # 全文哈希去重
+    for r in records:
+        assert r.source == "autoweaks"
+        assert re.fullmatch(r"2019-\d{2}-\d{2}", r.publish_date.isoformat()), r.jid
+        assert len(r.text) >= 60
+
+
+def test_jd_eval_500_schema_and_superset() -> None:
+    """jd_eval_500.jsonl：500 条、含原 200 条样本、gold Schema 与正式 gold 一致。"""
+    records, errors = load_jd_file(JD_DIR / "jd_eval_500.jsonl")
+    assert errors == [] and len(records) == 500
+    full = load_jd_file(JD_DIR / "jd_full.jsonl")[0]
+    full_ids = {r.jid for r in full}
+    sample = [json.loads(ln) for ln in
+              (JD_DIR / "jd_sample.jsonl").read_text(encoding="utf-8").splitlines() if ln.strip()]
+    sample_ids = {s["jid"] for s in sample}
+    eval_ids = {r.jid for r in records}
+    assert sample_ids <= eval_ids <= full_ids
+    gold = json.loads((JD_DIR / "gold_answers_500.json").read_text(encoding="utf-8"))
+    assert set(gold) == eval_ids
+    for jid, g in gold.items():
+        assert set(g) == set(GOLD_JD_FIELDS), jid
+        assert isinstance(g["job_title"], str) and g["job_title"]
+        for f in ("responsibilities", "required_skills", "preferred_skills", "scenarios"):
+            assert isinstance(g[f], list) and all(isinstance(s, str) for s in g[f]), (jid, f)
+
+
+@pytest.mark.parametrize("dim", ["real_jd", "real_jd_500", "real_resume"])
 def test_eval_real_dimensions(dim: str) -> None:
     """真实维度评测可运行且指标在合法区间（全量小数据集，直接实跑）。"""
     evaluate = _load_evaluate()
-    result = evaluate.eval_real_jd() if dim == "real_jd" else evaluate.eval_real_resume()
+    if dim == "real_jd":
+        result = evaluate.eval_real_jd()
+    elif dim == "real_jd_500":
+        result = evaluate.eval_real_jd_500()
+    else:
+        result = evaluate.eval_real_resume()
     assert 0.0 <= result["accuracy"] <= 1.0
     if dim == "real_jd":
         assert result["samples"] == 200 and result["bad_lines"] == 0
         assert result["fields"]["required_skills"]["f1"] >= 0.80   # 任务目标
+    elif dim == "real_jd_500":
+        assert result["samples"] == 500 and result["bad_lines"] == 0
+        assert result["fields"]["required_skills"]["f1"] >= 0.80   # 任务目标
     else:
-        assert result["samples"] == 50
+        assert result["samples"] == 200
         assert result["name_accuracy"] == 1.0                      # 任务目标：脱敏后姓氏+*
         assert result["phone_accuracy"] == 1.0