真实数据集扩产与被动语料接入
- scripts/expand_real_dataset.py：AutoWeakS 全量 2215 条 + 500 条分层评测集 + 简历扩量（seed=2026 可复现） - pipeline 被动语料机制（passive_sources）：真实无日期 JD 参与接入/清洗/抽取验证（2915 条命中率 99.8%），不进图谱/演化/发现 - 新岗位发现第四指标'差异度'与双年窗口增长率口径 - 评测新增 real_jd_500 维度（99.91%）；测试会话隔离 jd_extra_dirs 防止本机配置拖慢套件
This commit is contained in:
RoyirinRosmontis
@@ -16,3 +16,4 @@ data/**
|
||||
!data/**/.gitkeep
|
||||
.coverage
|
||||
htmlcov/
|
||||
.backups/
|
||||
|
||||
+2
-1
@@ -17,7 +17,7 @@ RUNTIME_KEYS = {
|
||||
"llm_api_key", "llm_base_url", "llm_model", "llm_timeout",
|
||||
"embed_model_path", "embed_provider", "embed_api_key",
|
||||
"embed_api_base_url", "embed_api_model",
|
||||
"guided",
|
||||
"guided", "jd_extra_dirs", "passive_sources",
|
||||
}
|
||||
|
||||
DEFAULTS: dict = {
|
||||
@@ -27,6 +27,7 @@ DEFAULTS: dict = {
|
||||
"data_dir": "data",
|
||||
"jd_raw_dir": "data/jd_raw",
|
||||
"jd_extra_dirs": "", # 附加 JD 数据源目录（逗号分隔，可切换到真实数据集）
|
||||
"passive_sources": "autoweaks", # 被动语料来源（无日期真实数据，不进图谱/演化/发现）
|
||||
"graph_path": "data/graph.json",
|
||||
"lexicon_path": "data/skills_lexicon.json",
|
||||
"pending_path": "data/review/pending.json", # 旧待审核文件（仅作 SQLite 一次性迁移种子）
|
||||
|
||||
@@ -129,6 +129,25 @@ def run_pipeline(config: dict | None = None) -> dict:
|
||||
stats["stages"]["extract"] = {"extracted": len(extractions),
|
||||
"extractor": extractor.name}
|
||||
|
||||
# 3.5 被动语料分流：真实但未标日期的数据集（如 autoweaks，统一标记 2019-06-01），
|
||||
# 参与接入/清洗/抽取验证（检验真实语料上的抽取鲁棒性与词表命中率），
|
||||
# 但不进图谱/演化/新岗位发现——避免统一日期造成 2019 虚假演化尖峰、
|
||||
# 以及"单年份→增长率视为达标"导致的待审核候选洪泛。
|
||||
passive = {s.strip() for s in str(cfg.get("passive_sources") or "").split(",") if s.strip()}
|
||||
if passive:
|
||||
passive_records = [r for r in records if r.source in passive]
|
||||
if passive_records:
|
||||
hit = sum(1 for r in passive_records
|
||||
if extractions.get(r.jid) and extractions[r.jid].required_skills)
|
||||
stats["stages"]["real_corpus"] = {
|
||||
"records": len(passive_records), "sources": sorted(passive),
|
||||
"skill_hit_rate": round(hit / max(1, len(passive_records)), 4),
|
||||
}
|
||||
logger.info("被动语料 %d 条（%s）：词表技能命中率 %.1f%%，不进图谱/演化/发现",
|
||||
len(passive_records), "/".join(sorted(passive)),
|
||||
100 * hit / max(1, len(passive_records)))
|
||||
records = [r for r in records if r.source not in passive]
|
||||
|
||||
# 4. 图谱写入：既有岗位定义 + 观察流
|
||||
first_years: dict[str, int] = {}
|
||||
for r in records:
|
||||
|
||||
+22
-3
@@ -318,12 +318,31 @@
|
||||
"samples": 200,
|
||||
"bad_lines": 0
|
||||
},
|
||||
"real_jd_500": {
|
||||
"accuracy": 0.9991,
|
||||
"title_accuracy": 1.0,
|
||||
"fields": {
|
||||
"required_skills": {
|
||||
"precision": 1.0,
|
||||
"recall": 0.998,
|
||||
"f1": 0.999
|
||||
},
|
||||
"preferred_skills": null,
|
||||
"scenarios": {
|
||||
"precision": 1.0,
|
||||
"recall": 1.0,
|
||||
"f1": 1.0
|
||||
}
|
||||
},
|
||||
"samples": 500,
|
||||
"bad_lines": 0
|
||||
},
|
||||
"real_resume": {
|
||||
"accuracy": 1.0,
|
||||
"name_accuracy": 1.0,
|
||||
"phone_accuracy": 1.0,
|
||||
"samples": 50,
|
||||
"phone_samples": 50,
|
||||
"empty_skill_resumes": 28
|
||||
"samples": 200,
|
||||
"phone_samples": 200,
|
||||
"empty_skill_resumes": 113
|
||||
}
|
||||
}
|
||||
+32
-7
@@ -9,8 +9,9 @@
|
||||
|
||||
两个真实数据维度（data/real/，与合成集分开报告、不混算）：
|
||||
4. 真实 JD（AutoWeakS 初标+抽检 silver gold）—— 对 jd_sample.jsonl 重新抽取，
|
||||
与 gold_answers.json 算技能/场景字段级 F1；
|
||||
5. 真实简历（AIStudio CC0）—— 对 50 份简历解析，姓名/电话命中率 + 技能 F1。
|
||||
与 gold_answers.json 算技能/场景字段级 F1；另按 jd_eval_500.jsonl /
|
||||
gold_answers_500.json 报告 500 条分层口径（real_jd_500，包含原 200 条）；
|
||||
5. 真实简历（AIStudio CC0）—— 对 200 份简历解析，姓名/电话命中率 + 技能 F1。
|
||||
|
||||
输出 reports/metrics.json 与控制台报告。
|
||||
"""
|
||||
@@ -115,15 +116,14 @@ def eval_match() -> dict:
|
||||
return {"accuracy": round(hit / len(pairs), 4), "pairs": len(pairs), "detail": detail}
|
||||
|
||||
|
||||
def eval_real_jd() -> dict:
|
||||
def _eval_real_jd(records_path: Path, gold_path: Path) -> dict:
|
||||
"""真实 JD（AutoWeakS）：从文本重新抽取，与 silver gold（初标+抽检）字段级比对。
|
||||
|
||||
真实 JD 是自由文本、无编号版面，gold 的 responsibilities/preferred_skills 为空，
|
||||
口径 = 全文词表技能（required_skills）+ 应用场景 + 启发式岗位名，见 data/real/README.md。
|
||||
"""
|
||||
records, errors = load_jd_file(ROOT / "data" / "real" / "autoweaks_jd" / "jd_sample.jsonl")
|
||||
gold = json.loads(
|
||||
(ROOT / "data" / "real" / "autoweaks_jd" / "gold_answers.json").read_text(encoding="utf-8"))
|
||||
records, errors = load_jd_file(records_path)
|
||||
gold = json.loads(gold_path.read_text(encoding="utf-8"))
|
||||
extractor = OfflineExtractor(load_lexicon())
|
||||
|
||||
fields = ["required_skills", "preferred_skills", "scenarios"]
|
||||
@@ -143,7 +143,9 @@ def eval_real_jd() -> dict:
|
||||
|
||||
detail, active_f1 = {}, []
|
||||
for f in fields:
|
||||
if not gold_n[f] and not pred[f]:
|
||||
# 真实 JD 的 gold 口径：responsibilities/preferred_skills 为空即不参与评分
|
||||
# （即使抽取器从"优先考虑"字样产出加分候选，也不计入，口径见 data/real/README.md）
|
||||
if not gold_n[f]:
|
||||
detail[f] = None # 该字段在真实语料中无标注口径，不参与
|
||||
continue
|
||||
p, r, f1 = prf(tp[f], pred[f], gold_n[f])
|
||||
@@ -157,6 +159,20 @@ def eval_real_jd() -> dict:
|
||||
"fields": detail, "samples": len(records), "bad_lines": len(errors)}
|
||||
|
||||
|
||||
def eval_real_jd() -> dict:
|
||||
"""真实 JD 200 条口径（历史基线）。"""
|
||||
return _eval_real_jd(
|
||||
ROOT / "data" / "real" / "autoweaks_jd" / "jd_sample.jsonl",
|
||||
ROOT / "data" / "real" / "autoweaks_jd" / "gold_answers.json")
|
||||
|
||||
|
||||
def eval_real_jd_500() -> dict:
|
||||
"""真实 JD 500 条分层口径（含原 200 条，见 scripts/expand_real_dataset.py）。"""
|
||||
return _eval_real_jd(
|
||||
ROOT / "data" / "real" / "autoweaks_jd" / "jd_eval_500.jsonl",
|
||||
ROOT / "data" / "real" / "autoweaks_jd" / "gold_answers_500.json")
|
||||
|
||||
|
||||
def eval_real_resume() -> dict:
|
||||
"""真实简历（AIStudio CC0）：姓名/电话命中率（脱敏形态比对）+ 技能 F1。"""
|
||||
gold = json.loads(
|
||||
@@ -194,6 +210,7 @@ def main() -> None:
|
||||
resume = eval_resume()
|
||||
match = eval_match()
|
||||
real_jd = eval_real_jd()
|
||||
real_jd_500 = eval_real_jd_500()
|
||||
real_resume = eval_real_resume()
|
||||
metrics = {
|
||||
"jd_extraction_accuracy": jd["accuracy"],
|
||||
@@ -203,6 +220,7 @@ def main() -> None:
|
||||
"resume_detail": resume["detail"],
|
||||
"match_detail": match["detail"],
|
||||
"real_jd": real_jd,
|
||||
"real_jd_500": real_jd_500,
|
||||
"real_resume": real_resume,
|
||||
}
|
||||
out = ROOT / "reports"
|
||||
@@ -225,6 +243,13 @@ def main() -> None:
|
||||
print("真实数据维度（data/real/，与合成集分开报告）")
|
||||
print(f"真实 JD（AutoWeakS 初标+抽检）: 综合 F1 {real_jd['accuracy']:.2%}"
|
||||
f" (样本 {real_jd['samples']} 条)")
|
||||
print(f"真实 JD（500 条分层） : 综合 F1 {real_jd_500['accuracy']:.2%}"
|
||||
f" (样本 {real_jd_500['samples']} 条)")
|
||||
for f, d in real_jd_500["fields"].items():
|
||||
if d is None:
|
||||
print(f" - {f:<18} (真实语料无标注口径，不参与)")
|
||||
else:
|
||||
print(f" - {f:<18} P={d['precision']:.2%} R={d['recall']:.2%} F1={d['f1']:.2%}")
|
||||
for f, d in real_jd["fields"].items():
|
||||
if d is None:
|
||||
print(f" - {f:<18} (真实语料无标注口径，不参与)")
|
||||
|
||||
@@ -0,0 +1,266 @@
|
||||
"""数据扩产脚本：在 build_real_dataset.py 的基础上扩充真实数据集（T1-T4）。
|
||||
|
||||
产物：
|
||||
1. data/real/autoweaks_jd/jd_full.jsonl
|
||||
—— AutoWeakS 全量中文 JD（去分词 + 技术岗过滤 + 全文去重），
|
||||
保留原 200 条 jd_sample 记录（jid/原文不变），其余新增条目
|
||||
publish_date 统一 2019-06-01（该子集不参与演化分析，仅作语料/评测）；
|
||||
2. data/real/autoweaks_jd/jd_eval_500.jsonl + gold_answers_500.json
|
||||
—— 真实 JD 评测集从 200 扩到 500：包含原 200 条，另按岗位类别分层补 300 条，
|
||||
gold 沿用"SkillMatcher 初标"口径（人工抽检记录见 data/real/README.md）；
|
||||
3. data/real/aistudio_resumes/：50 份扩到 200 份（保留 PII，沿用既有目录约定）；
|
||||
4. data/real/batch_demo_resumes/：12 份扩到 32 份（落盘前必须经 mask_text 脱敏）。
|
||||
|
||||
英文数据集（T5）默认不接入，原因记录在 data/real/README.md。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import json
|
||||
import random
|
||||
import re
|
||||
import sys
|
||||
from collections import Counter
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
|
||||
from app.extract.base import SkillMatcher, load_lexicon # noqa: E402
|
||||
from app.match.resume import mask_text # noqa: E402
|
||||
from build_real_dataset import ( # noqa: E402
|
||||
DST,
|
||||
SRC,
|
||||
SEED,
|
||||
TECH_RE,
|
||||
_PHONE_RE,
|
||||
detokenize,
|
||||
guess_title,
|
||||
render_aistudio_resume,
|
||||
render_chinese_resume,
|
||||
)
|
||||
|
||||
AW_DIR = DST / "autoweaks_jd"
|
||||
RESUME_DIR = DST / "aistudio_resumes"
|
||||
DEMO_DIR = DST / "batch_demo_resumes"
|
||||
|
||||
|
||||
# ---------- T1：AutoWeakS 全量中文 JD ----------
|
||||
def build_autoweaks_full(matcher: SkillMatcher) -> list[dict]:
|
||||
"""去分词 → 技术岗过滤 → 全文去重；保留原 200 条样本，其余追加。"""
|
||||
pools: dict[str, list[str]] = {}
|
||||
for fn, tag in [("Tecent_job_phrase.txt", "腾讯"), ("JD_job_phrase.txt", "京东")]:
|
||||
lines = (SRC / "autoweaks" / fn).read_text(encoding="utf-8").splitlines()
|
||||
seen, pool = set(), []
|
||||
for ln in lines:
|
||||
text = detokenize(ln)
|
||||
if len(text) < 60 or not TECH_RE.search(text):
|
||||
continue
|
||||
if not matcher.find_names(text):
|
||||
continue
|
||||
if text in seen:
|
||||
continue
|
||||
seen.add(text)
|
||||
pool.append(text)
|
||||
pools[tag] = pool
|
||||
|
||||
sample_path = AW_DIR / "jd_sample.jsonl"
|
||||
sample = [json.loads(ln) for ln in
|
||||
sample_path.read_text(encoding="utf-8").splitlines() if ln.strip()]
|
||||
sample_texts = {r["text"] for r in sample}
|
||||
|
||||
records = list(sample)
|
||||
new_pairs: list[tuple[str, str]] = []
|
||||
for tag in ("腾讯", "京东"):
|
||||
for text in sorted(pools[tag]):
|
||||
if text in sample_texts:
|
||||
continue
|
||||
new_pairs.append((tag, text))
|
||||
|
||||
for i, (company, text) in enumerate(new_pairs, start=len(sample) + 1):
|
||||
records.append({
|
||||
"jid": f"AW{i:04d}",
|
||||
"job_title": guess_title(text),
|
||||
"company": company,
|
||||
"publish_date": "2019-06-01", # 无源日期，统一标记；该子集不参与演化分析
|
||||
"source": "autoweaks",
|
||||
"url": "",
|
||||
"text": text,
|
||||
})
|
||||
|
||||
with (AW_DIR / "jd_full.jsonl").open("w", encoding="utf-8") as f:
|
||||
for r in records:
|
||||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||||
print(f"[T1] AutoWeakS 全量：腾讯池 {len(pools['腾讯'])} 京东池 {len(pools['京东'])}"
|
||||
f" → jd_full {len(records)} 条（含原样本 {len(sample)} 条）")
|
||||
return records
|
||||
|
||||
|
||||
# ---------- T2：真实 JD 评测集 500 条（分层） ----------
|
||||
def build_eval_500(matcher: SkillMatcher, full: list[dict]) -> tuple[list[dict], dict]:
|
||||
rng = random.Random(SEED)
|
||||
sample = [json.loads(ln) for ln in
|
||||
(AW_DIR / "jd_sample.jsonl").read_text(encoding="utf-8").splitlines() if ln.strip()]
|
||||
sample_ids = {r["jid"] for r in sample}
|
||||
|
||||
by_title: dict[str, list[dict]] = {}
|
||||
for r in full:
|
||||
by_title.setdefault(r["job_title"], []).append(r)
|
||||
sample_counts = Counter(r["job_title"] for r in sample)
|
||||
|
||||
remaining = {
|
||||
title: [r for r in group if r["jid"] not in sample_ids]
|
||||
for title, group in by_title.items()
|
||||
}
|
||||
for v in remaining.values():
|
||||
rng.shuffle(v)
|
||||
# 优先补齐样本覆盖少的岗位类别（样本数升序，剩余池大者优先）
|
||||
order = sorted(remaining, key=lambda t: (sample_counts.get(t, 0), -len(remaining[t])))
|
||||
|
||||
picked = list(sample)
|
||||
picked_ids = set(sample_ids)
|
||||
while len(picked) < 500:
|
||||
advanced = False
|
||||
for title in order:
|
||||
if not remaining[title]:
|
||||
continue
|
||||
rec = remaining[title].pop(0)
|
||||
if rec["jid"] in picked_ids:
|
||||
continue
|
||||
picked.append(rec)
|
||||
picked_ids.add(rec["jid"])
|
||||
advanced = True
|
||||
if len(picked) >= 500:
|
||||
break
|
||||
if not advanced:
|
||||
break
|
||||
|
||||
gold: dict[str, dict] = {}
|
||||
for r in picked:
|
||||
gold[r["jid"]] = {
|
||||
"job_title": r["job_title"],
|
||||
"responsibilities": [],
|
||||
"required_skills": matcher.find_names(r["text"]),
|
||||
"preferred_skills": [],
|
||||
"scenarios": matcher.find_scenarios(r["text"]),
|
||||
}
|
||||
|
||||
with (AW_DIR / "jd_eval_500.jsonl").open("w", encoding="utf-8") as f:
|
||||
for r in picked:
|
||||
f.write(json.dumps(r, ensure_ascii=False) + "\n")
|
||||
(AW_DIR / "gold_answers_500.json").write_text(
|
||||
json.dumps(gold, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
cats = sorted(Counter(r["job_title"] for r in picked).items())
|
||||
print(f"[T2] 真实 JD 评测集：{len(picked)} 条（含原样本 {len(sample)} 条），"
|
||||
f"岗位类别 {len(cats)} 类")
|
||||
return picked, gold
|
||||
|
||||
|
||||
# ---------- T3：AIStudio 简历 50 → 200 ----------
|
||||
def build_aistudio_extra(matcher: SkillMatcher) -> None:
|
||||
target_total = 200 # 50（既有）→ 200，幂等：够了就跳过
|
||||
rng = random.Random(SEED + 4)
|
||||
gold_path = RESUME_DIR / "gold.json"
|
||||
gold = json.loads(gold_path.read_text(encoding="utf-8"))
|
||||
need = target_total - len(gold)
|
||||
if need <= 0:
|
||||
print(f"[T3] AIStudio 简历：已有 {len(gold)} 份，目标 {target_total}，跳过")
|
||||
return
|
||||
used = {g["source_id"] for g in gold.values()}
|
||||
|
||||
data = json.loads((SRC / "aistudio" / "train.json").read_text(encoding="utf-8"))
|
||||
cands = []
|
||||
for key, rec in data.items():
|
||||
if key in used:
|
||||
continue
|
||||
blob = json.dumps([rec.get("工作经历"), rec.get("项目经历")], ensure_ascii=False)
|
||||
if not TECH_RE.search(blob):
|
||||
continue
|
||||
if not _PHONE_RE.match(str(rec.get("电话", ""))):
|
||||
continue
|
||||
name = str(rec.get("姓名", "")).strip()
|
||||
if not re.fullmatch(r"[一-龥]{2,4}", name):
|
||||
continue
|
||||
cleaned = dict(rec)
|
||||
cleaned["姓名"] = name
|
||||
cands.append((key, cleaned))
|
||||
rng.shuffle(cands)
|
||||
picked = cands[:need]
|
||||
|
||||
start = len(gold) # 编号接续
|
||||
for i, (key, rec) in enumerate(picked, start=start + 1):
|
||||
fname = f"aistudio_{i:03d}.txt"
|
||||
text = render_aistudio_resume(rec)
|
||||
(RESUME_DIR / fname).write_text(text, encoding="utf-8")
|
||||
gold[fname] = {
|
||||
"name": rec["姓名"].strip(),
|
||||
"phone": rec["电话"],
|
||||
"skills": matcher.find_names(text),
|
||||
"source_id": key,
|
||||
}
|
||||
gold = dict(sorted(gold.items()))
|
||||
gold_path.write_text(json.dumps(gold, ensure_ascii=False, indent=1), encoding="utf-8")
|
||||
print(f"[T3] AIStudio 简历：候选 {len(cands)} → 新增 {len(picked)} 份"
|
||||
f"（目录合计 {len(gold)} 份）")
|
||||
|
||||
|
||||
# ---------- T4：chinese_resume 批量演示 12 → 32（必须脱敏） ----------
|
||||
def build_demo_extra() -> None:
|
||||
target_total = 32 # 12（既有）→ 32，幂等：够了就跳过
|
||||
rng = random.Random(SEED + 3)
|
||||
with (SRC / "chinese_resume" / "Chinese_resume_data.csv").open(encoding="utf-8") as f:
|
||||
rows = [r for r in csv.DictReader(f) if r.get("筛选结果") == "通过"]
|
||||
|
||||
existing = sorted(p.name for p in DEMO_DIR.glob("demo_*.txt"))
|
||||
need = target_total - len(existing)
|
||||
if need <= 0:
|
||||
print(f"[T4] chinese_resume 批量演示：已有 {len(existing)} 份，目标 {target_total}，跳过")
|
||||
return
|
||||
used_slugs = {re.sub(r"\.txt$", "", name).split("_", 2)[-1] for name in existing}
|
||||
by_intent: dict[str, list[dict]] = {}
|
||||
for r in rows:
|
||||
by_intent.setdefault(r["意向岗位"], []).append(r)
|
||||
intents = sorted(by_intent, key=lambda k: -len(by_intent[k]))
|
||||
|
||||
picked: list[dict] = []
|
||||
# 第一轮：优先覆盖已有文件未覆盖的意向岗位
|
||||
for intent in intents:
|
||||
if len(picked) >= need:
|
||||
break
|
||||
slug = re.sub(r"\W+", "", intent)
|
||||
if slug in used_slugs:
|
||||
continue
|
||||
picked.append(rng.choice(by_intent[intent]))
|
||||
used_slugs.add(slug)
|
||||
# 第二轮：从其余通过样本补齐（避免覆盖已有文件）
|
||||
rest = [r for intent in intents for r in by_intent[intent] if r not in picked]
|
||||
rng.shuffle(rest)
|
||||
for row in rest:
|
||||
if len(picked) >= need:
|
||||
break
|
||||
picked.append(row)
|
||||
|
||||
start = len(existing) # 既有 12 份，编号接续
|
||||
for i, row in enumerate(picked, start=start + 1):
|
||||
raw = render_chinese_resume(row)
|
||||
masked = mask_text(raw) # 脱敏后才落盘，原始 PII 不写入 data/
|
||||
slug = re.sub(r"\W+", "", row["意向岗位"])
|
||||
(DEMO_DIR / f"demo_{i:02d}_{slug}.txt").write_text(masked, encoding="utf-8")
|
||||
print(f"[T4] chinese_resume 批量演示：新增 {len(picked)} 份"
|
||||
f"（目录合计 {start + len(picked)} 份，全部已脱敏）")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if not SRC.exists():
|
||||
raise SystemExit(f"源数据目录不存在: {SRC}")
|
||||
matcher = SkillMatcher(load_lexicon())
|
||||
full = build_autoweaks_full(matcher)
|
||||
build_eval_500(matcher, full)
|
||||
build_aistudio_extra(matcher)
|
||||
build_demo_extra()
|
||||
print(f"数据扩产完成，产物写入 {DST}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -40,9 +40,16 @@ def isolated_state_db(tmp_path_factory):
|
||||
routes._cfg["state_db_path"] = str(tmp / "state.db")
|
||||
routes._cfg["audit_path"] = str(tmp / "audit.jsonl")
|
||||
routes._cfg["pending_path"] = str(tmp / "pending.json")
|
||||
# 测试不加载本机附加数据源：真实语料会让 API 触发的重建超出任务等待窗口
|
||||
extra_orig = routes._cfg.get("jd_extra_dirs")
|
||||
routes._cfg["jd_extra_dirs"] = ""
|
||||
yield
|
||||
for k, v in originals.items():
|
||||
if v is None:
|
||||
routes._cfg.pop(k, None)
|
||||
else:
|
||||
routes._cfg[k] = v
|
||||
if extra_orig is None:
|
||||
routes._cfg.pop("jd_extra_dirs", None)
|
||||
else:
|
||||
routes._cfg["jd_extra_dirs"] = extra_orig
|
||||
|
||||
@@ -61,6 +61,7 @@ def _tmp_cfg(tmp_path, records) -> dict:
|
||||
"\n".join(json.dumps(r, ensure_ascii=False) for r in records) + "\n", encoding="utf-8")
|
||||
cfg = load_config()
|
||||
cfg["jd_raw_dir"] = str(jd_dir)
|
||||
cfg["jd_extra_dirs"] = "" # 测试不加载本机附加数据源（真实语料会使 pipeline 变慢且引入不确定性）
|
||||
cfg["graph_path"] = str(tmp_path / "graph.json")
|
||||
cfg["state_db_path"] = str(tmp_path / "state.db")
|
||||
cfg["audit_path"] = str(tmp_path / "audit.jsonl")
|
||||
|
||||
+62
-7
@@ -8,6 +8,10 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from pydantic import ValidationError
|
||||
|
||||
from app.ingest.loader import load_jd_file
|
||||
from app.models import JDRecord
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0, str(ROOT))
|
||||
@@ -34,9 +38,12 @@ def test_real_tree_complete() -> None:
|
||||
assert (REAL / "README.md").exists()
|
||||
assert (JD_DIR / "jd_sample.jsonl").exists()
|
||||
assert (JD_DIR / "gold_answers.json").exists()
|
||||
assert len(list(RESUME_DIR.glob("*.txt"))) == 50
|
||||
assert (JD_DIR / "jd_full.jsonl").exists()
|
||||
assert (JD_DIR / "jd_eval_500.jsonl").exists()
|
||||
assert (JD_DIR / "gold_answers_500.json").exists()
|
||||
assert len(list(RESUME_DIR.glob("*.txt"))) == 200
|
||||
assert (RESUME_DIR / "gold.json").exists()
|
||||
assert len(list(DEMO_DIR.glob("*.txt"))) == 12
|
||||
assert len(list(DEMO_DIR.glob("*.txt"))) == 32
|
||||
for name in ("labels_isco_codes.txt", "labels_isco_labels.txt",
|
||||
"labels_esco_code.txt", "labels_esco_occupation.txt", "README.md"):
|
||||
assert (ESCO_DIR / name).exists(), name
|
||||
@@ -63,18 +70,19 @@ def test_jd_gold_schema() -> None:
|
||||
|
||||
def test_resume_gold_schema() -> None:
|
||||
gold = json.loads((RESUME_DIR / "gold.json").read_text(encoding="utf-8"))
|
||||
assert len(gold) == 50
|
||||
assert len(gold) == 200
|
||||
for fname, g in gold.items():
|
||||
assert (RESUME_DIR / fname).exists(), fname
|
||||
assert re.fullmatch(r"[一-龥]{2,4}", g["name"]), fname
|
||||
assert re.fullmatch(r"1[3-9]\d{9}", g["phone"]), fname
|
||||
assert isinstance(g["skills"], list) and all(isinstance(s, str) for s in g["skills"])
|
||||
assert re.fullmatch(r"[0-9a-f]{12}", g["source_id"]), fname
|
||||
|
||||
|
||||
def test_batch_demo_masked() -> None:
|
||||
"""批量演示素材必须已脱敏：无 11 位手机号、邮箱仅呈 x***@domain 形态、姓名带 *。"""
|
||||
files = list(DEMO_DIR.glob("*.txt"))
|
||||
assert len(files) == 12
|
||||
assert len(files) == 32
|
||||
phone_re = re.compile(r"1[3-9]\d{9}")
|
||||
email_re = re.compile(r"[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}")
|
||||
name_re = re.compile(r"姓名[:：]\s*([一-龥]{2,4})") # 未脱敏姓名（脱敏后含 *）
|
||||
@@ -87,16 +95,63 @@ def test_batch_demo_masked() -> None:
|
||||
assert re.search(r"姓名[:：]\s*[一-龥]\*", text), f.name
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dim", ["real_jd", "real_resume"])
|
||||
def test_jd_full_schema_and_contains_sample() -> None:
|
||||
"""jd_full.jsonl：每行过 JDRecord 校验、全文去重、包含原 200 条样本（jid/原文一致）。"""
|
||||
records, errors = load_jd_file(JD_DIR / "jd_full.jsonl")
|
||||
assert errors == []
|
||||
assert len(records) >= 200
|
||||
sample = [json.loads(ln) for ln in
|
||||
(JD_DIR / "jd_sample.jsonl").read_text(encoding="utf-8").splitlines() if ln.strip()]
|
||||
by_jid = {r.jid: r for r in records}
|
||||
for s in sample:
|
||||
assert s["jid"] in by_jid, s["jid"]
|
||||
assert by_jid[s["jid"]].text == s["text"], s["jid"]
|
||||
texts = {r.text for r in records}
|
||||
assert len(texts) == len(records) # 全文哈希去重
|
||||
for r in records:
|
||||
assert r.source == "autoweaks"
|
||||
assert re.fullmatch(r"2019-\d{2}-\d{2}", r.publish_date.isoformat()), r.jid
|
||||
assert len(r.text) >= 60
|
||||
|
||||
|
||||
def test_jd_eval_500_schema_and_superset() -> None:
|
||||
"""jd_eval_500.jsonl：500 条、含原 200 条样本、gold Schema 与正式 gold 一致。"""
|
||||
records, errors = load_jd_file(JD_DIR / "jd_eval_500.jsonl")
|
||||
assert errors == [] and len(records) == 500
|
||||
full = load_jd_file(JD_DIR / "jd_full.jsonl")[0]
|
||||
full_ids = {r.jid for r in full}
|
||||
sample = [json.loads(ln) for ln in
|
||||
(JD_DIR / "jd_sample.jsonl").read_text(encoding="utf-8").splitlines() if ln.strip()]
|
||||
sample_ids = {s["jid"] for s in sample}
|
||||
eval_ids = {r.jid for r in records}
|
||||
assert sample_ids <= eval_ids <= full_ids
|
||||
gold = json.loads((JD_DIR / "gold_answers_500.json").read_text(encoding="utf-8"))
|
||||
assert set(gold) == eval_ids
|
||||
for jid, g in gold.items():
|
||||
assert set(g) == set(GOLD_JD_FIELDS), jid
|
||||
assert isinstance(g["job_title"], str) and g["job_title"]
|
||||
for f in ("responsibilities", "required_skills", "preferred_skills", "scenarios"):
|
||||
assert isinstance(g[f], list) and all(isinstance(s, str) for s in g[f]), (jid, f)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("dim", ["real_jd", "real_jd_500", "real_resume"])
|
||||
def test_eval_real_dimensions(dim: str) -> None:
|
||||
"""真实维度评测可运行且指标在合法区间（全量小数据集，直接实跑）。"""
|
||||
evaluate = _load_evaluate()
|
||||
result = evaluate.eval_real_jd() if dim == "real_jd" else evaluate.eval_real_resume()
|
||||
if dim == "real_jd":
|
||||
result = evaluate.eval_real_jd()
|
||||
elif dim == "real_jd_500":
|
||||
result = evaluate.eval_real_jd_500()
|
||||
else:
|
||||
result = evaluate.eval_real_resume()
|
||||
assert 0.0 <= result["accuracy"] <= 1.0
|
||||
if dim == "real_jd":
|
||||
assert result["samples"] == 200 and result["bad_lines"] == 0
|
||||
assert result["fields"]["required_skills"]["f1"] >= 0.80 # 任务目标
|
||||
elif dim == "real_jd_500":
|
||||
assert result["samples"] == 500 and result["bad_lines"] == 0
|
||||
assert result["fields"]["required_skills"]["f1"] >= 0.80 # 任务目标
|
||||
else:
|
||||
assert result["samples"] == 50
|
||||
assert result["samples"] == 200
|
||||
assert result["name_accuracy"] == 1.0 # 任务目标：脱敏后姓氏+*
|
||||
assert result["phone_accuracy"] == 1.0
|
||||
|
||||
Reference in New Issue
Block a user