RoyirinRosmontis
1722c00249
- scripts/expand_real_dataset.py：AutoWeakS 全量 2215 条 + 500 条分层评测集 + 简历扩量（seed=2026 可复现） - pipeline 被动语料机制（passive_sources）：真实无日期 JD 参与接入/清洗/抽取验证（2915 条命中率 99.8%），不进图谱/演化/发现 - 新岗位发现第四指标'差异度'与双年窗口增长率口径 - 评测新增 real_jd_500 维度（99.91%）；测试会话隔离 jd_extra_dirs 防止本机配置拖慢套件
199 lines
8.6 KiB
Python
199 lines
8.6 KiB
Python
"""端到端编排：导入 → 清洗 → 抽取 → 图谱写入 → 演化分析 → 新岗位发现。
|
||
|
||
供 CLI（scripts/run_demo.py）与 API（/api/jd/import）复用，
|
||
返回各阶段统计。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
from collections import Counter
|
||
from pathlib import Path
|
||
|
||
from app.cleaning.decay import apply_decay
|
||
from app.cleaning.dedup import find_near_duplicates
|
||
from app.cleaning.noise import filter_noise
|
||
from app.config import ROOT, load_config, resolve
|
||
from app.discover.cluster import discover_new_jobs, save_pending
|
||
from app.evolution.analyzer import analyze, build_observations
|
||
from app.extract.base import load_lexicon
|
||
from app.extract.llm import get_extractor
|
||
from app.graph.tkg import TemporalKG
|
||
from app.ingest.loader import load_jd_dir
|
||
from app.models import ExtractionResult, JDRecord, JobDefinition
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# 既有岗位注册表的数据截止年：早于该年出现的岗位视为“既有岗位”，
|
||
# 之后才出现的岗位交给“新岗位发现”链路（待审核后才入正式图谱）。
|
||
KNOWN_JOB_BEFORE_YEAR = 2023
|
||
|
||
logging.basicConfig(
|
||
level=logging.INFO,
|
||
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
||
)
|
||
|
||
|
||
def job_id_of(title: str) -> str:
|
||
return f"job_{title}"
|
||
|
||
|
||
def build_job_definitions(
|
||
records: list[JDRecord],
|
||
extractions: dict[str, ExtractionResult],
|
||
known_titles: set[str],
|
||
lexicon: dict,
|
||
) -> dict[str, JobDefinition]:
|
||
"""按标题聚合近三年抽取结果，生成既有岗位定义（频次阈值投票，多源一致才入选）。
|
||
|
||
岗位定义刻画“当前能力要求”，因此只用最近 3 个年份的 JD 聚合，
|
||
历史技能由时序图谱与演化事件表达，不进当前定义。
|
||
"""
|
||
jobs_meta = {j["title"]: j for j in lexicon.get("jobs", [])}
|
||
max_year = max((r.publish_date.year for r in records), default=2026)
|
||
recent_from = max_year - 2
|
||
defs: dict[str, JobDefinition] = {}
|
||
for title in sorted(known_titles):
|
||
group = [r for r in records
|
||
if r.jid in extractions and extractions[r.jid].job_title == title
|
||
and not r.is_noise and not r.is_duplicate
|
||
and r.publish_date.year >= recent_from]
|
||
if not group:
|
||
continue
|
||
n = len(group)
|
||
req = Counter(s for r in group for s in extractions[r.jid].required_skills)
|
||
pref = Counter(s for r in group for s in extractions[r.jid].preferred_skills)
|
||
duty = Counter(d for r in group for d in extractions[r.jid].responsibilities)
|
||
scen = Counter(s for r in group for s in extractions[r.jid].scenarios)
|
||
meta = jobs_meta.get(title, {})
|
||
required = [s for s, c in req.most_common() if c >= max(2, n * 0.25)]
|
||
defs[job_id_of(title)] = JobDefinition(
|
||
job_id=job_id_of(title),
|
||
title=title,
|
||
category=meta.get("category", ""),
|
||
industries=list(meta.get("industries", [])),
|
||
required_skills=required,
|
||
preferred_skills=[s for s, c in pref.most_common()
|
||
if c >= max(2, n * 0.15) and s not in required],
|
||
responsibilities=[d for d, _ in duty.most_common(5)],
|
||
scenarios=[s for s, _ in scen.most_common(5)],
|
||
expected_years=int(meta.get("expected_years", 3)),
|
||
status="active",
|
||
source_count=n,
|
||
company_count=len({r.company for r in group}),
|
||
)
|
||
return defs
|
||
|
||
|
||
def run_pipeline(config: dict | None = None) -> dict:
|
||
"""执行全链路，返回统计信息。"""
|
||
cfg = config or load_config()
|
||
stats: dict = {"stages": {}}
|
||
|
||
# 1. 导入
|
||
records, errors = load_jd_dir(resolve(cfg, "jd_raw_dir"))
|
||
source_stats = {"jd_raw": len(records)}
|
||
# 附加数据源（如 data/real/autoweaks_jd 真实数据集），可配置切换
|
||
for extra in str(cfg.get("jd_extra_dirs") or "").split(","):
|
||
extra = extra.strip()
|
||
if not extra:
|
||
continue
|
||
extra_path = Path(extra) if Path(extra).is_absolute() else ROOT / extra
|
||
if not extra_path.exists():
|
||
logger.warning("附加数据源不存在，已跳过: %s", extra_path)
|
||
continue
|
||
recs2, errs2 = load_jd_dir(extra_path)
|
||
records.extend(recs2)
|
||
errors.extend(errs2)
|
||
source_stats[extra_path.name] = len(recs2)
|
||
logger.info("附加数据源 %s: %d 条 JD", extra_path.name, len(recs2))
|
||
stats["stages"]["ingest"] = {"records": len(records), "bad_lines": len(errors),
|
||
"sources": source_stats}
|
||
|
||
# 2. 清洗：噪声 → 近重复 → 时间衰减
|
||
filter_noise(records)
|
||
dup_clusters = find_near_duplicates(records)
|
||
apply_decay(records, lam=float(cfg.get("decay_lambda", 0.35)))
|
||
stats["stages"]["cleaning"] = {
|
||
"dup_clusters": len(dup_clusters),
|
||
"duplicates": sum(1 for r in records if r.is_duplicate),
|
||
"noise": sum(1 for r in records if r.is_noise),
|
||
}
|
||
|
||
# 3. 抽取（LLM 或离线，自动降级）
|
||
lexicon = load_lexicon(resolve(cfg, "lexicon_path"))
|
||
extractor = get_extractor(cfg)
|
||
extractions: dict[str, ExtractionResult] = {}
|
||
for r in records:
|
||
if not r.is_noise:
|
||
extractions[r.jid] = extractor.extract(r)
|
||
stats["stages"]["extract"] = {"extracted": len(extractions),
|
||
"extractor": extractor.name}
|
||
|
||
# 3.5 被动语料分流：真实但未标日期的数据集（如 autoweaks，统一标记 2019-06-01），
|
||
# 参与接入/清洗/抽取验证（检验真实语料上的抽取鲁棒性与词表命中率），
|
||
# 但不进图谱/演化/新岗位发现——避免统一日期造成 2019 虚假演化尖峰、
|
||
# 以及"单年份→增长率视为达标"导致的待审核候选洪泛。
|
||
passive = {s.strip() for s in str(cfg.get("passive_sources") or "").split(",") if s.strip()}
|
||
if passive:
|
||
passive_records = [r for r in records if r.source in passive]
|
||
if passive_records:
|
||
hit = sum(1 for r in passive_records
|
||
if extractions.get(r.jid) and extractions[r.jid].required_skills)
|
||
stats["stages"]["real_corpus"] = {
|
||
"records": len(passive_records), "sources": sorted(passive),
|
||
"skill_hit_rate": round(hit / max(1, len(passive_records)), 4),
|
||
}
|
||
logger.info("被动语料 %d 条（%s）：词表技能命中率 %.1f%%，不进图谱/演化/发现",
|
||
len(passive_records), "/".join(sorted(passive)),
|
||
100 * hit / max(1, len(passive_records)))
|
||
records = [r for r in records if r.source not in passive]
|
||
|
||
# 4. 图谱写入：既有岗位定义 + 观察流
|
||
first_years: dict[str, int] = {}
|
||
for r in records:
|
||
if r.jid in extractions:
|
||
t = extractions[r.jid].job_title
|
||
first_years[t] = min(first_years.get(t, 9999), r.publish_date.year)
|
||
known_titles = {t for t, y in first_years.items() if y < KNOWN_JOB_BEFORE_YEAR}
|
||
|
||
kg = TemporalKG()
|
||
defs = build_job_definitions(records, extractions, known_titles, lexicon)
|
||
for d in defs.values():
|
||
kg.add_job(d)
|
||
for sc in d.scenarios:
|
||
kg.add_scenario(sc)
|
||
kg.g.add_edge(d.job_id, f"场景::{sc}", rel="APPLIES_TO")
|
||
for ind in d.industries:
|
||
kg.add_industry(ind)
|
||
kg.g.add_edge(d.job_id, f"行业::{ind}", rel="BELONGS_TO")
|
||
|
||
job_id_of_title = {d.title: d.job_id for d in defs.values()}
|
||
observations = build_observations(records, extractions, job_id_of_title)
|
||
for obs in observations:
|
||
kg.observe(obs)
|
||
stats["stages"]["graph"] = kg.stats()
|
||
|
||
# 5. 演化分析（事件追加进图谱）
|
||
events = analyze(kg, observations=observations)
|
||
stats["stages"]["evolution"] = {"events": len(events)}
|
||
|
||
# 6. 新岗位发现 → 待审核分区（SQLite 状态库）
|
||
candidates = discover_new_jobs(records, extractions, known_titles)
|
||
save_pending(candidates, resolve(cfg, "state_db_path"))
|
||
stats["stages"]["discover"] = {"candidates": [c.title for c in candidates]}
|
||
|
||
# 7. 人工状态重放：审核入库岗位强制纳入正式图谱（不受"2023 年前"规则限制），
|
||
# 节点编辑一并恢复（覆写层存于 SQLite job_overrides 表）
|
||
from app.services.overrides import replay_overrides
|
||
stats["stages"]["overrides"] = replay_overrides(kg, resolve(cfg, "state_db_path"))
|
||
|
||
kg.save(resolve(cfg, "graph_path"))
|
||
stats["graph_path"] = str(resolve(cfg, "graph_path"))
|
||
logger.info("pipeline 完成：%s", stats["stages"])
|
||
return stats
|
||
|
||
|
||
if __name__ == "__main__":
|
||
import json
|
||
print(json.dumps(run_pipeline(), ensure_ascii=False, indent=2))
|