RoyirinRosmontis
a10442d78c
- 全链路：多源JD接入/清洗交叉验证/结构化抽取(星火+离线双通道)/时序知识图谱/新岗位发现/演化分析/人岗匹配/学习路径 - WebUI 五视图：驾驶舱/图谱工作台/岗位演化/匹配(单人+批量)/学习路径 - 设置中心(config.json)/Token鉴权/SQLite状态库/语义向量通道(BGE-M3本地或联网API) - 71 个 pytest 用例；数据目录仅保留结构(.gitkeep)，数据由 scripts/generate_dataset.py 复现
129 lines
5.1 KiB
Python
129 lines
5.1 KiB
Python
"""离线确定性抽取器：技能词表 + 版面规则，从 JD 文本抽出结构化岗位定义。
|
||
|
||
版面约定（数据集与真实主流招聘平台格式兼容）：
|
||
- “岗位名称：X”行给出岗位名称；
|
||
- 职责段（岗位职责/工作职责/工作内容/职位职责）逐条编号；
|
||
- 要求段（任职要求/岗位要求/任职条件/任职资格）出现必备技能；
|
||
- 加分段（加分项/优先考虑/优先条件）出现加分技能；
|
||
- 尾部段落（团队与福利/福利待遇/岗位福利/公司福利）不属于任何技能区，
|
||
作为加分段的结束边界，避免福利文案中的技能词混入加分技能；
|
||
- 应用场景词表全文匹配。
|
||
每个字段都附原文证据片段，供人机协同审核防“幻觉”。
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import logging
|
||
import re
|
||
|
||
from app.extract.base import SkillMatcher, load_lexicon
|
||
from app.models import ExtractionResult, JDRecord
|
||
|
||
logger = logging.getLogger(__name__)
|
||
|
||
# 版面标题词表（生成器与抽取器共用，保证规则与数据一致）
|
||
DUTY_HEADERS = ["岗位职责", "工作职责", "工作内容", "职位职责"]
|
||
REQ_HEADERS = ["任职要求", "岗位要求", "任职条件", "任职资格"]
|
||
PREF_HEADERS = ["加分项", "优先考虑", "优先条件"]
|
||
# 尾部段落标题：出现在加分段之后的团队/福利介绍，作为版面区域结束边界
|
||
TAIL_HEADERS = ["团队与福利", "福利待遇", "岗位福利", "公司福利"]
|
||
|
||
_TITLE_RE = re.compile(r"岗位名称[:：]\s*([^\n]+)")
|
||
_NUMBER_RE = re.compile(r"^\s*(?:\d+\s*[、.．)）]|[-•·])\s*")
|
||
|
||
|
||
def _find_header(text: str, headers: list[str]) -> tuple[int, int] | None:
|
||
"""返回第一处标题的 (起, 止)，找不到返回 None。"""
|
||
best: tuple[int, int] | None = None
|
||
for h in headers:
|
||
i = text.find(h)
|
||
if i >= 0 and (best is None or i < best[0]):
|
||
best = (i, i + len(h))
|
||
return best
|
||
|
||
|
||
class OfflineExtractor:
|
||
"""离线确定性抽取器（默认通道，无需任何外部依赖）。"""
|
||
|
||
name = "offline"
|
||
|
||
def __init__(self, lexicon: dict | None = None):
|
||
self.lexicon = lexicon or load_lexicon()
|
||
self.matcher = SkillMatcher(self.lexicon)
|
||
|
||
def _zones(self, text: str) -> dict[str, str]:
|
||
"""把文本切成 职责/要求/加分 三个版面区域（尾部福利段落不计入）。"""
|
||
duty = _find_header(text, DUTY_HEADERS)
|
||
req = _find_header(text, REQ_HEADERS)
|
||
pref = _find_header(text, PREF_HEADERS)
|
||
tail = _find_header(text, TAIL_HEADERS)
|
||
|
||
def zone(start_h, *end_hs):
|
||
if start_h is None:
|
||
return ""
|
||
s = start_h[1]
|
||
ends = [h[0] for h in end_hs if h is not None and h[0] >= s]
|
||
e = min(ends) if ends else len(text)
|
||
return text[s:e]
|
||
|
||
return {
|
||
"duty": zone(duty, req, pref, tail),
|
||
"req": zone(req, pref, tail),
|
||
"pref": zone(pref, tail),
|
||
}
|
||
|
||
@staticmethod
|
||
def _parse_duties(duty_zone: str) -> list[str]:
|
||
"""逐行解析编号职责，去掉编号前缀。"""
|
||
out: list[str] = []
|
||
for line in duty_zone.splitlines():
|
||
line = _NUMBER_RE.sub("", line).strip().strip("：:")
|
||
if line:
|
||
out.append(line)
|
||
return out
|
||
|
||
def extract(self, record: JDRecord) -> ExtractionResult:
|
||
text = record.text
|
||
zones = self._zones(text)
|
||
|
||
m = _TITLE_RE.search(text)
|
||
title = m.group(1).strip() if m else record.job_title
|
||
|
||
duties = self._parse_duties(zones["duty"])
|
||
if zones["req"]:
|
||
required = self.matcher.find_names(zones["req"])
|
||
else:
|
||
# 无版面结构的真实 JD（如 AutoWeakS 语料）没有"任职要求"段，
|
||
# 兜底为全文扫描，避免技能字段整体漏抽
|
||
required = self.matcher.find_names(text)
|
||
preferred = self.matcher.find_names(zones["pref"]) if zones["pref"] else []
|
||
# 加分技能不应重复出现在必备里（极少数版面混排的兜底）
|
||
preferred = [s for s in preferred if s not in required]
|
||
scenarios = self.matcher.find_scenarios(text)
|
||
|
||
def snippet(zone_text: str, limit: int = 80) -> str:
|
||
return " ".join(zone_text.split())[:limit]
|
||
|
||
evidence = {
|
||
"job_title": m.group(0) if m else "",
|
||
"responsibilities": snippet(zones["duty"]),
|
||
"required_skills": snippet(zones["req"]),
|
||
"preferred_skills": snippet(zones["pref"]),
|
||
"scenarios": "、".join(scenarios),
|
||
}
|
||
# 置信度：五个字段命中比例（离线规则全命中即 1.0）
|
||
hits = sum([bool(title), bool(duties), bool(required), True, bool(scenarios)])
|
||
confidence = round(hits / 5.0, 4)
|
||
|
||
return ExtractionResult(
|
||
jid=record.jid,
|
||
job_title=title,
|
||
responsibilities=duties,
|
||
required_skills=required,
|
||
preferred_skills=preferred,
|
||
scenarios=scenarios,
|
||
evidence=evidence,
|
||
confidence=confidence,
|
||
extractor=self.name,
|
||
prompt_version="offline-v1",
|
||
)
|