CareerNexus 职联：岗位能力图谱构建与动态演化分析平台（挑战杯 XH-202621）
- 全链路：多源JD接入/清洗交叉验证/结构化抽取(星火+离线双通道)/时序知识图谱/新岗位发现/演化分析/人岗匹配/学习路径 - WebUI 五视图：驾驶舱/图谱工作台/岗位演化/匹配(单人+批量)/学习路径 - 设置中心(config.json)/Token鉴权/SQLite状态库/语义向量通道(BGE-M3本地或联网API) - 71 个 pytest 用例；数据目录仅保留结构(.gitkeep)，数据由 scripts/generate_dataset.py 复现
This commit is contained in:
RoyirinRosmontis
@@ -0,0 +1,33 @@
|
||||
"""时间衰减：招聘数据存在时滞，越旧的 JD 对当前能力刻画的贡献越低。
|
||||
|
||||
权重公式：w = exp(-λ · age)，age 为发布日期距参考日期的年数。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import math
|
||||
from datetime import date
|
||||
|
||||
from app.models import JDRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def decay_weight(publish_date: date, ref_date: date, lam: float = 0.35) -> float:
|
||||
"""计算单条 JD 的时间衰减权重。"""
|
||||
age_years = max(0.0, (ref_date - publish_date).days / 365.25)
|
||||
return round(math.exp(-lam * age_years), 4)
|
||||
|
||||
|
||||
def apply_decay(
|
||||
records: list[JDRecord],
|
||||
ref_date: date | None = None,
|
||||
lam: float = 0.35,
|
||||
) -> list[JDRecord]:
|
||||
"""就地更新每条记录的 weight（与已有降权相乘，不会覆盖抄袭降权）。"""
|
||||
if ref_date is None:
|
||||
ref_date = max((r.publish_date for r in records), default=date.today())
|
||||
for r in records:
|
||||
r.weight = round(r.weight * decay_weight(r.publish_date, ref_date, lam), 4)
|
||||
logger.info("时间衰减完成：λ=%.3f，参考日期 %s", lam, ref_date.isoformat())
|
||||
return records
|
||||
@@ -0,0 +1,119 @@
|
||||
"""近重复（抄袭）JD 检测：纯 Python MinHash（64 哈希、5-gram shingles）+ LSH banding。
|
||||
|
||||
招聘网站之间存在大量"同文不同公司"的抄袭 JD，直接入图会导致技能频次虚高。
|
||||
本模块把近重复 JD 聚簇：簇内保留最早来源，其余标记 is_duplicate 并降权。
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import random
|
||||
|
||||
from app.models import JDRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
NUM_HASHES = 128 # MinHash 哈希函数个数（>=64，越高估计越稳）
|
||||
SHINGLE_N = 5 # 字符 5-gram shingles
|
||||
LSH_BANDS = 16 # 16 bands × 8 rows（严格分带，抑制模板化文本误配）
|
||||
_PRIME = (1 << 61) - 1 # 梅森素数，用于 (a*x+b) mod p 哈希族
|
||||
_MAXINT = (1 << 32) - 1
|
||||
|
||||
|
||||
def shingles(text: str, n: int = SHINGLE_N) -> set[int]:
|
||||
"""把文本切成字符 n-gram shingles，并映射为 32 位整数集合。"""
|
||||
text = "".join(text.split()) # 去掉空白，抵抗排版差异
|
||||
if len(text) < n:
|
||||
text = text + " " * (n - len(text))
|
||||
out = set()
|
||||
for i in range(len(text) - n + 1):
|
||||
h = hashlib.md5(text[i:i + n].encode("utf-8")).digest()
|
||||
out.add(int.from_bytes(h[:4], "little"))
|
||||
return out
|
||||
|
||||
|
||||
class MinHash:
|
||||
"""(a*x+b) mod p 置换哈希族实现的 MinHash 签名。"""
|
||||
|
||||
def __init__(self, num_hashes: int = NUM_HASHES, seed: int = 42):
|
||||
self.num_hashes = num_hashes
|
||||
# 确定性大随机系数，保证 (a*x+b) mod p 是真置换哈希且可复现
|
||||
rng = random.Random(seed)
|
||||
self._a = [rng.randrange(1, _PRIME) for _ in range(num_hashes)]
|
||||
self._b = [rng.randrange(0, _PRIME) for _ in range(num_hashes)]
|
||||
|
||||
def signature(self, shingle_set: set[int]) -> list[int]:
|
||||
return [min((a * x + b) % _PRIME for x in shingle_set)
|
||||
for a, b in zip(self._a, self._b)]
|
||||
|
||||
def estimate(self, s1: list[int], s2: list[int]) -> float:
|
||||
return sum(1 for x, y in zip(s1, s2) if x == y) / self.num_hashes
|
||||
|
||||
|
||||
def _lsh_candidates(sig: list[int], bands: int, buckets: dict) -> None:
|
||||
"""把签名按 band 切段放入哈希桶，同桶者成为候选对。"""
|
||||
rows = len(sig) // bands
|
||||
for b in range(bands):
|
||||
key = (b, tuple(sig[b * rows:(b + 1) * rows]))
|
||||
buckets.setdefault(key, []).append(sig)
|
||||
|
||||
|
||||
def find_near_duplicates(
|
||||
records: list[JDRecord],
|
||||
threshold: float = 0.85,
|
||||
num_hashes: int = NUM_HASHES,
|
||||
bands: int = LSH_BANDS,
|
||||
) -> list[list[str]]:
|
||||
"""对全部 JD 做近重复聚簇，返回簇列表（每簇为 jid 列表，保留者排第一）。
|
||||
|
||||
簇内按 (publish_date, jid) 排序，最早来源为保留者；其余记录被标记
|
||||
``is_duplicate=True``、``dup_of=<保留者 jid>`` 并降权 0.2。
|
||||
"""
|
||||
mh = MinHash(num_hashes)
|
||||
sigs: dict[str, list[int]] = {}
|
||||
for r in records:
|
||||
sigs[r.jid] = mh.signature(shingles(r.text))
|
||||
|
||||
buckets: dict = {}
|
||||
for sig in sigs.values():
|
||||
_lsh_candidates(sig, bands, buckets)
|
||||
|
||||
# 并查集聚簇：候选对需通过完整签名相似度复核
|
||||
parent = {r.jid: r.jid for r in records}
|
||||
|
||||
def find(x: str) -> str:
|
||||
while parent[x] != x:
|
||||
parent[x] = parent[parent[x]]
|
||||
x = parent[x]
|
||||
return x
|
||||
|
||||
def union(x: str, y: str) -> None:
|
||||
parent[find(x)] = find(y)
|
||||
|
||||
jid_of_sig = {id(sig): jid for jid, sig in sigs.items()}
|
||||
for members in buckets.values():
|
||||
for i in range(len(members)):
|
||||
for j in range(i + 1, len(members)):
|
||||
if mh.estimate(members[i], members[j]) >= threshold:
|
||||
union(jid_of_sig[id(members[i])], jid_of_sig[id(members[j])])
|
||||
|
||||
groups: dict[str, list[JDRecord]] = {}
|
||||
by_jid = {r.jid: r for r in records}
|
||||
for r in records:
|
||||
groups.setdefault(find(r.jid), []).append(r)
|
||||
|
||||
clusters: list[list[str]] = []
|
||||
for members in groups.values():
|
||||
if len(members) < 2:
|
||||
continue
|
||||
members.sort(key=lambda r: (r.publish_date, r.jid))
|
||||
keeper = members[0]
|
||||
cluster = [keeper.jid]
|
||||
for dup in members[1:]:
|
||||
dup.is_duplicate = True
|
||||
dup.dup_of = keeper.jid
|
||||
dup.weight = round(dup.weight * 0.2, 4) # 抄袭来源降权
|
||||
cluster.append(dup.jid)
|
||||
clusters.append(cluster)
|
||||
logger.info("检出近重复簇：保留 %s，降权 %d 条", keeper.jid, len(members) - 1)
|
||||
return clusters
|
||||
@@ -0,0 +1,47 @@
|
||||
"""噪声过滤：识别广告尾注、无关条款等非岗位描述内容。"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import re
|
||||
|
||||
from app.models import JDRecord
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 噪声特征：广告导流、扫码领资料、与岗位无关的商业条款等
|
||||
NOISE_PATTERNS = [
|
||||
r"扫码|点击关注|关注公众号|领取.{0,6}(资料|福利|红包)",
|
||||
r"【广告】|推广信息|商务合作请加",
|
||||
r"长期回收|代办|加盟代理|刷单",
|
||||
r"内部推荐奖.{0,10}扫码",
|
||||
]
|
||||
_NOISE_RE = re.compile("|".join(NOISE_PATTERNS))
|
||||
|
||||
|
||||
def is_noise(text: str) -> bool:
|
||||
"""判断文本是否含有噪声内容。"""
|
||||
return bool(_NOISE_RE.search(text))
|
||||
|
||||
|
||||
def strip_noise_lines(text: str) -> str:
|
||||
"""逐行剔除噪声行，保留正文（用于噪声行与正文混排的情况）。"""
|
||||
kept = [ln for ln in text.splitlines() if ln.strip() and not is_noise(ln)]
|
||||
return "\n".join(kept)
|
||||
|
||||
|
||||
def filter_noise(records: list[JDRecord]) -> list[JDRecord]:
|
||||
"""标记噪声记录：整条以噪声为主的记录 is_noise=True 并降权；混排的只剔除噪声行。"""
|
||||
marked = 0
|
||||
for r in records:
|
||||
lines = [ln for ln in r.text.splitlines() if ln.strip()]
|
||||
noisy = [ln for ln in lines if is_noise(ln)]
|
||||
if not noisy:
|
||||
continue
|
||||
if len(noisy) >= max(1, len(lines) // 2):
|
||||
r.is_noise = True
|
||||
r.weight = round(r.weight * 0.1, 4)
|
||||
marked += 1
|
||||
else:
|
||||
r.text = strip_noise_lines(r.text)
|
||||
logger.info("噪声过滤：%d 条记录被标记为噪声", marked)
|
||||
return records
|
||||
Reference in New Issue
Block a user