forked from admin/PlantAI-rag-new
fix(aqua): DOI pairing decoder and MD extraction regexes
- MD: allow parens in DOI (old-style 10.1016/0006-3207(93)90088-i was truncated at ")"); prioritize DOI-line doi.org URL; strip trailing paren only when unbalanced - PDF: require 10. at segment start (fixes year artifacts like 2010.0170 masquerading as DOI); keep full candidate on no-hit instead of shortest prefix (10.1002/pd.71 was truncated to 10.1002); absorb paren segments (10.1061/(asce)wr.1943-...); stop at DOI未知 marker keeping registrant prefix; accept 10_ segment start variant - undecodable PDFs: 71 -> 54; manifest DOI values now complete - pairing count unchanged (3404) = true DOI overlap of the two corpora Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5
@@ -29,8 +29,9 @@ from concurrent.futures import ThreadPoolExecutor
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
MD_URL_RE = re.compile(r"https?://doi\.org/([^\s\)\]）]+)")
|
||||
MD_DOI_RE = re.compile(r"DOI：([^\s\]（(]+)")
|
||||
MD_DOI_LINE_URL_RE = re.compile(r"DOI[：:][^\n]*https?://doi\.org/([^\s\]）]+)")
|
||||
MD_URL_RE = re.compile(r"https?://doi\.org/([^\s\]）]+)")
|
||||
MD_DOI_RE = re.compile(r"DOI[：:]\s*([^\s\]（]+)")
|
||||
MD_TITLE_RE = re.compile(r"^#\s+(.+)$")
|
||||
MD_YEAR_RE = re.compile(r"年份[：:]\s*(\d{4})")
|
||||
DIGITS_RE = re.compile(r"^\d{4,9}$")
|
||||
@@ -43,13 +44,21 @@ CSV_HEADER = [
|
||||
|
||||
|
||||
def norm_doi(d: str) -> str:
|
||||
"""归一化:去首尾空白、小写、去尾部标点;内部 _ 不换 /(MD 卡里已是斜杠形态)。"""
|
||||
return d.strip().lower().rstrip(".,;)]")
|
||||
"""归一化:去首尾空白、小写、去尾部标点(含中文标点);尾部 ")" 仅在括号
|
||||
不配对时剥掉(markdown 链接闭合符假象),保证旧式带括号 DOI 完整;
|
||||
内部 _ 不换 /(MD 卡里已是斜杠形态)。"""
|
||||
s = d.strip().lower().rstrip(".,;]。，；、")
|
||||
if s.endswith(")") and s.count("(") < s.count(")"):
|
||||
s = s[:-1]
|
||||
return s
|
||||
|
||||
|
||||
def extract_md_doi(text: str) -> str:
|
||||
"""两种 DOI 形态均尝试,取首个非空。"""
|
||||
m = MD_URL_RE.search(text)
|
||||
"""DOI 提取:优先「DOI:」标签行内的 doi.org URL;其次任意 doi.org URL;
|
||||
最后裸「DOI:」文本。取首个非空。"""
|
||||
m = MD_DOI_LINE_URL_RE.search(text)
|
||||
if not m:
|
||||
m = MD_URL_RE.search(text)
|
||||
if not m:
|
||||
m = MD_DOI_RE.search(text)
|
||||
return m.group(1).strip() if m else ""
|
||||
@@ -69,15 +78,28 @@ def md_metadata(text: str) -> tuple[str, str]:
|
||||
def decode_pdf_doi(filename: str, md_norms) -> tuple[str, str]:
|
||||
"""从 PDF 文件名解码 DOI,返回 (doi_norm, doi_norm) 或 ("", "")。
|
||||
|
||||
算法:找最后一个 10. 出现位置;此后按下划线切段;首段必须为 4-9 位数字
|
||||
(即「10 + 4-9 位数字」);从第 2 段起,段开头为字母数字的并入 DOI;
|
||||
末段恰好为 8 位十六进制(其后只剩 .pdf)则停止并入。歧义消解:对每处
|
||||
合法的 10. 位置生成「前缀固定、尾部可延伸」的候选集合,优先取下划线
|
||||
换成斜杠后能命中 md_norms 的最长解释;仍不中的取最短解释。
|
||||
文件名 DOI 形态: 年份_标题_10.xxx_xxx(下划线化)_8位hex.pdf
|
||||
算法:
|
||||
- 起始点: `10.` 必须位于段首(前一个字符是 `_` 或文件名开头),排除
|
||||
标题/年份中的 `2010.xxx` 假象;
|
||||
- 按下划线切段;首段必须为 4-9 位数字(「10 + 4-9 位数字」);
|
||||
- 从第 2 段起并入 DOI:段开头为 ASCII 字母数字的并入;段开头为 "(" 且
|
||||
当前已并入部分不以 ")" 结尾时并入(DOI 后缀可含括号,如
|
||||
10.1061_(asce)wr.1943-5452.0000194);遇 "DOI未知" 段停止(文件名
|
||||
标注 DOI 后缀未知,只有注册机构前缀,如 10_1016_j_aquaculture);
|
||||
- 末段为 8 位十六进制(其后只剩 .pdf)时停止;
|
||||
- 遇 "DOI未知" 段:注册机构前缀(如 10.1016)仍生成候选后停止
|
||||
(文件名标注 DOI 后缀未知);
|
||||
- 歧义消解:候选全集按「能命中 md_norms 的最长解释」优先;无命中时取
|
||||
每个合法起始点的完整候选中最长者(DOI 本身可含下划线,如
|
||||
10.58782_flmnh.flvk7707,完整候选才是正确 DOI)。
|
||||
"""
|
||||
stem = filename[:-4] if filename.lower().endswith(".pdf") else filename
|
||||
candidates: list[str] = []
|
||||
for pos in reversed([m.start() for m in re.finditer(r"10\.", stem)]):
|
||||
for m in re.finditer(r"10[._]", stem):
|
||||
pos = m.start()
|
||||
if pos > 0 and stem[pos - 1] != "_":
|
||||
continue # 10. 不在段首:年份/标题中的假象
|
||||
segs = stem[pos + 3:].split("_")
|
||||
if not segs or not DIGITS_RE.match(segs[0]):
|
||||
continue # 首段不满足「10 + 4-9 位数字」,该 10. 位置无效
|
||||
@@ -86,8 +108,15 @@ def decode_pdf_doi(filename: str, md_norms) -> tuple[str, str]:
|
||||
seg = segs[i]
|
||||
if i == len(segs) - 1 and HEX8_RE.match(seg):
|
||||
break # 8 位十六进制哈希(其后只剩 .pdf),停止并入
|
||||
if seg and seg[0].isalnum():
|
||||
if seg == "DOI未知":
|
||||
candidates.append(norm_doi("10." + "/".join(parts)))
|
||||
break # 文件名标注 DOI 后缀未知,仅保留注册机构前缀
|
||||
if not seg:
|
||||
continue # 空段(如双下划线分隔):跳过
|
||||
if seg[0].isascii() and seg[0].isalnum():
|
||||
parts.append(seg)
|
||||
elif seg.startswith("(") and not "".join(parts).endswith(")"):
|
||||
parts.append(seg) # DOI 后缀括号段,如 (asce)wr.1943-...
|
||||
else:
|
||||
break
|
||||
for k in range(1, len(parts) + 1):
|
||||
@@ -95,7 +124,7 @@ def decode_pdf_doi(filename: str, md_norms) -> tuple[str, str]:
|
||||
if not candidates:
|
||||
return "", ""
|
||||
hits = [c for c in candidates if c in md_norms]
|
||||
chosen = max(hits, key=len) if hits else min(candidates, key=len)
|
||||
chosen = max(hits, key=len) if hits else max(candidates, key=len)
|
||||
return chosen, chosen
|
||||
|
||||
|
||||
@@ -108,11 +137,22 @@ def sha256_of(path: Path) -> str:
|
||||
|
||||
|
||||
def pdf_title_year(stem: str) -> tuple[str, str]:
|
||||
"""从文件名提取年份(首段 4 位数字)与标题(年份后、DOI 起始前)。"""
|
||||
"""从文件名提取年份(首段 4 位数字)与标题(年份后、DOI 起始前)。
|
||||
|
||||
DOI 起始位置用「段首 10. + 首段 4-9 位数字」判据定位,与 decode 一致,
|
||||
避免标题/年份里的 2010. 假象导致标题截断。
|
||||
"""
|
||||
parts = stem.split("_")
|
||||
year = parts[0] if len(parts) > 1 and re.fullmatch(r"\d{4}", parts[0]) else ""
|
||||
idx = -1
|
||||
for m in re.finditer(r"10\.", stem):
|
||||
if m.start() > 0 and stem[m.start() - 1] != "_":
|
||||
continue
|
||||
segs = stem[m.start() + 3:].split("_")
|
||||
if segs and DIGITS_RE.match(segs[0]):
|
||||
idx = m.start()
|
||||
break
|
||||
title = ""
|
||||
idx = stem.rfind("10.")
|
||||
if idx > 0:
|
||||
start = len(parts[0]) + 1 if year else 0
|
||||
title = stem[start:idx].strip("_").replace("_", " ").strip()
|
||||
|
||||
Reference in New Issue
Block a user