jerryandClaude Fable 5
2f82c4a8dc
- 按生产 7900 容器黄金清单迁移(compile.manifest 449 条目,0 缺失) - 入口双态:api.app(生产同源 3 路由)/ api.main(裁剪 11 路由族 91 路径) - 修复:advanced/__init__ 空文件(包标记误覆盖,已从主项目恢复) - 修复:api/rag_routes.py 跨环境 site-packages 注入改 LEGACY_GF_SITE_PACKAGES 门控 - 修复:scripts/utils 缺失(动态加载 reasoning_engine) - 影子验证:kg/stats 数据连通(6.2s,与 7900 同源 normalized 后端) Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
127 lines
4.2 KiB
Python
Executable File
127 lines
4.2 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
按章节分块存储 - 参考 PlantConnectome 格式
|
|
"""
|
|
|
|
import fitz
|
|
import json
|
|
|
|
|
|
def extract_sections(pdf_path: str) -> dict:
|
|
"""提取PDF各章节"""
|
|
doc = fitz.open(pdf_path)
|
|
|
|
# 读取全文
|
|
full_text = ""
|
|
for page in doc:
|
|
full_text += page.get_text() + "\n"
|
|
doc.close()
|
|
|
|
# 简单章节识别
|
|
sections = {
|
|
'full_text': full_text,
|
|
'abstract': '',
|
|
'background': '',
|
|
'methods': '',
|
|
'results_phylogeny': '',
|
|
'results_expression': '',
|
|
'results_synteny': '',
|
|
'discussion': ''
|
|
}
|
|
|
|
lines = full_text.split('\n')
|
|
current_section = None
|
|
current_text = []
|
|
|
|
for line in lines:
|
|
line_lower = line.lower().strip()
|
|
|
|
# 检测章节
|
|
if 'abstract' in line_lower and len(line_lower) < 20:
|
|
if current_section:
|
|
sections[current_section] = '\n'.join(current_text).strip()
|
|
current_section = 'abstract'
|
|
current_text = []
|
|
elif any(x in line_lower for x in ['background', 'introduction']) and len(line_lower) < 30:
|
|
if current_section:
|
|
sections[current_section] = '\n'.join(current_text).strip()
|
|
current_section = 'background'
|
|
current_text = []
|
|
elif 'methods' in line_lower and len(line_lower) < 20:
|
|
if current_section:
|
|
sections[current_section] = '\n'.join(current_text).strip()
|
|
current_section = 'methods'
|
|
current_text = []
|
|
elif 'results' in line_lower and len(line_lower) < 20:
|
|
if current_section:
|
|
sections[current_section] = '\n'.join(current_text).strip()
|
|
current_section = 'results_phylogeny'
|
|
current_text = []
|
|
elif 'discussion' in line_lower and len(line_lower) < 20:
|
|
if current_section:
|
|
sections[current_section] = '\n'.join(current_text).strip()
|
|
current_section = 'discussion'
|
|
current_text = []
|
|
elif current_section:
|
|
# Results 内子章节检测
|
|
if current_section.startswith('results_'):
|
|
if any(x in line_lower for x in ['phylogenetic', 'gene structure', 'conserved motif']):
|
|
sections[current_section] = '\n'.join(current_text).strip()
|
|
current_text = []
|
|
current_section = 'results_phylogeny'
|
|
elif any(x in line_lower for x in ['expression pattern', 'tissue', 'qrt-pcr']):
|
|
sections[current_section] = '\n'.join(current_text).strip()
|
|
current_text = []
|
|
current_section = 'results_expression'
|
|
elif 'synteny' in line_lower:
|
|
sections[current_section] = '\n'.join(current_text).strip()
|
|
current_text = []
|
|
current_section = 'results_synteny'
|
|
|
|
current_text.append(line)
|
|
|
|
# 保存最后一个章节
|
|
if current_section and current_text:
|
|
sections[current_section] = '\n'.join(current_text).strip()
|
|
|
|
return sections
|
|
|
|
|
|
def main():
|
|
pdf_path = "paper/test_v2_reproduce/input/苦荞SPL2019BMCplantbios12870-019-1916-6.pdf"
|
|
pmid = "31286919"
|
|
|
|
print("=" * 70)
|
|
print("Extracting Sections (PlantConnectome Format)")
|
|
print("=" * 70)
|
|
|
|
sections = extract_sections(pdf_path)
|
|
|
|
# 创建 PlantConnectome 格式的 chunks
|
|
chunks = {}
|
|
for section_name, text in sections.items():
|
|
if text and len(text) > 100:
|
|
chunk_id = f"{pmid}_{section_name}"
|
|
chunks[chunk_id] = {
|
|
"pmid": pmid,
|
|
"section": section_name,
|
|
"text": text[:2000], # 限制长度
|
|
"char_count": len(text)
|
|
}
|
|
print(f"\n✓ {chunk_id}: {len(text)} chars")
|
|
|
|
# 保存
|
|
output_path = "evaluation/chunks/pmid31286919_chunks.json"
|
|
import os
|
|
os.makedirs("evaluation/chunks", exist_ok=True)
|
|
|
|
with open(output_path, 'w') as f:
|
|
json.dump(chunks, f, indent=2)
|
|
|
|
print(f"\n✓ Saved to: {output_path}")
|
|
print(f"Total chunks: {len(chunks)}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|