556 lines
13 KiB
Markdown
Executable File
556 lines
13 KiB
Markdown
Executable File
# V2 版本使用指南
|
|
|
|
> 版本: v2.0
|
|
> 日期: 2026-03-18
|
|
> 用途: 使用 PlantConnectome 优化版提示词提取 70+ 维结构化信息
|
|
|
|
---
|
|
|
|
## 快速开始 (3步)
|
|
|
|
### 第1步: 准备环境
|
|
|
|
```bash
|
|
# 进入项目目录
|
|
cd /home/xia-lab/gene-family-mas
|
|
|
|
# 确保依赖已安装
|
|
pip install openai PyMuPDF -q
|
|
|
|
# 设置 API 密钥
|
|
export DASHSCOPE_API_KEY="your-api-key"
|
|
```
|
|
|
|
### 第2步: 准备文件
|
|
|
|
```bash
|
|
# 确保提示词文件存在
|
|
ls -la prompts/extraction_prompt_v2.md
|
|
|
|
# 准备 PDF 文件
|
|
# 将你的论文 PDF 放在 paper/ceshi/1pian/ 目录下
|
|
```
|
|
|
|
### 第3步: 运行提取
|
|
|
|
```bash
|
|
# 使用 V2 提示词提取
|
|
python3 scripts/extract_structured.py \
|
|
paper/ceshi/1pian/你的论文.pdf \
|
|
--output paper/ceshi/1pian/extracted/v2_result.json
|
|
```
|
|
|
|
---
|
|
|
|
## 详细使用说明
|
|
|
|
## 方法一: 使用现有提取脚本 (推荐)
|
|
|
|
### 1.1 基本用法
|
|
|
|
```bash
|
|
# 基本提取
|
|
python3 scripts/extract_structured.py \
|
|
paper/ceshi/1pian/苦荞SPL2019BMCplantbios12870-019-1916-6.pdf \
|
|
--output paper/ceshi/1pian/extracted/v2_extraction.json
|
|
```
|
|
|
|
### 1.2 修改脚本使用 V2 提示词
|
|
|
|
编辑 `scripts/extract_structured.py`，修改提示词加载部分:
|
|
|
|
```python
|
|
# 找到 call_llm_extract 函数
|
|
# 修改提示词加载路径
|
|
|
|
def call_llm_extract(pdf_text: str, api_key: str = "", model: str = DEFAULT_MODEL):
|
|
"""调用LLM提取结构化信息"""
|
|
|
|
# 加载 V2 提示词
|
|
prompt_path = Path(__file__).parent.parent / "prompts" / "extraction_prompt_v2.md"
|
|
with open(prompt_path, 'r', encoding='utf-8') as f:
|
|
prompt_template = f.read()
|
|
|
|
# 替换占位符
|
|
user_prompt = prompt_template.replace("{chunk_text}", pdf_text[:120000])
|
|
|
|
# 调用 API...
|
|
```
|
|
|
|
### 1.3 批量提取
|
|
|
|
```bash
|
|
# 创建批量提取脚本
|
|
for pdf in paper/ceshi/1pian/*.pdf; do
|
|
echo "处理: $pdf"
|
|
python3 scripts/extract_structured.py "$pdf" \
|
|
--output "paper/ceshi/1pian/extracted/$(basename $pdf .pdf)_v2.json"
|
|
done
|
|
```
|
|
|
|
---
|
|
|
|
## 方法二: 使用 Python 直接调用
|
|
|
|
### 2.1 完整示例代码
|
|
|
|
创建 `run_v2_extraction.py`:
|
|
|
|
```python
|
|
#!/usr/bin/env python3
|
|
"""
|
|
V2 版本提取示例
|
|
"""
|
|
import os
|
|
import json
|
|
from pathlib import Path
|
|
|
|
# 配置
|
|
API_KEY = os.getenv("DASHSCOPE_API_KEY", "")
|
|
MODEL = "kimi-k2.5" # 或 "qwen-max"
|
|
|
|
def extract_from_pdf(pdf_path: str, output_path: str = None):
|
|
"""
|
|
使用 V2 提示词从 PDF 提取结构化信息
|
|
|
|
Args:
|
|
pdf_path: PDF 文件路径
|
|
output_path: 输出 JSON 文件路径 (可选)
|
|
|
|
Returns:
|
|
提取结果字典
|
|
"""
|
|
from openai import OpenAI
|
|
import fitz # PyMuPDF
|
|
|
|
# 1. 读取 PDF
|
|
print(f"📄 读取 PDF: {pdf_path}")
|
|
doc = fitz.open(pdf_path)
|
|
text = ""
|
|
for page in doc:
|
|
text += page.get_text("text") + "\n"
|
|
doc.close()
|
|
print(f" 文本长度: {len(text)} 字符")
|
|
|
|
# 2. 加载 V2 提示词
|
|
prompt_path = "/home/xia-lab/gene-family-mas/prompts/extraction_prompt_v2.md"
|
|
with open(prompt_path, 'r', encoding='utf-8') as f:
|
|
prompt_template = f.read()
|
|
|
|
# 3. 替换占位符
|
|
user_prompt = prompt_template.replace("{chunk_text}", text[:120000])
|
|
|
|
# 4. 调用 API
|
|
print(f"🤖 调用 {MODEL} 提取...")
|
|
client = OpenAI(
|
|
api_key=API_KEY,
|
|
base_url="https://coding.dashscope.aliyuncs.com/v1"
|
|
)
|
|
|
|
response = client.chat.completions.create(
|
|
model=f"bailian/{MODEL}",
|
|
messages=[
|
|
{"role": "system", "content": "You are a specialized bioinformatics paper information extraction expert."},
|
|
{"role": "user", "content": user_prompt}
|
|
],
|
|
temperature=0.1,
|
|
max_tokens=6000
|
|
)
|
|
|
|
# 5. 解析结果
|
|
result_text = response.choices[0].message.content
|
|
|
|
# 清理 JSON
|
|
import re
|
|
result_text = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f]', '', result_text)
|
|
if "```json" in result_text:
|
|
result_text = result_text.split("```json")[1]
|
|
if "```" in result_text:
|
|
result_text = result_text.split("```")[0]
|
|
result_text = result_text.strip()
|
|
|
|
try:
|
|
result = json.loads(result_text)
|
|
print("✅ 提取成功!")
|
|
except json.JSONDecodeError as e:
|
|
print(f"⚠️ JSON 解析失败: {e}")
|
|
result = {"error": "JSON解析失败", "raw": result_text[:500]}
|
|
|
|
# 6. 保存结果
|
|
if output_path:
|
|
os.makedirs(os.path.dirname(output_path), exist_ok=True)
|
|
with open(output_path, 'w', encoding='utf-8') as f:
|
|
json.dump(result, f, ensure_ascii=False, indent=2)
|
|
print(f"💾 结果已保存: {output_path}")
|
|
|
|
return result
|
|
|
|
# 使用示例
|
|
if __name__ == "__main__":
|
|
pdf_path = "paper/ceshi/1pian/苦荞SPL2019BMCplantbios12870-019-1916-6.pdf"
|
|
output_path = "paper/ceshi/1pian/extracted/v2_result.json"
|
|
|
|
result = extract_from_pdf(pdf_path, output_path)
|
|
|
|
# 打印统计
|
|
if "error" not in result:
|
|
entities = result.get("entities", [])
|
|
relations = result.get("relations", [])
|
|
print(f"\n📊 提取统计:")
|
|
print(f" 实体数量: {len(entities)}")
|
|
print(f" 关系数量: {len(relations)}")
|
|
```
|
|
|
|
### 2.2 运行代码
|
|
|
|
```bash
|
|
# 设置 API 密钥
|
|
export DASHSCOPE_API_KEY="your-api-key"
|
|
|
|
# 运行提取
|
|
python3 run_v2_extraction.py
|
|
```
|
|
|
|
---
|
|
|
|
## 方法三: 使用 Jupyter Notebook
|
|
|
|
### 3.1 创建 Notebook
|
|
|
|
```python
|
|
# Cell 1: 导入依赖
|
|
import os
|
|
import json
|
|
from openai import OpenAI
|
|
import fitz
|
|
|
|
# Cell 2: 配置
|
|
API_KEY = os.getenv("DASHSCOPE_API_KEY")
|
|
MODEL = "kimi-k2.5"
|
|
|
|
# Cell 3: 读取 PDF
|
|
pdf_path = "paper/ceshi/1pian/苦荞SPL2019BMCplantbios12870-019-1916-6.pdf"
|
|
doc = fitz.open(pdf_path)
|
|
text = ""
|
|
for page in doc:
|
|
text += page.get_text("text") + "\n"
|
|
doc.close()
|
|
print(f"文本长度: {len(text)} 字符")
|
|
|
|
# Cell 4: 加载 V2 提示词
|
|
with open("prompts/extraction_prompt_v2.md", 'r') as f:
|
|
prompt_template = f.read()
|
|
|
|
user_prompt = prompt_template.replace("{chunk_text}", text[:120000])
|
|
|
|
# Cell 5: 调用 API
|
|
client = OpenAI(api_key=API_KEY, base_url="https://coding.dashscope.aliyuncs.com/v1")
|
|
|
|
response = client.chat.completions.create(
|
|
model=f"bailian/{MODEL}",
|
|
messages=[
|
|
{"role": "system", "content": "You are a specialized bioinformatics paper information extraction expert."},
|
|
{"role": "user", "content": user_prompt}
|
|
],
|
|
temperature=0.1,
|
|
max_tokens=6000
|
|
)
|
|
|
|
# Cell 6: 解析结果
|
|
result_text = response.choices[0].message.content
|
|
result = json.loads(result_text) # 可能需要清理
|
|
|
|
# Cell 7: 查看结果
|
|
print(f"实体数量: {len(result.get('entities', []))}")
|
|
print(f"关系数量: {len(result.get('relations', []))}")
|
|
|
|
# Cell 8: 保存
|
|
with open("v2_result.json", 'w') as f:
|
|
json.dump(result, f, ensure_ascii=False, indent=2)
|
|
```
|
|
|
|
---
|
|
|
|
## 验证提取结果
|
|
|
|
### 检查实体
|
|
|
|
```python
|
|
import json
|
|
|
|
with open("paper/ceshi/1pian/extracted/v2_result.json", 'r') as f:
|
|
result = json.load(f)
|
|
|
|
entities = result.get("entities", [])
|
|
print(f"实体总数: {len(entities)}")
|
|
|
|
# 按类型分组
|
|
from collections import Counter
|
|
type_counts = Counter(e.get("type", "unknown") for e in entities)
|
|
print("\n实体类型分布:")
|
|
for t, c in type_counts.most_common():
|
|
print(f" {t}: {c}")
|
|
|
|
# 显示前5个实体
|
|
print("\n前5个实体:")
|
|
for e in entities[:5]:
|
|
print(f" - {e.get('display_name', e.get('name'))}")
|
|
```
|
|
|
|
### 检查关系
|
|
|
|
```python
|
|
relations = result.get("relations", [])
|
|
print(f"关系总数: {len(relations)}")
|
|
|
|
# 按类型分组
|
|
rel_counts = Counter(r.get("relationship", "unknown") for r in relations)
|
|
print("\n关系类型分布 (Top 10):")
|
|
for r, c in rel_counts.most_common(10):
|
|
print(f" {r}: {c}")
|
|
|
|
# 显示前5个关系
|
|
print("\n前5个关系:")
|
|
for r in relations[:5]:
|
|
print(f" [{r.get('source_type')}] {r.get('source')}")
|
|
print(f" {r.get('relationship')}")
|
|
print(f" [{r.get('target_type')}] {r.get('target')}")
|
|
```
|
|
|
|
---
|
|
|
|
## 常见问题
|
|
|
|
### Q1: API 密钥未设置
|
|
|
|
**错误**: `未设置API密钥: DASHSCOPE_API_KEY`
|
|
|
|
**解决**:
|
|
```bash
|
|
export DASHSCOPE_API_KEY="your-api-key"
|
|
# 或在代码中设置
|
|
os.environ["DASHSCOPE_API_KEY"] = "your-api-key"
|
|
```
|
|
|
|
### Q2: PyMuPDF 未安装
|
|
|
|
**错误**: `No module named 'fitz'`
|
|
|
|
**解决**:
|
|
```bash
|
|
pip install PyMuPDF
|
|
```
|
|
|
|
### Q3: JSON 解析失败
|
|
|
|
**错误**: `JSON解析失败`
|
|
|
|
**解决**: 检查并清理返回的文本
|
|
```python
|
|
import re
|
|
# 移除控制字符
|
|
text = re.sub(r'[\x00-\x08\x0b-\x0c\x0e-\x1f]', '', text)
|
|
# 提取 JSON 块
|
|
if "```json" in text:
|
|
text = text.split("```json")[1]
|
|
if "```" in text:
|
|
text = text.split("```")[0]
|
|
```
|
|
|
|
### Q4: Token 超限
|
|
|
|
**错误**: `max_tokens 超限`
|
|
|
|
**解决**: 截断 PDF 文本
|
|
```python
|
|
# 只取前 120000 字符
|
|
text = text[:120000]
|
|
```
|
|
|
|
### Q5: 提取结果为空
|
|
|
|
**检查**:
|
|
1. PDF 是否可读
|
|
2. 提示词是否正确加载
|
|
3. API 调用是否成功
|
|
4. 返回内容是否包含 JSON
|
|
|
|
---
|
|
|
|
## 高级用法
|
|
|
|
### 批量处理多篇论文
|
|
|
|
```python
|
|
import os
|
|
import glob
|
|
from pathlib import Path
|
|
|
|
def batch_extract(pdf_dir: str, output_dir: str):
|
|
"""批量提取多篇论文"""
|
|
pdf_files = glob.glob(f"{pdf_dir}/*.pdf")
|
|
|
|
for pdf_path in pdf_files:
|
|
filename = Path(pdf_path).stem
|
|
output_path = f"{output_dir}/{filename}_v2.json"
|
|
|
|
print(f"\n{'='*60}")
|
|
print(f"处理: {filename}")
|
|
|
|
try:
|
|
result = extract_from_pdf(pdf_path, output_path)
|
|
|
|
if "error" not in result:
|
|
entities = len(result.get("entities", []))
|
|
relations = len(result.get("relations", []))
|
|
print(f"✅ 成功: {entities} 实体, {relations} 关系")
|
|
else:
|
|
print(f"❌ 失败: {result.get('error')}")
|
|
|
|
except Exception as e:
|
|
print(f"❌ 错误: {e}")
|
|
|
|
# 使用
|
|
batch_extract(
|
|
"paper/ceshi/1pian",
|
|
"paper/ceshi/1pian/extracted"
|
|
)
|
|
```
|
|
|
|
### 对比 V1 和 V2 结果
|
|
|
|
```python
|
|
def compare_results(v1_path: str, v2_path: str):
|
|
"""对比 V1 和 V2 提取结果"""
|
|
|
|
with open(v1_path) as f:
|
|
v1 = json.load(f)
|
|
with open(v2_path) as f:
|
|
v2 = json.load(f)
|
|
|
|
# 统计
|
|
v1_entities = len(v1.get("entities", []))
|
|
v2_entities = len(v2.get("entities", []))
|
|
v1_relations = len(v1.get("relations", []))
|
|
v2_relations = len(v2.get("relations", []))
|
|
|
|
print("V1 vs V2 对比:")
|
|
print(f" 实体: {v1_entities} → {v2_entities} ({v2_entities/v1_entities:.1f}x)")
|
|
print(f" 关系: {v1_relations} → {v2_relations} (提升: {v2_relations - v1_relations})")
|
|
```
|
|
|
|
---
|
|
|
|
## 最佳实践
|
|
|
|
### 1. 预处理 PDF
|
|
|
|
```python
|
|
# 清理 PDF 文本
|
|
def clean_pdf_text(text: str) -> str:
|
|
"""清理 PDF 提取的文本"""
|
|
# 移除多余空白
|
|
text = re.sub(r'\n+', '\n', text)
|
|
# 移除页眉页脚
|
|
lines = [l for l in text.split('\n') if not l.strip().isdigit()]
|
|
return '\n'.join(lines)
|
|
```
|
|
|
|
### 2. 后处理结果
|
|
|
|
```python
|
|
def post_process(result: dict) -> dict:
|
|
"""后处理提取结果"""
|
|
# 去重实体
|
|
entities = result.get("entities", [])
|
|
seen = set()
|
|
unique_entities = []
|
|
for e in entities:
|
|
key = (e.get("name"), e.get("type"))
|
|
if key not in seen:
|
|
seen.add(key)
|
|
unique_entities.append(e)
|
|
result["entities"] = unique_entities
|
|
|
|
# 去重关系
|
|
relations = result.get("relations", [])
|
|
seen = set()
|
|
unique_relations = []
|
|
for r in relations:
|
|
key = (r.get("source"), r.get("relationship"), r.get("target"))
|
|
if key not in seen:
|
|
seen.add(key)
|
|
unique_relations.append(r)
|
|
result["relations"] = unique_relations
|
|
|
|
return result
|
|
```
|
|
|
|
### 3. 质量检查
|
|
|
|
```python
|
|
def quality_check(result: dict) -> dict:
|
|
"""检查提取质量"""
|
|
issues = []
|
|
|
|
# 检查实体
|
|
entities = result.get("entities", [])
|
|
if len(entities) < 10:
|
|
issues.append(f"实体数量过少: {len(entities)}")
|
|
|
|
for e in entities:
|
|
if not e.get("type"):
|
|
issues.append(f"实体缺少类型: {e.get('name')}")
|
|
if not e.get("display_name"):
|
|
issues.append(f"实体缺少display_name: {e.get('name')}")
|
|
|
|
# 检查关系
|
|
relations = result.get("relations", [])
|
|
if len(relations) < 5:
|
|
issues.append(f"关系数量过少: {len(relations)}")
|
|
|
|
for r in relations:
|
|
if not r.get("source_type"):
|
|
issues.append(f"关系缺少source_type")
|
|
if not r.get("target_type"):
|
|
issues.append(f"关系缺少target_type")
|
|
|
|
return {
|
|
"quality_score": max(0, 100 - len(issues) * 10),
|
|
"issues": issues
|
|
}
|
|
```
|
|
|
|
---
|
|
|
|
## 总结
|
|
|
|
### 最简单的使用方式
|
|
|
|
```bash
|
|
# 1. 设置密钥
|
|
export DASHSCOPE_API_KEY="your-key"
|
|
|
|
# 2. 运行提取
|
|
python3 scripts/extract_structured.py \
|
|
paper/ceshi/1pian/你的论文.pdf \
|
|
--output paper/ceshi/1pian/extracted/v2_result.json
|
|
|
|
# 3. 查看结果
|
|
cat paper/ceshi/1pian/extracted/v2_result.json | jq '.entities | length'
|
|
cat paper/ceshi/1pian/extracted/v2_result.json | jq '.relations | length'
|
|
```
|
|
|
|
### 关键文件
|
|
|
|
| 文件 | 路径 | 说明 |
|
|
|------|------|------|
|
|
| V2 提示词 | `prompts/extraction_prompt_v2.md` | PlantConnectome优化版 |
|
|
| 提取脚本 | `scripts/extract_structured.py` | 主提取程序 |
|
|
| 使用指南 | `docs/V2_USAGE_GUIDE.md` | 本文档 |
|
|
|
|
---
|
|
|
|
*文档版本: v1.0*
|
|
*更新日期: 2026-03-18*
|