PlantAI-rag-new/agent/legacy-rag/scripts/utils/fetch_supplementary_data.py

#!/usr/bin/env python3
"""
自动获取补充材料方案

可行性分析:
1. 通过DOI/PMID访问期刊网站
2. 定位补充材料链接
3. 下载并解析

支持的来源:
- NCBI PMC (PubMed Central)
- 期刊官网 (BMC, Springer等)
- Figshare / Zenodo等数据仓库
"""

import json
from pathlib import Path
from typing import Dict, List, Optional


class SupplementaryDataFetcher:
    """补充材料获取器"""
    
    def __init__(self):
        self.supported_journals = {
            "bmc": "https://www.biomedcentral.com",
            "springer": "https://link.springer.com",
            "ncbi": "https://www.ncbi.nlm.nih.gov/pmc",
        }
    
    def analyze_paper(self, pmid: str, pmcid: str, doi: str) -> Dict:
        """
        分析论文补充材料可用性
        
        返回:
        - available: 是否可获取
        - sources: 可能的来源列表
        - files: 补充材料文件列表
        - confidence: 获取成功率估计
        """
        
        result = {
            "pmid": pmid,
            "pmcid": pmcid,
            "doi": doi,
            "available": False,
            "sources": [],
            "files": [],
            "confidence": 0.0,
            "notes": []
        }
        
        # 1. 检查PMC可用性
        if pmcid:
            result["sources"].append({
                "name": "NCBI PMC",
                "url": f"https://www.ncbi.nlm.nih.gov/pmc/articles/{pmcid}/",
                "type": "full_text",
                "access": "open"
            })
            result["confidence"] += 0.3
        
        # 2. 检查DOI可用性
        if doi:
            result["sources"].append({
                "name": "DOI Resolver",
                "url": f"https://doi.org/{doi}",
                "type": "journal_page",
                "access": "depends"
            })
            result["confidence"] += 0.3
        
        # 3. 预测补充材料类型
        result["predicted_files"] = self._predict_files(doi)
        
        # 4. 评估获取难度
        if "bmc" in doi.lower() or "biomedcentral" in doi.lower():
            result["notes"].append("BMC期刊通常提供开放获取的补充材料")
            result["confidence"] += 0.3
        elif "springer" in doi.lower():
            result["notes"].append("Springer可能需要机构访问")
            result["confidence"] += 0.1
        
        result["available"] = result["confidence"] > 0.5
        
        return result
    
    def _predict_files(self, doi: str) -> List[Dict]:
        """预测可能的补充材料文件"""
        
        # 基因家族论文常见的补充材料
        common_files = [
            {"name": "Additional file 1", "type": "xlsx", "content": "基因列表和分组信息"},
            {"name": "Additional file 2", "type": "xlsx", "content": "引物序列"},
            {"name": "Additional file 3", "type": "pdf/tif", "content": "系统发育树"},
            {"name": "Additional file 4", "type": "xlsx", "content": "内含子分析"},
            {"name": "Additional file 5", "type": "xlsx", "content": "启动子分析"},
        ]
        
        return common_files
    
    def fetch_strategy(self, paper_info: Dict) -> Dict:
        """
        制定获取策略
        
        对于番茄P450 (PMID:30732561):
        - 优先从BMC官网获取
        - Additional file 1包含基因列表
        - Additional file 3包含系统发育树
        """
        
        pmid = paper_info.get("pmid")
        pmcid = paper_info.get("pmcid")
        doi = paper_info.get("doi")
        
        strategy = {
            "paper": f"PMID:{pmid}",
            "priority": [],
            "expected_files": [],
            "fallback": []
        }
        
        # 策略1: BMC官网 (对于BMC Genomics)
        if doi and "1186" in doi:
            strategy["priority"].append({
                "step": 1,
                "source": "BMC官网",
                "url": f"https://bmcgenomics.biomedcentral.com/articles/10.1186/s12864-019-5483-x",
                "action": "爬取补充材料链接",
                "expected": ["Additional file 1", "Additional file 3"]
            })
        
        # 策略2: PMC FTP
        if pmcid:
            strategy["priority"].append({
                "step": 2,
                "source": "PMC FTP",
                "url": f"ftp://ftp.ncbi.nlm.nih.gov/pub/pmc/{pmcid[-2:]}/{pmcid}/",
                "action": "FTP下载补充材料",
                "expected": ["*.xlsx", "*.pdf"]
            })
        
        # 策略3: Europe PMC
        strategy["fallback"].append({
            "source": "Europe PMC",
            "url": f"https://europepmc.org/article/PMC/{pmid}",
            "action": "备用来源"
        })
        
        return strategy


def main():
    """演示"""
    fetcher = SupplementaryDataFetcher()
    
    # 番茄P450
    tomato_p450 = {
        "pmid": "30732561",
        "pmcid": "PMC6367802",
        "doi": "10.1186/s12864-019-5483-x"
    }
    
    print("=" * 80)
    print("补充材料自动获取可行性分析")
    print("=" * 80)
    print()
    
    # 分析可用性
    analysis = fetcher.analyze_paper(**tomato_p450)
    print(f"📄 PMID: {analysis['pmid']}")
    print(f"   可用性: {'✅ 可获取' if analysis['available'] else '❌ 困难'}")
    print(f"   置信度: {analysis['confidence']:.0%}")
    print()
    
    print("可能的来源:")
    for src in analysis['sources']:
        print(f"  • {src['name']}: {src['url'][:60]}...")
    print()
    
    print("预测的文件:")
    for f in analysis['predicted_files']:
        print(f"  • {f['name']}.{f['type']}: {f['content']}")
    print()
    
    # 获取策略
    print("=" * 80)
    print("获取策略")
    print("=" * 80)
    strategy = fetcher.fetch_strategy(tomato_p450)
    
    for step in strategy['priority']:
        print(f"\n步骤 {step['step']}: {step['source']}")
        print(f"  URL: {step['url']}")
        print(f"  操作: {step['action']}")
        print(f"  期望文件: {', '.join(step['expected'])}")
    
    print()
    print("=" * 80)
    print("技术实现建议")
    print("=" * 80)
    print("""
1. 爬虫方案:
   - 使用selenium/playwright模拟浏览器
   - 处理JavaScript动态加载
   - 绕过反爬虫机制

2. API方案:
   - Europe PMC API: https://www.ebi.ac.uk/europepmc/webservices/rest/
   - 可获取元数据和部分全文

3. 混合方案:
   - 先尝试API获取
   - 失败时再用爬虫
   - 缓存结果避免重复访问

4. 限制:
   - 部分期刊需要订阅
   - 反爬虫机制
   - 文件格式多样(xlsx, pdf, doc)
   - 需要解析不同格式的表格
""")


if __name__ == "__main__":
    main()