#!/usr/bin/env python3 """ 规则验证层 - 验证LLM提取结果的准确性 原则：实事求是，过滤编造，标注不确定 """ import json import re from typing import Dict, List, Set, Tuple class ExtractionValidator: """LLM提取结果验证器""" def __init__(self, gene_list_path: str = None): """ 初始化验证器 :param gene_list_path: 附表基因列表路径，如果有的话 """ self.known_genes: Set[str] = set() self.known_groups: Set[str] = { 'Group_I', 'Group_II', 'Group_III', 'Group_IV', 'Group_V', 'Group_VI', 'Group_VII', 'Group_VIII' } self.known_domains: Set[str] = {'SBP', 'ANK'} if gene_list_path: self._load_gene_list(gene_list_path) def _load_gene_list(self, path: str): """加载附表中的基因列表作为ground truth""" try: with open(path, 'r') as f: data = json.load(f) for gene in data: symbol = gene.get('symbol') if symbol: self.known_genes.add(symbol) print(f"✓ 加载附表基因列表: {len(self.known_genes)} 个基因") except Exception as e: print(f"⚠ 无法加载基因列表: {e}") def extract_genes_from_text(self, text: str) -> Set[str]: """从原文中提取所有基因名（正则）""" # 匹配 FtSPL1, FtSPL2, ... FtSPL99 pattern = r'FtSPL\d+' return set(re.findall(pattern, text)) def validate_entity(self, entity: Dict, source_text: str = None) -> Tuple[bool, str]: """ 验证单个实体 :return: (是否有效, 原因) """ entity_id = entity.get('id', '') entity_type = entity.get('type', '') # 1. 验证基因实体 if entity_type == 'gene': # 检查格式 if not re.match(r'^FtSPL\d+$', entity_id): return False, f"基因ID格式错误: {entity_id}" # 如果有附表，检查是否在附表中 if self.known_genes and entity_id not in self.known_genes: return False, f"基因不在附表中: {entity_id}" # 如果有原文，检查是否在原文中 if source_text: text_genes = self.extract_genes_from_text(source_text) if entity_id not in text_genes: return False, f"基因不在原文中: {entity_id}" return True, "有效" # 2. 验证组实体 elif entity_type == 'gene_subfamily': if entity_id not in self.known_groups: return False, f"未知的组: {entity_id}" return True, "有效" # 3. 验证domain实体 elif entity_type == 'protein_domain': if entity_id not in self.known_domains: return False, f"未知的domain: {entity_id}" return True, "有效" # 其他类型暂不验证 return True, "未验证类型" def validate_relation(self, relation: Dict, valid_entities: Set[str]) -> Tuple[bool, str]: """ 验证关系 :return: (是否有效, 原因) """ source = relation.get('source', '') target = relation.get('target', '') rel_type = relation.get('type', '') # 1. 检查源实体是否存在 if source not in valid_entities: return False, f"源实体无效或不存在: {source}" # 2. 检查目标实体是否存在 if target not in valid_entities: return False, f"目标实体无效或不存在: {target}" # 3. 检查关系类型是否合理 valid_relations = { 'classified_into', 'member_of', 'contains', 'expressed_in', 'upregulated_in', 'downregulated_in', 'involved_in', 'belongs_to' } if rel_type not in valid_relations: return False, f"未知的关系类型: {rel_type}" return True, "有效" def validate_extraction(self, extraction_result: Dict, source_text: str = None) -> Dict: """ 验证完整的提取结果 """ validated = { "metadata": { "original_entities": len(extraction_result.get('entities', [])), "original_relations": len(extraction_result.get('relations', [])), "validation_time": "2026-03-30" }, "entities": { "valid": [], "invalid": [] }, "relations": { "valid": [], "invalid": [] }, "statistics": {} } # 验证实体 valid_entity_ids = set() for entity in extraction_result.get('entities', []): is_valid, reason = self.validate_entity(entity, source_text) entity['_validation'] = { "is_valid": is_valid, "reason": reason } if is_valid: validated["entities"]["valid"].append(entity) valid_entity_ids.add(entity.get('id')) else: validated["entities"]["invalid"].append(entity) # 验证关系（只验证指向有效实体的关系） for relation in extraction_result.get('relations', []): is_valid, reason = self.validate_relation(relation, valid_entity_ids) relation['_validation'] = { "is_valid": is_valid, "reason": reason } if is_valid: validated["relations"]["valid"].append(relation) else: validated["relations"]["invalid"].append(relation) # 统计 validated["statistics"] = { "entity_precision": len(validated["entities"]["valid"]) / validated["metadata"]["original_entities"] if validated["metadata"]["original_entities"] > 0 else 0, "relation_precision": len(validated["relations"]["valid"]) / validated["metadata"]["original_relations"] if validated["metadata"]["original_relations"] > 0 else 0, "invalid_entities": len(validated["entities"]["invalid"]), "invalid_relations": len(validated["relations"]["invalid"]) } return validated def generate_clean_output(self, validated: Dict) -> Dict: """ 生成清理后的输出（只包含验证通过的内容） """ return { "metadata": { "extraction_method": "LLM + Rule Validation", "validation_applied": True, "original_entities": validated["metadata"]["original_entities"], "original_relations": validated["metadata"]["original_relations"], "filtered_entities": len(validated["entities"]["invalid"]), "filtered_relations": len(validated["relations"]["invalid"]) }, "entities": validated["entities"]["valid"], "relations": validated["relations"]["valid"], "filtered": { "entities": validated["entities"]["invalid"], "relations": validated["relations"]["invalid"] } } def main(): """测试验证器""" import sys # 初始化验证器 validator = ExtractionValidator( gene_list_path="ftspl_gene_list.json" ) # 加载LLM提取结果 input_path = "evaluation/auto_results/pmid31286919_sections.json" try: with open(input_path, 'r') as f: extraction_result = json.load(f) except Exception as e: print(f"✗ 无法加载提取结果: {e}") sys.exit(1) print(f"\n{'='*70}") print("LLM提取结果验证") print(f"{'='*70}") # 执行验证 validated = validator.validate_extraction(extraction_result) # 显示统计 print(f"\n📊 验证统计:") print(f" 原始实体数: {validated['metadata']['original_entities']}") print(f" 有效实体: {len(validated['entities']['valid'])} ({validated['statistics']['entity_precision']*100:.1f}%)") print(f" 无效实体: {validated['statistics']['invalid_entities']}") print(f" 原始关系数: {validated['metadata']['original_relations']}") print(f" 有效关系: {len(validated['relations']['valid'])} ({validated['statistics']['relation_precision']*100:.1f}%)") print(f" 无效关系: {validated['statistics']['invalid_relations']}") # 显示被过滤的内容 if validated['entities']['invalid']: print(f"\n⚠️ 被过滤的实体:") for e in validated['entities']['invalid']: print(f" - {e.get('id')} ({e.get('type')}): {e['_validation']['reason']}") if validated['relations']['invalid']: print(f"\n⚠️ 被过滤的关系:") for r in validated['relations']['invalid']: print(f" - {r.get('source')} --{r.get('type')}--> {r.get('target')}: {r['_validation']['reason']}") # 生成清理后的输出 clean_output = validator.generate_clean_output(validated) # 保存结果 output_path = "evaluation/auto_results/pmid31286919_sections_validated.json" with open(output_path, 'w') as f: json.dump(clean_output, f, indent=2, ensure_ascii=False) print(f"\n✓ 验证结果已保存到: {output_path}") print(f"\n{'='*70}") print("清理后统计:") print(f" 实体: {len(clean_output['entities'])} (过滤了 {clean_output['metadata']['filtered_entities']} 个)") print(f" 关系: {len(clean_output['relations'])} (过滤了 {clean_output['metadata']['filtered_relations']} 个)") print(f"{'='*70}") if __name__ == "__main__": main()