1739 lines
72 KiB
Python
1739 lines
72 KiB
Python
|
"""Bridge saved literature methodology plans into bbav5 job state.
|
||
|
|
|
||
|
|
This module deliberately does not execute stages. It binds a reviewed plan to a
|
||
|
|
job, checks whether user data/config are present, and prepares params for the
|
||
|
|
existing MAS planner.
|
||
|
|
"""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import csv
|
||
|
|
import copy
|
||
|
|
import json
|
||
|
|
import re
|
||
|
|
import shutil
|
||
|
|
from dataclasses import asdict, dataclass
|
||
|
|
from datetime import datetime
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import Any
|
||
|
|
|
||
|
|
|
||
|
|
WHOLE_GENOME_EXTENSIONS = ('.fa', '.fasta', '.faa', '.fna', '.fas', '.pep', '.cds')
|
||
|
|
GFF_EXTENSIONS = ('.gff', '.gff3', '.gtf')
|
||
|
|
FIXED_METHODOLOGY_STAGE_ORDER = [
|
||
|
|
'genome_prepare',
|
||
|
|
'identify',
|
||
|
|
'phylogeny',
|
||
|
|
'motif',
|
||
|
|
'domain',
|
||
|
|
'structure',
|
||
|
|
'promoter',
|
||
|
|
'combo',
|
||
|
|
'reference_classification',
|
||
|
|
'subfamily_confirm',
|
||
|
|
'synteny',
|
||
|
|
'interspecies_synteny',
|
||
|
|
'karyotype',
|
||
|
|
'ppi',
|
||
|
|
'expression',
|
||
|
|
'pep',
|
||
|
|
'rnaseq_qc',
|
||
|
|
'rnaseq_align',
|
||
|
|
'rnaseq_quant',
|
||
|
|
'rnaseq_sample_qc',
|
||
|
|
'rnaseq_de',
|
||
|
|
'rnaseq_go_enrich',
|
||
|
|
'rnaseq_kegg_enrich',
|
||
|
|
'rnaseq_fgsea',
|
||
|
|
'rnaseq_wgcna',
|
||
|
|
]
|
||
|
|
METHODOLOGY_RAW_RNASEQ_STAGES = {
|
||
|
|
'rnaseq_qc', 'rnaseq_align', 'rnaseq_quant', 'rnaseq_sample_qc', 'rnaseq_de',
|
||
|
|
'rnaseq_go_enrich', 'rnaseq_kegg_enrich', 'rnaseq_fgsea', 'rnaseq_wgcna',
|
||
|
|
}
|
||
|
|
METHODOLOGY_COMPLETION_PARADIGM_VERSION = 'methodology_evidence_completion.v2'
|
||
|
|
COMMON_PLANT_SPECIES_ALIASES = {
|
||
|
|
'A. thaliana': 'Arabidopsis thaliana',
|
||
|
|
'B. vulgaris': 'Beta vulgaris',
|
||
|
|
'G. max': 'Glycine max',
|
||
|
|
'O. sativa': 'Oryza sativa',
|
||
|
|
'S. lycopersicum': 'Solanum lycopersicum',
|
||
|
|
'V. vinifera': 'Vitis vinifera',
|
||
|
|
'Z. mays': 'Zea mays',
|
||
|
|
}
|
||
|
|
COMMON_SPECIES_NAME_ALIASES = {
|
||
|
|
'Fagopyrum tataricum': ['tartary buckwheat'],
|
||
|
|
'Arabidopsis thaliana': ['arabidopsis'],
|
||
|
|
'Oryza sativa': ['rice'],
|
||
|
|
}
|
||
|
|
METHODOLOGY_STAGE_ALIASES = {
|
||
|
|
'gene_family_identification': 'identify',
|
||
|
|
'identification': 'identify',
|
||
|
|
'identify': 'identify',
|
||
|
|
'phylogenetic_analysis': 'phylogeny',
|
||
|
|
'phylogeny': 'phylogeny',
|
||
|
|
'motif_analysis': 'motif',
|
||
|
|
'motif': 'motif',
|
||
|
|
'domain_analysis': 'domain',
|
||
|
|
'conserved_domain': 'domain',
|
||
|
|
'domain': 'domain',
|
||
|
|
'gene_structure': 'structure',
|
||
|
|
'intron_exon_structure': 'structure',
|
||
|
|
'structure': 'structure',
|
||
|
|
'promoter_analysis': 'promoter',
|
||
|
|
'cis_element': 'promoter',
|
||
|
|
'synteny': 'synteny',
|
||
|
|
'collinearity': 'synteny',
|
||
|
|
'chromosome_location': 'karyotype',
|
||
|
|
'chromosomal_location': 'karyotype',
|
||
|
|
'karyotype': 'karyotype',
|
||
|
|
'expression_analysis': 'expression',
|
||
|
|
'expression': 'expression',
|
||
|
|
'protein_properties': 'pep',
|
||
|
|
'pep': 'pep',
|
||
|
|
'ppi': 'ppi',
|
||
|
|
}
|
||
|
|
ACCESSION_PATTERN = re.compile(r'\b(?:GSE|GSM|SRP|SRR|SRX|SRS|PRJNA|PRJEB|ERP|ERR|ERX|ERS|DRP|DRR|DRX|DRS)\d+\b', re.I)
|
||
|
|
REFERENCE_DB_PATTERN = re.compile(r'\b(?:UniProt|TAIR|Pfam|InterPro|PlantTFDB|Ensembl|Phytozome)\b', re.I)
|
||
|
|
FIGURE_TABLE_PATTERN = re.compile(r'\b((?:Supplementary\s+)?(?:Fig(?:ure)?|Table)\.?\s+[A-Za-z]?\d+[A-Za-z0-9.-]*)\s*[:. ]\s*([^.\n]{0,220})', re.I)
|
||
|
|
ACCESSION_CONTEXT_PATTERN = re.compile(r'(GEO|SRA|BioProject|ENA|NCBI|RNA[- ]?seq|transcriptome|expression|raw\s+data|downloaded|deposited|accession)', re.I)
|
||
|
|
RAW_RNASEQ_CONTEXT_PATTERN = re.compile(r'(RNA[- ]?seq|raw\s+data|SRA|BioProject|transcriptome|sequencing)', re.I)
|
||
|
|
EXPRESSION_MATRIX_CONTEXT_PATTERN = re.compile(r'(expression\s+matrix|FPKM|TPM|RPKM|counts?\s+matrix|normalized\s+expression)', re.I)
|
||
|
|
|
||
|
|
|
||
|
|
@dataclass(frozen=True)
|
||
|
|
class EvidenceCandidate:
|
||
|
|
object_type: str
|
||
|
|
value: str
|
||
|
|
role: str = ''
|
||
|
|
target_field: str = ''
|
||
|
|
source_file: str = ''
|
||
|
|
source_field: str = ''
|
||
|
|
evidence_text: str = ''
|
||
|
|
confidence_rule: str = ''
|
||
|
|
guardrail_status: str = 'accepted'
|
||
|
|
|
||
|
|
def to_record(self) -> dict[str, Any]:
|
||
|
|
return asdict(self)
|
||
|
|
|
||
|
|
|
||
|
|
def load_methodology_plan(path: str | Path) -> dict[str, Any]:
|
||
|
|
plan_path = Path(path)
|
||
|
|
if not plan_path.exists():
|
||
|
|
raise FileNotFoundError(f'Methodology plan not found: {plan_path}')
|
||
|
|
data = json.loads(plan_path.read_text(encoding='utf-8'))
|
||
|
|
if not isinstance(data, dict):
|
||
|
|
raise ValueError(f'Methodology plan must be a JSON object: {plan_path}')
|
||
|
|
return data
|
||
|
|
|
||
|
|
|
||
|
|
def bound_methodology_plan_path(input_root: Path, job_id: str) -> Path:
|
||
|
|
return Path(input_root) / job_id / 'methodology_plan.json'
|
||
|
|
|
||
|
|
|
||
|
|
def bind_methodology_plan(
|
||
|
|
plan: dict[str, Any],
|
||
|
|
*,
|
||
|
|
job_id: str,
|
||
|
|
input_root: Path,
|
||
|
|
workspace_root: Path,
|
||
|
|
source: str = '',
|
||
|
|
) -> dict[str, Any]:
|
||
|
|
if not job_id:
|
||
|
|
raise ValueError('job_id is required')
|
||
|
|
input_dir = Path(input_root) / job_id
|
||
|
|
final_dir = Path(workspace_root) / job_id / 'final'
|
||
|
|
input_dir.mkdir(parents=True, exist_ok=True)
|
||
|
|
final_dir.mkdir(parents=True, exist_ok=True)
|
||
|
|
plan = complete_methodology_plan_for_execution(plan, source=source)
|
||
|
|
plan_path = input_dir / 'methodology_plan.json'
|
||
|
|
plan_path.write_text(json.dumps(plan, ensure_ascii=False, indent=2), encoding='utf-8')
|
||
|
|
binding = {
|
||
|
|
'job_id': job_id,
|
||
|
|
'plan_id': plan.get('plan_id', ''),
|
||
|
|
'paper_id': plan.get('paper_id', ''),
|
||
|
|
'gene_family': plan.get('gene_family', ''),
|
||
|
|
'source': source,
|
||
|
|
'plan_path': str(plan_path),
|
||
|
|
'bound_at': datetime.now().isoformat(timespec='seconds'),
|
||
|
|
'schema_version': plan.get('schema_version', ''),
|
||
|
|
}
|
||
|
|
binding_path = final_dir / 'methodology_plan.binding.json'
|
||
|
|
binding_path.write_text(json.dumps(binding, ensure_ascii=False, indent=2), encoding='utf-8')
|
||
|
|
return {'success': True, 'job_id': job_id, 'plan_path': str(plan_path), 'binding_path': str(binding_path), 'binding': binding}
|
||
|
|
|
||
|
|
|
||
|
|
def bind_methodology_plan_from_path(
|
||
|
|
source_path: str | Path,
|
||
|
|
*,
|
||
|
|
job_id: str,
|
||
|
|
input_root: Path,
|
||
|
|
workspace_root: Path,
|
||
|
|
) -> dict[str, Any]:
|
||
|
|
plan = load_methodology_plan(source_path)
|
||
|
|
result = bind_methodology_plan(plan, job_id=job_id, input_root=input_root, workspace_root=workspace_root, source=str(source_path))
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def methodology_agent_params(plan: dict[str, Any]) -> dict[str, Any]:
|
||
|
|
plan = complete_methodology_plan_for_execution(plan)
|
||
|
|
prefill = plan.get('bbav5_prefill') if isinstance(plan.get('bbav5_prefill'), dict) else {}
|
||
|
|
route_stages = []
|
||
|
|
for route in plan.get('execution_routes') or []:
|
||
|
|
if isinstance(route, dict) and isinstance(route.get('stage_order'), list):
|
||
|
|
route_stages = [str(item) for item in route.get('stage_order') if str(item)]
|
||
|
|
break
|
||
|
|
stages = _fixed_methodology_stage_order(route_stages)
|
||
|
|
gene_family = prefill.get('geneFamily') or plan.get('gene_family') or _infer_gene_family_from_plan(plan)
|
||
|
|
params = {
|
||
|
|
'source': 'methodology',
|
||
|
|
'methodology': True,
|
||
|
|
'methodologyPlanId': plan.get('plan_id', ''),
|
||
|
|
'sourcePaperId': plan.get('paper_id', ''),
|
||
|
|
'geneFamily': gene_family,
|
||
|
|
'targetSpecies': _list(prefill.get('targetSpecies')),
|
||
|
|
'referenceSpecies': _list(prefill.get('referenceSpecies')),
|
||
|
|
'comparativeSpecies': _list(prefill.get('comparativeSpecies')),
|
||
|
|
'identifyBlastReferenceSpecies': _list(prefill.get('identifyBlastReferenceSpecies')),
|
||
|
|
'primaryReferenceSpecies': _list(prefill.get('primaryReferenceSpecies')),
|
||
|
|
'requiredPfams': prefill.get('requiredPfams') or _infer_required_pfams_from_plan(plan),
|
||
|
|
'scopeMode': prefill.get('scopeMode') or _scope_mode(prefill),
|
||
|
|
'requestedMethodologyStages': stages,
|
||
|
|
}
|
||
|
|
params['refSpeciesList'] = _unique([*params['referenceSpecies'], *params['comparativeSpecies']])
|
||
|
|
params['methodology_workflow_contract_version'] = 'methodology_fixed_workflow.v1'
|
||
|
|
params['requiresDataPreparation'] = True
|
||
|
|
params['requiresSubfamilyClassification'] = True
|
||
|
|
if params['targetSpecies']:
|
||
|
|
first = params['targetSpecies'][0]
|
||
|
|
family = params['geneFamily']
|
||
|
|
if first and family:
|
||
|
|
params['prefix'] = f'{first}.{family}' if len(params['targetSpecies']) == 1 else f'Target.{family}'
|
||
|
|
return params
|
||
|
|
|
||
|
|
|
||
|
|
def _fixed_methodology_stage_order(route_stages: list[str]) -> list[str]:
|
||
|
|
route_set = {stage for stage in route_stages if stage and stage != 'external_or_manual'}
|
||
|
|
ordered = []
|
||
|
|
for stage in FIXED_METHODOLOGY_STAGE_ORDER:
|
||
|
|
if stage in route_set:
|
||
|
|
ordered.append(stage)
|
||
|
|
return _unique(ordered)
|
||
|
|
|
||
|
|
|
||
|
|
def complete_methodology_plan_for_execution(plan: dict[str, Any], *, source: str = '') -> dict[str, Any]:
|
||
|
|
"""Complete a methodology plan through one evidence-to-execution paradigm."""
|
||
|
|
if not isinstance(plan, dict):
|
||
|
|
return plan
|
||
|
|
original = copy.deepcopy(plan)
|
||
|
|
candidates = _collect_methodology_evidence_candidates(plan, source=source)
|
||
|
|
completed = complete_methodology_plan_species(plan, source=source)
|
||
|
|
completed = _complete_methodology_required_pfams(completed)
|
||
|
|
completed = _complete_methodology_stage_order(completed)
|
||
|
|
completed = _complete_methodology_dataset_gates(completed, candidates)
|
||
|
|
completed = _complete_methodology_expression_requirements(completed, candidates)
|
||
|
|
completed = _complete_methodology_reproduction_targets(completed, candidates)
|
||
|
|
_attach_methodology_completion_audit(completed, original, candidates, source=source)
|
||
|
|
return completed
|
||
|
|
|
||
|
|
|
||
|
|
def _collect_methodology_evidence_candidates(plan: dict[str, Any], *, source: str = '') -> list[EvidenceCandidate]:
|
||
|
|
candidates: list[EvidenceCandidate] = []
|
||
|
|
candidates.extend(_candidate_records_from_plan(plan))
|
||
|
|
for batch_dir in _methodology_batch_dirs(plan, source=source):
|
||
|
|
candidates.extend(_candidate_records_from_batch_dir(batch_dir))
|
||
|
|
return _unique_candidates(candidates)
|
||
|
|
|
||
|
|
|
||
|
|
def _candidate_records_from_plan(plan: dict[str, Any]) -> list[EvidenceCandidate]:
|
||
|
|
candidates: list[EvidenceCandidate] = []
|
||
|
|
for record in _species_records_from_prefill(plan):
|
||
|
|
candidates.append(EvidenceCandidate(
|
||
|
|
object_type='species',
|
||
|
|
value=_canonical_latin_species(record.get('species_name')),
|
||
|
|
role=_normalize_species_role(record.get('species_role')),
|
||
|
|
target_field='bbav5_prefill',
|
||
|
|
source_field='bbav5_prefill',
|
||
|
|
confidence_rule='prefill_species',
|
||
|
|
))
|
||
|
|
for record in _species_records_from_requirements(plan.get('data_requirements')):
|
||
|
|
candidates.append(EvidenceCandidate(
|
||
|
|
object_type='species',
|
||
|
|
value=_canonical_latin_species(record.get('species_name')),
|
||
|
|
role=_normalize_species_role(record.get('species_role')),
|
||
|
|
target_field='data_requirements',
|
||
|
|
source_field='data_requirements',
|
||
|
|
confidence_rule='existing_requirement_species',
|
||
|
|
))
|
||
|
|
for record in _species_records_from_plan_field(plan.get('species_summary')):
|
||
|
|
candidates.append(EvidenceCandidate(
|
||
|
|
object_type='species',
|
||
|
|
value=_canonical_latin_species(record.get('species_name')),
|
||
|
|
role=_normalize_species_role(record.get('species_role')),
|
||
|
|
target_field='bbav5_prefill',
|
||
|
|
source_field='species_summary',
|
||
|
|
confidence_rule='structured_species_summary',
|
||
|
|
))
|
||
|
|
|
||
|
|
text = _plan_species_text(plan)
|
||
|
|
candidates.extend(_text_evidence_candidates(text, source_field='plan_text'))
|
||
|
|
candidates.extend(_candidate_stage_records_from_plan_value(plan))
|
||
|
|
return [candidate for candidate in candidates if candidate.value]
|
||
|
|
|
||
|
|
|
||
|
|
def _candidate_stage_records_from_plan_value(value: Any, *, source_field: str = 'plan') -> list[EvidenceCandidate]:
|
||
|
|
candidates: list[EvidenceCandidate] = []
|
||
|
|
|
||
|
|
def walk(item: Any, field: str = source_field, context: str = '') -> None:
|
||
|
|
if isinstance(item, dict):
|
||
|
|
text = _row_text(item)
|
||
|
|
for key, child in item.items():
|
||
|
|
if key in {'bbav5_stage', 'mapped_bbav5_stage', 'bbav5_stage_id', 'normalized_stage', 'stage'}:
|
||
|
|
stage = _normalize_methodology_stage(child)
|
||
|
|
if stage:
|
||
|
|
candidates.append(EvidenceCandidate(
|
||
|
|
object_type='stage',
|
||
|
|
value=stage,
|
||
|
|
target_field='execution_routes.stage_order',
|
||
|
|
source_field=key,
|
||
|
|
evidence_text=(text or context)[:500],
|
||
|
|
confidence_rule='structured_stage_field',
|
||
|
|
))
|
||
|
|
walk(child, str(key), text or context)
|
||
|
|
elif isinstance(item, list):
|
||
|
|
for child in item:
|
||
|
|
walk(child, field, context)
|
||
|
|
|
||
|
|
walk(value)
|
||
|
|
return candidates
|
||
|
|
|
||
|
|
|
||
|
|
def _candidate_records_from_batch_dir(batch_dir: Path) -> list[EvidenceCandidate]:
|
||
|
|
candidates: list[EvidenceCandidate] = []
|
||
|
|
for csv_name in ('species_summary.csv', 'review_template.csv', 'summary.csv', 'methodology_parameter_audit.csv'):
|
||
|
|
path = batch_dir / csv_name
|
||
|
|
if not path.exists():
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
with path.open('r', encoding='utf-8', newline='') as handle:
|
||
|
|
reader = csv.DictReader(handle)
|
||
|
|
for row in reader:
|
||
|
|
candidates.extend(_candidate_records_from_row(row, source_file=str(path), source_field=csv_name))
|
||
|
|
except Exception:
|
||
|
|
continue
|
||
|
|
for json_name in ('observations.json', 'alignment.json'):
|
||
|
|
path = batch_dir / json_name
|
||
|
|
if not path.exists():
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
data = json.loads(path.read_text(encoding='utf-8'))
|
||
|
|
except Exception:
|
||
|
|
continue
|
||
|
|
candidates.extend(_candidate_records_from_json_value(data, source_file=str(path)))
|
||
|
|
return candidates
|
||
|
|
|
||
|
|
|
||
|
|
def _candidate_records_from_row(row: dict[str, Any], *, source_file: str, source_field: str) -> list[EvidenceCandidate]:
|
||
|
|
text = _row_text(row)
|
||
|
|
candidates = _text_evidence_candidates(text, source_file=source_file, source_field=source_field)
|
||
|
|
species = row.get('species_name') or row.get('species')
|
||
|
|
if species:
|
||
|
|
candidates.append(EvidenceCandidate(
|
||
|
|
object_type='species',
|
||
|
|
value=_canonical_latin_species(species),
|
||
|
|
role=_normalize_species_role(row.get('species_role') or row.get('role')),
|
||
|
|
target_field='bbav5_prefill',
|
||
|
|
source_file=source_file,
|
||
|
|
source_field='species_name',
|
||
|
|
evidence_text=str(row.get('evidence_sentence') or text)[:500],
|
||
|
|
confidence_rule='structured_species_summary',
|
||
|
|
))
|
||
|
|
for key in ('bbav5_stage', 'mapped_bbav5_stage', 'bbav5_stage_id', 'normalized_stage', 'stage'):
|
||
|
|
stage = _normalize_methodology_stage(row.get(key))
|
||
|
|
if stage:
|
||
|
|
candidates.append(EvidenceCandidate(
|
||
|
|
object_type='stage',
|
||
|
|
value=stage,
|
||
|
|
target_field='execution_routes.stage_order',
|
||
|
|
source_file=source_file,
|
||
|
|
source_field=key,
|
||
|
|
evidence_text=text[:500],
|
||
|
|
confidence_rule='structured_stage_field',
|
||
|
|
))
|
||
|
|
return candidates
|
||
|
|
|
||
|
|
|
||
|
|
def _candidate_records_from_json_value(value: Any, *, source_file: str) -> list[EvidenceCandidate]:
|
||
|
|
candidates: list[EvidenceCandidate] = []
|
||
|
|
|
||
|
|
def walk(item: Any, source_field: str = '') -> None:
|
||
|
|
if isinstance(item, dict):
|
||
|
|
text = _row_text(item)
|
||
|
|
candidates.extend(_text_evidence_candidates(text, source_file=source_file, source_field=source_field or 'json_object'))
|
||
|
|
species = item.get('species_name') or item.get('species')
|
||
|
|
if species:
|
||
|
|
candidates.append(EvidenceCandidate(
|
||
|
|
object_type='species',
|
||
|
|
value=_canonical_latin_species(species),
|
||
|
|
role=_normalize_species_role(item.get('species_role') or item.get('role')),
|
||
|
|
target_field='bbav5_prefill',
|
||
|
|
source_file=source_file,
|
||
|
|
source_field='species_name',
|
||
|
|
evidence_text=str(item.get('evidence_sentence') or text)[:500],
|
||
|
|
confidence_rule='structured_json_species',
|
||
|
|
))
|
||
|
|
for key, child in item.items():
|
||
|
|
if key in {'bbav5_stage', 'mapped_bbav5_stage', 'bbav5_stage_id', 'normalized_stage', 'stage'}:
|
||
|
|
stage = _normalize_methodology_stage(child)
|
||
|
|
if stage:
|
||
|
|
candidates.append(EvidenceCandidate(
|
||
|
|
object_type='stage',
|
||
|
|
value=stage,
|
||
|
|
target_field='execution_routes.stage_order',
|
||
|
|
source_file=source_file,
|
||
|
|
source_field=key,
|
||
|
|
evidence_text=text[:500],
|
||
|
|
confidence_rule='structured_stage_field',
|
||
|
|
))
|
||
|
|
walk(child, str(key))
|
||
|
|
elif isinstance(item, list):
|
||
|
|
for child in item:
|
||
|
|
walk(child, source_field)
|
||
|
|
|
||
|
|
walk(value)
|
||
|
|
return candidates
|
||
|
|
|
||
|
|
|
||
|
|
def _text_evidence_candidates(text: str, *, source_file: str = '', source_field: str = '') -> list[EvidenceCandidate]:
|
||
|
|
value = str(text or '')
|
||
|
|
if not value:
|
||
|
|
return []
|
||
|
|
candidates: list[EvidenceCandidate] = []
|
||
|
|
for record in _species_records_from_text(value, default_role='reference_species'):
|
||
|
|
candidates.append(EvidenceCandidate(
|
||
|
|
object_type='species',
|
||
|
|
value=_canonical_latin_species(record.get('species_name')),
|
||
|
|
role='reference',
|
||
|
|
target_field='bbav5_prefill.referenceSpecies',
|
||
|
|
source_file=source_file,
|
||
|
|
source_field=source_field,
|
||
|
|
evidence_text=_evidence_excerpt(value, record.get('species_name')),
|
||
|
|
confidence_rule='alias_or_full_name_in_method_evidence',
|
||
|
|
))
|
||
|
|
for pfam in re.findall(r'PF\d{5}', value):
|
||
|
|
candidates.append(EvidenceCandidate(
|
||
|
|
object_type='pfam',
|
||
|
|
value=pfam,
|
||
|
|
target_field='bbav5_prefill.requiredPfams',
|
||
|
|
source_file=source_file,
|
||
|
|
source_field=source_field,
|
||
|
|
evidence_text=_evidence_excerpt(value, pfam),
|
||
|
|
confidence_rule='pfam_evidence_completion',
|
||
|
|
))
|
||
|
|
for accession in ACCESSION_PATTERN.findall(value):
|
||
|
|
excerpt = _sentence_excerpt(value, accession)
|
||
|
|
accession_role = _classify_accession_context(excerpt)
|
||
|
|
guardrail = 'accepted' if accession_role in {'raw_rnaseq_accession', 'expression_matrix_accession'} else accession_role
|
||
|
|
target_field = 'manual_gates' if accession_role == 'raw_rnaseq_accession' else 'data_requirements' if accession_role == 'expression_matrix_accession' else 'methodology_completion_audit'
|
||
|
|
candidates.append(EvidenceCandidate(
|
||
|
|
object_type='dataset_accession',
|
||
|
|
value=accession.upper(),
|
||
|
|
role=accession_role,
|
||
|
|
target_field=target_field,
|
||
|
|
source_file=source_file,
|
||
|
|
source_field=source_field,
|
||
|
|
evidence_text=excerpt,
|
||
|
|
confidence_rule='accession_with_method_context',
|
||
|
|
guardrail_status=guardrail,
|
||
|
|
))
|
||
|
|
for match in REFERENCE_DB_PATTERN.finditer(value):
|
||
|
|
db_name = match.group(0)
|
||
|
|
candidates.append(EvidenceCandidate(
|
||
|
|
object_type='reference_database',
|
||
|
|
value=db_name,
|
||
|
|
role='reference_database_accession',
|
||
|
|
target_field='methodology_completion_audit',
|
||
|
|
source_file=source_file,
|
||
|
|
source_field=source_field,
|
||
|
|
evidence_text=_sentence_excerpt(value, db_name),
|
||
|
|
confidence_rule='reference_database_source',
|
||
|
|
guardrail_status='audit_only_reference_database',
|
||
|
|
))
|
||
|
|
for label, title in FIGURE_TABLE_PATTERN.findall(value):
|
||
|
|
target_type = 'table' if re.search(r'Table', label, re.I) else 'figure'
|
||
|
|
candidates.append(EvidenceCandidate(
|
||
|
|
object_type='figure_table_target',
|
||
|
|
value=_normalize_figure_table_label(label),
|
||
|
|
role=target_type,
|
||
|
|
target_field='reproduction_targets',
|
||
|
|
source_file=source_file,
|
||
|
|
source_field=source_field,
|
||
|
|
evidence_text=(title or '').strip()[:500],
|
||
|
|
confidence_rule='caption_or_figure_table_label',
|
||
|
|
))
|
||
|
|
return candidates
|
||
|
|
|
||
|
|
|
||
|
|
def _classify_accession_context(excerpt: str) -> str:
|
||
|
|
value = str(excerpt or '')
|
||
|
|
if EXPRESSION_MATRIX_CONTEXT_PATTERN.search(value):
|
||
|
|
return 'expression_matrix_accession'
|
||
|
|
if RAW_RNASEQ_CONTEXT_PATTERN.search(value):
|
||
|
|
return 'raw_rnaseq_accession'
|
||
|
|
if ACCESSION_CONTEXT_PATTERN.search(value):
|
||
|
|
return 'rejected_ambiguous_accession_context'
|
||
|
|
return 'rejected_no_method_data_context'
|
||
|
|
|
||
|
|
|
||
|
|
def _row_text(row: dict[str, Any]) -> str:
|
||
|
|
preferred = []
|
||
|
|
for key in ('evidence_sentence', 'raw_method_text', 'caption', 'title', 'input_data', 'output_data', 'parameters'):
|
||
|
|
value = row.get(key)
|
||
|
|
if value:
|
||
|
|
preferred.append(str(value))
|
||
|
|
if preferred:
|
||
|
|
return '\n'.join(preferred)
|
||
|
|
return '\n'.join(str(value) for value in row.values() if value)
|
||
|
|
|
||
|
|
|
||
|
|
def complete_methodology_plan_species(plan: dict[str, Any], *, source: str = '') -> dict[str, Any]:
|
||
|
|
"""Fill executable species lists from reviewed methodology evidence.
|
||
|
|
|
||
|
|
This is intentionally a bridge-side safety net. It only promotes species
|
||
|
|
that appear in structured species_summary rows or in method/figure evidence,
|
||
|
|
and it ignores species that occur only in abbreviation-definition clauses.
|
||
|
|
"""
|
||
|
|
if not isinstance(plan, dict):
|
||
|
|
return plan
|
||
|
|
completed = copy.deepcopy(plan)
|
||
|
|
records = _methodology_species_records(completed, source=source)
|
||
|
|
if not records:
|
||
|
|
return completed
|
||
|
|
|
||
|
|
role_groups = _resolve_methodology_species_role_groups(completed, records, source=source)
|
||
|
|
prefill = completed.get('bbav5_prefill')
|
||
|
|
if not isinstance(prefill, dict):
|
||
|
|
prefill = {}
|
||
|
|
completed['bbav5_prefill'] = prefill
|
||
|
|
|
||
|
|
targets = role_groups['targets']
|
||
|
|
references = role_groups['references']
|
||
|
|
comparatives = role_groups['comparatives']
|
||
|
|
target_keys = {_species_key(item) for item in targets}
|
||
|
|
reference_keys = {_species_key(item) for item in references}
|
||
|
|
comparative_keys = {_species_key(item) for item in comparatives}
|
||
|
|
|
||
|
|
prefill['targetSpecies'] = targets
|
||
|
|
prefill['referenceSpecies'] = references
|
||
|
|
prefill['comparativeSpecies'] = comparatives
|
||
|
|
if references and not _list(prefill.get('primaryReferenceSpecies')):
|
||
|
|
prefill['primaryReferenceSpecies'] = [references[0]]
|
||
|
|
if references and not _list(prefill.get('identifyBlastReferenceSpecies')):
|
||
|
|
prefill['identifyBlastReferenceSpecies'] = [references[0]]
|
||
|
|
prefill['scopeMode'] = prefill.get('scopeMode') or _scope_mode(prefill)
|
||
|
|
|
||
|
|
completed['data_requirements'] = _complete_species_requirements(
|
||
|
|
completed.get('data_requirements'),
|
||
|
|
targets=targets,
|
||
|
|
references=references,
|
||
|
|
comparatives=comparatives,
|
||
|
|
)
|
||
|
|
return completed
|
||
|
|
|
||
|
|
|
||
|
|
def _complete_methodology_required_pfams(plan: dict[str, Any]) -> dict[str, Any]:
|
||
|
|
completed = copy.deepcopy(plan)
|
||
|
|
prefill = completed.get('bbav5_prefill')
|
||
|
|
if not isinstance(prefill, dict):
|
||
|
|
prefill = {}
|
||
|
|
completed['bbav5_prefill'] = prefill
|
||
|
|
if not str(prefill.get('requiredPfams') or '').strip():
|
||
|
|
inferred = _infer_required_pfams_from_plan(completed) or _infer_required_pfams_from_batch_dirs(completed)
|
||
|
|
if inferred:
|
||
|
|
prefill['requiredPfams'] = inferred
|
||
|
|
return completed
|
||
|
|
|
||
|
|
|
||
|
|
def _complete_methodology_stage_order(plan: dict[str, Any]) -> dict[str, Any]:
|
||
|
|
completed = copy.deepcopy(plan)
|
||
|
|
inferred = _infer_stage_order_from_plan(completed)
|
||
|
|
if not inferred:
|
||
|
|
return completed
|
||
|
|
routes = completed.get('execution_routes')
|
||
|
|
if not isinstance(routes, list) or not routes:
|
||
|
|
completed['execution_routes'] = [{'route_id': 'methodology_default', 'stage_order': inferred}]
|
||
|
|
return completed
|
||
|
|
for route in routes:
|
||
|
|
if not isinstance(route, dict):
|
||
|
|
continue
|
||
|
|
current = route.get('stage_order') if isinstance(route.get('stage_order'), list) else []
|
||
|
|
route['stage_order'] = _fixed_methodology_stage_order([*current, *inferred])
|
||
|
|
return completed
|
||
|
|
completed['execution_routes'].append({'route_id': 'methodology_default', 'stage_order': inferred})
|
||
|
|
return completed
|
||
|
|
|
||
|
|
|
||
|
|
def _complete_methodology_dataset_gates(plan: dict[str, Any], candidates: list[EvidenceCandidate]) -> dict[str, Any]:
|
||
|
|
completed = copy.deepcopy(plan)
|
||
|
|
gates = [copy.deepcopy(gate) for gate in completed.get('manual_gates') or [] if isinstance(gate, dict)]
|
||
|
|
seen = {str(gate.get('accession') or gate.get('value') or '').upper() for gate in gates}
|
||
|
|
for candidate in candidates:
|
||
|
|
if candidate.object_type != 'dataset_accession' or candidate.guardrail_status != 'accepted' or candidate.role != 'raw_rnaseq_accession':
|
||
|
|
continue
|
||
|
|
if candidate.value in seen:
|
||
|
|
continue
|
||
|
|
gates.append({
|
||
|
|
'gate_type': 'dataset_accession',
|
||
|
|
'accession': candidate.value,
|
||
|
|
'dataset_kind': candidate.role,
|
||
|
|
'status': 'requires_user_data_or_download_confirmation',
|
||
|
|
'source_file': candidate.source_file,
|
||
|
|
'source_field': candidate.source_field,
|
||
|
|
'evidence_text': candidate.evidence_text,
|
||
|
|
'reason': 'methodology_mentions_external_dataset_accession',
|
||
|
|
})
|
||
|
|
seen.add(candidate.value)
|
||
|
|
if gates:
|
||
|
|
completed['manual_gates'] = gates
|
||
|
|
return completed
|
||
|
|
|
||
|
|
|
||
|
|
def _complete_methodology_expression_requirements(plan: dict[str, Any], candidates: list[EvidenceCandidate]) -> dict[str, Any]:
|
||
|
|
completed = copy.deepcopy(plan)
|
||
|
|
requirements = [copy.deepcopy(req) for req in completed.get('data_requirements') or [] if isinstance(req, dict)]
|
||
|
|
seen = {
|
||
|
|
(str(req.get('requirement_type') or ''), str(req.get('accession') or req.get('value') or '').upper())
|
||
|
|
for req in requirements
|
||
|
|
}
|
||
|
|
for candidate in candidates:
|
||
|
|
if candidate.object_type != 'dataset_accession' or candidate.guardrail_status != 'accepted' or candidate.role != 'expression_matrix_accession':
|
||
|
|
continue
|
||
|
|
key = ('expression_matrix', candidate.value)
|
||
|
|
if key in seen:
|
||
|
|
continue
|
||
|
|
requirements.append({
|
||
|
|
'requirement_type': 'expression_matrix',
|
||
|
|
'accession': candidate.value,
|
||
|
|
'bbav5_role': 'expression_data',
|
||
|
|
'required_files': ['expression_matrix'],
|
||
|
|
'source': 'methodology_expression_accession_completion',
|
||
|
|
'source_file': candidate.source_file,
|
||
|
|
'source_field': candidate.source_field,
|
||
|
|
'evidence_text': candidate.evidence_text,
|
||
|
|
})
|
||
|
|
seen.add(key)
|
||
|
|
if requirements:
|
||
|
|
completed['data_requirements'] = requirements
|
||
|
|
return completed
|
||
|
|
|
||
|
|
|
||
|
|
def _complete_methodology_reproduction_targets(plan: dict[str, Any], candidates: list[EvidenceCandidate]) -> dict[str, Any]:
|
||
|
|
completed = copy.deepcopy(plan)
|
||
|
|
targets = [copy.deepcopy(target) for target in completed.get('reproduction_targets') or [] if isinstance(target, dict)]
|
||
|
|
seen = {_normalize_figure_table_label(target.get('label') or target.get('value')).lower() for target in targets}
|
||
|
|
for candidate in candidates:
|
||
|
|
if candidate.object_type != 'figure_table_target' or candidate.guardrail_status != 'accepted':
|
||
|
|
continue
|
||
|
|
key = _normalize_figure_table_label(candidate.value).lower()
|
||
|
|
if not key or key in seen:
|
||
|
|
continue
|
||
|
|
targets.append({
|
||
|
|
'target_type': candidate.role or 'figure_table',
|
||
|
|
'label': candidate.value,
|
||
|
|
'title': candidate.evidence_text,
|
||
|
|
**_figure_table_coverage(candidate, plan),
|
||
|
|
'source_file': candidate.source_file,
|
||
|
|
'source_field': candidate.source_field,
|
||
|
|
'status': 'planning_target_only',
|
||
|
|
'reason': 'figure_or_table_caption_detected',
|
||
|
|
})
|
||
|
|
seen.add(key)
|
||
|
|
if targets:
|
||
|
|
completed['reproduction_targets'] = targets
|
||
|
|
return completed
|
||
|
|
|
||
|
|
|
||
|
|
def _figure_table_coverage(candidate: EvidenceCandidate, plan: dict[str, Any]) -> dict[str, str]:
|
||
|
|
suggested = _infer_stage_from_figure_table_text(f'{candidate.value} {candidate.evidence_text}')
|
||
|
|
stages = set(_first_route_stages(plan) or _infer_stage_order_from_plan(plan))
|
||
|
|
if suggested and suggested in stages:
|
||
|
|
return {
|
||
|
|
'suggested_stage': suggested,
|
||
|
|
'coverage_status': 'covered_by_stage',
|
||
|
|
'coverage_reason': 'caption_keywords_match_existing_methodology_stage',
|
||
|
|
}
|
||
|
|
if suggested:
|
||
|
|
return {
|
||
|
|
'suggested_stage': suggested,
|
||
|
|
'coverage_status': 'needs_method_inference_or_default_tool',
|
||
|
|
'coverage_reason': 'caption_suggests_stage_without_methodology_stage_evidence',
|
||
|
|
}
|
||
|
|
return {
|
||
|
|
'suggested_stage': '',
|
||
|
|
'coverage_status': 'unclassified_target',
|
||
|
|
'coverage_reason': 'caption_has_no_supported_stage_keyword',
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def _infer_stage_from_figure_table_text(text: str) -> str:
|
||
|
|
value = str(text or '').lower()
|
||
|
|
keyword_map = [
|
||
|
|
('expression', 'expression'),
|
||
|
|
('heatmap', 'expression'),
|
||
|
|
('phylogen', 'phylogeny'),
|
||
|
|
('tree', 'phylogeny'),
|
||
|
|
('motif', 'motif'),
|
||
|
|
('domain', 'domain'),
|
||
|
|
('structure', 'structure'),
|
||
|
|
('intron', 'structure'),
|
||
|
|
('exon', 'structure'),
|
||
|
|
('promoter', 'promoter'),
|
||
|
|
('cis', 'promoter'),
|
||
|
|
('synten', 'synteny'),
|
||
|
|
('collinear', 'synteny'),
|
||
|
|
('chromosome', 'karyotype'),
|
||
|
|
('location', 'karyotype'),
|
||
|
|
('ppi', 'ppi'),
|
||
|
|
('protein interaction', 'ppi'),
|
||
|
|
]
|
||
|
|
for keyword, stage in keyword_map:
|
||
|
|
if keyword in value:
|
||
|
|
return stage
|
||
|
|
return ''
|
||
|
|
|
||
|
|
|
||
|
|
def _infer_stage_order_from_plan(plan: dict[str, Any]) -> list[str]:
|
||
|
|
stages: list[str] = []
|
||
|
|
|
||
|
|
def add(value: Any) -> None:
|
||
|
|
stage = _normalize_methodology_stage(value)
|
||
|
|
if stage:
|
||
|
|
stages.append(stage)
|
||
|
|
|
||
|
|
def walk(value: Any, key: str = '') -> None:
|
||
|
|
if isinstance(value, dict):
|
||
|
|
for child_key, child_value in value.items():
|
||
|
|
if child_key in {'bbav5_stage_id', 'bbav5_stage', 'mapped_bbav5_stage', 'stage', 'normalized_stage'}:
|
||
|
|
add(child_value)
|
||
|
|
walk(child_value, str(child_key))
|
||
|
|
elif isinstance(value, list):
|
||
|
|
for item in value:
|
||
|
|
walk(item, key)
|
||
|
|
|
||
|
|
for section in ('stages', 'method_components', 'evidence', 'observations', 'species_summary'):
|
||
|
|
walk(plan.get(section), section)
|
||
|
|
stages.extend(_infer_stage_order_from_batch_dirs(plan))
|
||
|
|
return _fixed_methodology_stage_order(_unique(stages))
|
||
|
|
|
||
|
|
|
||
|
|
def _infer_stage_order_from_batch_dirs(plan: dict[str, Any]) -> list[str]:
|
||
|
|
stages: list[str] = []
|
||
|
|
for batch_dir in _methodology_batch_dirs(plan):
|
||
|
|
for csv_name in ('review_template.csv', 'species_summary.csv', 'summary.csv'):
|
||
|
|
path = batch_dir / csv_name
|
||
|
|
if not path.exists():
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
with path.open('r', encoding='utf-8', newline='') as handle:
|
||
|
|
reader = csv.DictReader(handle)
|
||
|
|
for row in reader:
|
||
|
|
for key in ('bbav5_stage', 'mapped_bbav5_stage', 'bbav5_stage_id', 'normalized_stage', 'stage'):
|
||
|
|
stage = _normalize_methodology_stage(row.get(key))
|
||
|
|
if stage:
|
||
|
|
stages.append(stage)
|
||
|
|
except Exception:
|
||
|
|
continue
|
||
|
|
for json_name in ('observations.json', 'alignment.json'):
|
||
|
|
path = batch_dir / json_name
|
||
|
|
if not path.exists():
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
data = json.loads(path.read_text(encoding='utf-8'))
|
||
|
|
except Exception:
|
||
|
|
continue
|
||
|
|
stages.extend(_infer_stage_order_from_json_value(data))
|
||
|
|
return _fixed_methodology_stage_order(_unique(stages))
|
||
|
|
|
||
|
|
|
||
|
|
def _infer_stage_order_from_json_value(value: Any) -> list[str]:
|
||
|
|
stages: list[str] = []
|
||
|
|
|
||
|
|
def walk(item: Any) -> None:
|
||
|
|
if isinstance(item, dict):
|
||
|
|
for key, child in item.items():
|
||
|
|
if key in {'bbav5_stage', 'mapped_bbav5_stage', 'bbav5_stage_id', 'normalized_stage', 'stage'}:
|
||
|
|
stage = _normalize_methodology_stage(child)
|
||
|
|
if stage:
|
||
|
|
stages.append(stage)
|
||
|
|
walk(child)
|
||
|
|
elif isinstance(item, list):
|
||
|
|
for child in item:
|
||
|
|
walk(child)
|
||
|
|
|
||
|
|
walk(value)
|
||
|
|
return _fixed_methodology_stage_order(_unique(stages))
|
||
|
|
|
||
|
|
|
||
|
|
def _infer_required_pfams_from_batch_dirs(plan: dict[str, Any]) -> str:
|
||
|
|
pfams: list[str] = []
|
||
|
|
for batch_dir in _methodology_batch_dirs(plan):
|
||
|
|
for name in ('review_template.csv', 'observations.json', 'alignment.json', 'methodology_parameter_audit.csv'):
|
||
|
|
path = batch_dir / name
|
||
|
|
if not path.exists() or not path.is_file():
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
text = path.read_text(encoding='utf-8', errors='ignore')
|
||
|
|
except Exception:
|
||
|
|
continue
|
||
|
|
for hit in re.findall(r'PF\d{5}', text):
|
||
|
|
if hit not in pfams:
|
||
|
|
pfams.append(hit)
|
||
|
|
return ','.join(pfams)
|
||
|
|
|
||
|
|
|
||
|
|
def _normalize_methodology_stage(value: Any) -> str:
|
||
|
|
token = re.sub(r'[^a-z0-9]+', '_', str(value or '').strip().lower()).strip('_')
|
||
|
|
if not token:
|
||
|
|
return ''
|
||
|
|
stage = METHODOLOGY_STAGE_ALIASES.get(token) or token
|
||
|
|
return stage if stage in FIXED_METHODOLOGY_STAGE_ORDER else ''
|
||
|
|
|
||
|
|
|
||
|
|
def _attach_methodology_completion_audit(
|
||
|
|
completed: dict[str, Any],
|
||
|
|
original: dict[str, Any],
|
||
|
|
candidates: list[EvidenceCandidate],
|
||
|
|
*,
|
||
|
|
source: str = '',
|
||
|
|
) -> None:
|
||
|
|
if not isinstance(completed, dict):
|
||
|
|
return
|
||
|
|
before_prefill = original.get('bbav5_prefill') if isinstance(original.get('bbav5_prefill'), dict) else {}
|
||
|
|
after_prefill = completed.get('bbav5_prefill') if isinstance(completed.get('bbav5_prefill'), dict) else {}
|
||
|
|
previous_audit = original.get('methodology_completion_audit') if isinstance(original.get('methodology_completion_audit'), dict) else {}
|
||
|
|
previous_records = [copy.deepcopy(record) for record in previous_audit.get('records') or [] if isinstance(record, dict)]
|
||
|
|
new_records: list[dict[str, Any]] = []
|
||
|
|
|
||
|
|
def added_list(field: str, rule: str) -> None:
|
||
|
|
before = {_species_key(item) for item in _list(before_prefill.get(field))}
|
||
|
|
for item in _list(after_prefill.get(field)):
|
||
|
|
if _species_key(item) not in before:
|
||
|
|
new_records.append(_audit_record('species', item, field, rule, candidates))
|
||
|
|
|
||
|
|
added_list('targetSpecies', 'role_aware_species_completion')
|
||
|
|
added_list('referenceSpecies', 'role_aware_species_completion')
|
||
|
|
added_list('comparativeSpecies', 'role_aware_species_completion')
|
||
|
|
if str(after_prefill.get('requiredPfams') or '').strip() and not str(before_prefill.get('requiredPfams') or '').strip():
|
||
|
|
new_records.append(_audit_record('domain_requirement', after_prefill.get('requiredPfams'), 'requiredPfams', 'pfam_evidence_completion', candidates))
|
||
|
|
|
||
|
|
before_stages = _first_route_stages(original)
|
||
|
|
for stage in _first_route_stages(completed):
|
||
|
|
if stage not in before_stages:
|
||
|
|
new_records.append(_audit_record('stage', stage, 'execution_routes.stage_order', 'stage_evidence_completion', candidates))
|
||
|
|
|
||
|
|
before_gates = {str(gate.get('accession') or '').upper() for gate in original.get('manual_gates') or [] if isinstance(gate, dict)}
|
||
|
|
for gate in completed.get('manual_gates') or []:
|
||
|
|
if not isinstance(gate, dict):
|
||
|
|
continue
|
||
|
|
accession = str(gate.get('accession') or '').upper()
|
||
|
|
if accession and accession not in before_gates:
|
||
|
|
new_records.append(_audit_record('dataset_accession', accession, 'manual_gates', 'accession_manual_gate_completion', candidates))
|
||
|
|
|
||
|
|
before_expression = {
|
||
|
|
str(req.get('accession') or '').upper()
|
||
|
|
for req in original.get('data_requirements') or []
|
||
|
|
if isinstance(req, dict) and str(req.get('requirement_type') or '') == 'expression_matrix'
|
||
|
|
}
|
||
|
|
for req in completed.get('data_requirements') or []:
|
||
|
|
if not isinstance(req, dict) or str(req.get('requirement_type') or '') != 'expression_matrix':
|
||
|
|
continue
|
||
|
|
accession = str(req.get('accession') or '').upper()
|
||
|
|
if accession and accession not in before_expression:
|
||
|
|
new_records.append(_audit_record('dataset_accession', accession, 'data_requirements', 'expression_matrix_requirement_completion', candidates))
|
||
|
|
|
||
|
|
before_targets = {_normalize_figure_table_label(target.get('label') or '').lower() for target in original.get('reproduction_targets') or [] if isinstance(target, dict)}
|
||
|
|
for target in completed.get('reproduction_targets') or []:
|
||
|
|
if not isinstance(target, dict):
|
||
|
|
continue
|
||
|
|
label = str(target.get('label') or '')
|
||
|
|
if label and _normalize_figure_table_label(label).lower() not in before_targets:
|
||
|
|
new_records.append(_audit_record('figure_table_target', label, 'reproduction_targets', 'figure_table_target_completion', candidates))
|
||
|
|
|
||
|
|
records = _merge_audit_records(previous_records, new_records)
|
||
|
|
accepted_candidates = [candidate.to_record() for candidate in candidates if candidate.guardrail_status == 'accepted']
|
||
|
|
rejected_candidates = [candidate.to_record() for candidate in candidates if candidate.guardrail_status != 'accepted']
|
||
|
|
completion_run_count = int(previous_audit.get('completion_run_count') or 0) + 1
|
||
|
|
|
||
|
|
completed['methodology_completion_audit'] = {
|
||
|
|
'paradigm_version': METHODOLOGY_COMPLETION_PARADIGM_VERSION,
|
||
|
|
'source': source,
|
||
|
|
'principle': 'evidence_candidate_to_role_to_requirement_with_false_positive_guards',
|
||
|
|
'applied_rules': _unique([str(record['rule']) for record in records]),
|
||
|
|
'records': records,
|
||
|
|
'accepted_candidates': accepted_candidates,
|
||
|
|
'rejected_candidates': rejected_candidates,
|
||
|
|
'candidate_count': len(candidates),
|
||
|
|
'accepted_candidate_count': len(accepted_candidates),
|
||
|
|
'rejected_candidate_count': len(rejected_candidates),
|
||
|
|
'candidate_types': _unique([candidate.object_type for candidate in candidates]),
|
||
|
|
'records_preserved_from_previous_completion': len(previous_records),
|
||
|
|
'completion_run_count': completion_run_count,
|
||
|
|
'guardrails': [
|
||
|
|
'do_not_promote_abbreviation_definition_only_species',
|
||
|
|
'do_not_guess_latin_binomials_without_structured_or_alias_evidence',
|
||
|
|
'do_not_promote_accession_without_method_data_context',
|
||
|
|
'figure_table_targets_do_not_force_execution_stage_without_method_evidence',
|
||
|
|
'preserve_existing_user_or_rag_prefill_values',
|
||
|
|
],
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def _merge_audit_records(previous: list[dict[str, Any]], new: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||
|
|
result: list[dict[str, Any]] = []
|
||
|
|
seen: set[tuple[str, str, str, str]] = set()
|
||
|
|
for record in [*previous, *new]:
|
||
|
|
key = (
|
||
|
|
str(record.get('object_type') or ''),
|
||
|
|
str(record.get('target_field') or record.get('field') or ''),
|
||
|
|
_normalize_figure_table_label(record.get('value')) if str(record.get('object_type') or '') == 'figure_table_target' else str(record.get('value') or ''),
|
||
|
|
str(record.get('rule') or ''),
|
||
|
|
)
|
||
|
|
if key in seen:
|
||
|
|
continue
|
||
|
|
result.append(record)
|
||
|
|
seen.add(key)
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _methodology_completion_summary(plan: dict[str, Any]) -> dict[str, Any]:
|
||
|
|
audit = plan.get('methodology_completion_audit') if isinstance(plan.get('methodology_completion_audit'), dict) else {}
|
||
|
|
targets = [target for target in plan.get('reproduction_targets') or [] if isinstance(target, dict)]
|
||
|
|
return {
|
||
|
|
'paradigm_version': audit.get('paradigm_version') or '',
|
||
|
|
'completed_record_count': len(audit.get('records') or []),
|
||
|
|
'accepted_candidate_count': int(audit.get('accepted_candidate_count') or 0),
|
||
|
|
'rejected_candidate_count': int(audit.get('rejected_candidate_count') or 0),
|
||
|
|
'manual_gate_count': len([gate for gate in plan.get('manual_gates') or [] if isinstance(gate, dict)]),
|
||
|
|
'data_requirement_count': len([req for req in plan.get('data_requirements') or [] if isinstance(req, dict)]),
|
||
|
|
'reproduction_target_count': len(targets),
|
||
|
|
'covered_reproduction_target_count': len([target for target in targets if target.get('coverage_status') == 'covered_by_stage']),
|
||
|
|
'completion_run_count': int(audit.get('completion_run_count') or 0),
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def _audit_record(
|
||
|
|
object_type: str,
|
||
|
|
value: Any,
|
||
|
|
target_field: str,
|
||
|
|
rule: str,
|
||
|
|
candidates: list[EvidenceCandidate],
|
||
|
|
) -> dict[str, Any]:
|
||
|
|
value_text = str(value or '')
|
||
|
|
candidate = _best_candidate(object_type, value_text, candidates)
|
||
|
|
record = {
|
||
|
|
'object_type': object_type,
|
||
|
|
'field': target_field,
|
||
|
|
'target_field': target_field,
|
||
|
|
'value': value,
|
||
|
|
'rule': rule,
|
||
|
|
'source_file': candidate.source_file if candidate else '',
|
||
|
|
'source_field': candidate.source_field if candidate else '',
|
||
|
|
'evidence_text': candidate.evidence_text if candidate else '',
|
||
|
|
'confidence_rule': candidate.confidence_rule if candidate else rule,
|
||
|
|
'guardrail_status': candidate.guardrail_status if candidate else 'accepted',
|
||
|
|
}
|
||
|
|
return record
|
||
|
|
|
||
|
|
|
||
|
|
def _best_candidate(object_type: str, value: str, candidates: list[EvidenceCandidate]) -> EvidenceCandidate | None:
|
||
|
|
aliases = {
|
||
|
|
'domain_requirement': {'pfam', 'domain_requirement'},
|
||
|
|
'species': {'species'},
|
||
|
|
'stage': {'stage'},
|
||
|
|
'dataset_accession': {'dataset_accession'},
|
||
|
|
'figure_table_target': {'figure_table_target'},
|
||
|
|
}.get(object_type, {object_type})
|
||
|
|
normalized_value = _normalize_candidate_value(value)
|
||
|
|
for candidate in candidates:
|
||
|
|
if candidate.object_type not in aliases or candidate.guardrail_status != 'accepted':
|
||
|
|
continue
|
||
|
|
if candidate.object_type == 'pfam' and candidate.value and candidate.value in value:
|
||
|
|
return candidate
|
||
|
|
if _normalize_candidate_value(candidate.value) == normalized_value:
|
||
|
|
return candidate
|
||
|
|
for candidate in candidates:
|
||
|
|
if candidate.object_type in aliases and candidate.guardrail_status == 'accepted':
|
||
|
|
return candidate
|
||
|
|
return None
|
||
|
|
|
||
|
|
|
||
|
|
def _normalize_candidate_value(value: Any) -> str:
|
||
|
|
return re.sub(r'[^a-z0-9]+', '', str(value or '').lower())
|
||
|
|
|
||
|
|
|
||
|
|
def _normalize_figure_table_label(value: Any) -> str:
|
||
|
|
label = re.sub(r'\s+', ' ', str(value or '').strip()).rstrip('.')
|
||
|
|
label = re.sub(r'^(?:Fig)\.?\s+', 'Figure ', label, flags=re.I)
|
||
|
|
label = re.sub(r'^(?:Tab)\.?\s+', 'Table ', label, flags=re.I)
|
||
|
|
return label
|
||
|
|
|
||
|
|
|
||
|
|
def _first_route_stages(plan: dict[str, Any]) -> list[str]:
|
||
|
|
for route in plan.get('execution_routes') or []:
|
||
|
|
if isinstance(route, dict) and isinstance(route.get('stage_order'), list):
|
||
|
|
return [str(item) for item in route.get('stage_order') if str(item)]
|
||
|
|
return []
|
||
|
|
|
||
|
|
|
||
|
|
def _methodology_species_records(plan: dict[str, Any], *, source: str = '') -> list[dict[str, str]]:
|
||
|
|
records: list[dict[str, str]] = []
|
||
|
|
records.extend(_species_records_from_prefill(plan))
|
||
|
|
records.extend(_species_records_from_requirements(plan.get('data_requirements')))
|
||
|
|
records.extend(_species_records_from_plan_field(plan.get('species_summary')))
|
||
|
|
batch_dirs = _methodology_batch_dirs(plan, source=source)
|
||
|
|
for batch_dir in batch_dirs:
|
||
|
|
records.extend(_species_records_from_species_summary_csv(batch_dir / 'species_summary.csv'))
|
||
|
|
records.extend(_species_records_from_text(_batch_species_text(batch_dir), default_role='reference_species'))
|
||
|
|
if not batch_dirs:
|
||
|
|
records.extend(_species_records_from_text(_plan_species_text(plan), default_role='reference_species'))
|
||
|
|
return _unique_species_records(records)
|
||
|
|
|
||
|
|
|
||
|
|
def _resolve_methodology_species_role_groups(
|
||
|
|
plan: dict[str, Any],
|
||
|
|
records: list[dict[str, str]],
|
||
|
|
*,
|
||
|
|
source: str = '',
|
||
|
|
) -> dict[str, list[str]]:
|
||
|
|
prefill = plan.get('bbav5_prefill') if isinstance(plan.get('bbav5_prefill'), dict) else {}
|
||
|
|
prefill_targets = [_canonical_latin_species(item) for item in _list(prefill.get('targetSpecies'))]
|
||
|
|
prefill_targets = [item for item in prefill_targets if item]
|
||
|
|
strong_targets = _strong_target_species_from_methodology_evidence(plan, source=source)
|
||
|
|
target_keys = {_species_key(item) for item in strong_targets}
|
||
|
|
|
||
|
|
if strong_targets:
|
||
|
|
targets = strong_targets
|
||
|
|
elif len(prefill_targets) == 1:
|
||
|
|
targets = prefill_targets
|
||
|
|
target_keys = {_species_key(item) for item in targets}
|
||
|
|
else:
|
||
|
|
targets = []
|
||
|
|
for record in records:
|
||
|
|
species = _canonical_latin_species(record.get('species_name'))
|
||
|
|
if species and _normalize_species_role(record.get('species_role')) == 'target' and _species_key(species) not in target_keys:
|
||
|
|
targets.append(species)
|
||
|
|
target_keys.add(_species_key(species))
|
||
|
|
|
||
|
|
references: list[str] = []
|
||
|
|
reference_keys: set[str] = set()
|
||
|
|
comparatives: list[str] = []
|
||
|
|
comparative_keys: set[str] = set()
|
||
|
|
|
||
|
|
for record in records:
|
||
|
|
species = _canonical_latin_species(record.get('species_name'))
|
||
|
|
if not species:
|
||
|
|
continue
|
||
|
|
key = _species_key(species)
|
||
|
|
if key in target_keys:
|
||
|
|
continue
|
||
|
|
role = _normalize_species_role(record.get('species_role'))
|
||
|
|
if role == 'comparative':
|
||
|
|
if key not in comparative_keys and key not in reference_keys:
|
||
|
|
comparatives.append(species)
|
||
|
|
comparative_keys.add(key)
|
||
|
|
elif role == 'reference':
|
||
|
|
if key not in reference_keys and key not in comparative_keys:
|
||
|
|
references.append(species)
|
||
|
|
reference_keys.add(key)
|
||
|
|
|
||
|
|
return {'targets': targets, 'references': references, 'comparatives': comparatives}
|
||
|
|
|
||
|
|
|
||
|
|
def _strong_target_species_from_methodology_evidence(plan: dict[str, Any], *, source: str = '') -> list[str]:
|
||
|
|
targets: list[str] = []
|
||
|
|
|
||
|
|
def add(species: Any) -> None:
|
||
|
|
canonical = _canonical_latin_species(species)
|
||
|
|
if canonical and _species_key(canonical) not in {_species_key(item) for item in targets}:
|
||
|
|
targets.append(canonical)
|
||
|
|
|
||
|
|
for row in plan.get('species_summary') or []:
|
||
|
|
if not isinstance(row, dict):
|
||
|
|
continue
|
||
|
|
if _normalize_species_role(row.get('species_role') or row.get('role')) != 'target':
|
||
|
|
continue
|
||
|
|
species = row.get('species_name') or row.get('species')
|
||
|
|
evidence = str(row.get('evidence_sentence') or '')
|
||
|
|
if _is_supported_target_species_evidence(species, evidence):
|
||
|
|
add(species)
|
||
|
|
|
||
|
|
for batch_dir in _methodology_batch_dirs(plan, source=source):
|
||
|
|
path = batch_dir / 'species_summary.csv'
|
||
|
|
if not path.exists():
|
||
|
|
continue
|
||
|
|
try:
|
||
|
|
with path.open('r', encoding='utf-8', newline='') as handle:
|
||
|
|
reader = csv.DictReader(handle)
|
||
|
|
for row in reader:
|
||
|
|
if _normalize_species_role(row.get('species_role')) != 'target':
|
||
|
|
continue
|
||
|
|
if _is_supported_target_species_evidence(row.get('species_name'), row.get('evidence_sentence')):
|
||
|
|
add(row.get('species_name'))
|
||
|
|
except Exception:
|
||
|
|
continue
|
||
|
|
|
||
|
|
return targets
|
||
|
|
|
||
|
|
|
||
|
|
def _is_supported_target_species_evidence(species: Any, evidence: Any) -> bool:
|
||
|
|
canonical = _canonical_latin_species(species)
|
||
|
|
text = str(evidence or '')
|
||
|
|
if not canonical or not text:
|
||
|
|
return False
|
||
|
|
if _species_mention_is_abbreviation_definition_only(canonical, text):
|
||
|
|
return False
|
||
|
|
if not _species_evidence_mentions_species(canonical, text):
|
||
|
|
return False
|
||
|
|
return _species_has_target_context(canonical, text)
|
||
|
|
|
||
|
|
|
||
|
|
def _species_has_target_context(species: str, text: str) -> bool:
|
||
|
|
stripped = _strip_species_abbreviation_definitions(str(text or ''))
|
||
|
|
lower = stripped.lower()
|
||
|
|
aliases = _species_context_aliases(species)
|
||
|
|
for alias in aliases:
|
||
|
|
alias_lower = alias.lower()
|
||
|
|
if alias_lower not in lower:
|
||
|
|
continue
|
||
|
|
for match in re.finditer(re.escape(alias_lower), lower):
|
||
|
|
start = max(0, match.start() - 120)
|
||
|
|
end = min(len(lower), match.end() + 120)
|
||
|
|
window = lower[start:end]
|
||
|
|
if 'genome-wide' in window:
|
||
|
|
return True
|
||
|
|
if re.search(r'(identification|identified|screened).{0,80}(genes?|proteins?)', window) and alias_lower in window:
|
||
|
|
return True
|
||
|
|
if re.search(r'(genes?|proteins?).{0,80}(identification|identified|screened)', window) and alias_lower in window:
|
||
|
|
return True
|
||
|
|
if f'{alias_lower} genome' in window or f'{alias_lower} genes' in window or f'{alias_lower} proteins' in window:
|
||
|
|
return True
|
||
|
|
if re.search(rf'(from|in)\s+(the\s+)?{re.escape(alias_lower)}\s+genome', window):
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
def _text_species_record_has_method_context(species: str, role: str, text: str) -> bool:
|
||
|
|
if _normalize_species_role(role) != 'reference':
|
||
|
|
return True
|
||
|
|
return _species_has_reference_context(species, text)
|
||
|
|
|
||
|
|
|
||
|
|
def _species_has_reference_context(species: str, text: str) -> bool:
|
||
|
|
stripped = _strip_species_abbreviation_definitions(str(text or ''))
|
||
|
|
lower = stripped.lower()
|
||
|
|
aliases = _species_context_aliases(species)
|
||
|
|
for alias in aliases:
|
||
|
|
alias_lower = alias.lower()
|
||
|
|
if alias_lower not in lower:
|
||
|
|
continue
|
||
|
|
for match in re.finditer(re.escape(alias_lower), lower):
|
||
|
|
start = max(0, match.start() - 180)
|
||
|
|
end = min(len(lower), match.end() + 180)
|
||
|
|
window = lower[start:end]
|
||
|
|
if re.search(r'(phylogen|tree|n-j|neighbor|aligned|alignment|clustal|mega|uniprot|tair|protein sequences?|amino acid sequences?|reference proteins?|included|comparing|multiple species|blast)', window):
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
def _species_context_aliases(species: str) -> list[str]:
|
||
|
|
canonical = _canonical_latin_species(species)
|
||
|
|
aliases = [canonical] if canonical else []
|
||
|
|
aliases.extend(COMMON_SPECIES_NAME_ALIASES.get(canonical, []))
|
||
|
|
for alias, alias_species in COMMON_PLANT_SPECIES_ALIASES.items():
|
||
|
|
if _species_key(alias_species) == _species_key(canonical):
|
||
|
|
aliases.append(alias)
|
||
|
|
return _unique([alias for alias in aliases if alias])
|
||
|
|
|
||
|
|
|
||
|
|
def _species_records_from_prefill(plan: dict[str, Any]) -> list[dict[str, str]]:
|
||
|
|
prefill = plan.get('bbav5_prefill') if isinstance(plan.get('bbav5_prefill'), dict) else {}
|
||
|
|
records: list[dict[str, str]] = []
|
||
|
|
for species in _list(prefill.get('targetSpecies')):
|
||
|
|
records.append({'species_name': species, 'species_role': 'target_species'})
|
||
|
|
for species in _list(prefill.get('referenceSpecies')):
|
||
|
|
records.append({'species_name': species, 'species_role': 'reference_species'})
|
||
|
|
for species in _list(prefill.get('comparativeSpecies')):
|
||
|
|
records.append({'species_name': species, 'species_role': 'comparative_species'})
|
||
|
|
for species in _list(prefill.get('identifyBlastReferenceSpecies')):
|
||
|
|
records.append({'species_name': species, 'species_role': 'reference_species'})
|
||
|
|
return records
|
||
|
|
|
||
|
|
|
||
|
|
def _species_records_from_requirements(requirements: Any) -> list[dict[str, str]]:
|
||
|
|
records: list[dict[str, str]] = []
|
||
|
|
if not isinstance(requirements, list):
|
||
|
|
return records
|
||
|
|
for req in requirements:
|
||
|
|
if not isinstance(req, dict):
|
||
|
|
continue
|
||
|
|
species = req.get('species_name')
|
||
|
|
if species:
|
||
|
|
records.append({'species_name': str(species), 'species_role': str(req.get('bbav5_role') or 'reference_species')})
|
||
|
|
return records
|
||
|
|
|
||
|
|
|
||
|
|
def _species_records_from_plan_field(value: Any) -> list[dict[str, str]]:
|
||
|
|
records: list[dict[str, str]] = []
|
||
|
|
if not isinstance(value, list):
|
||
|
|
return records
|
||
|
|
for row in value:
|
||
|
|
if not isinstance(row, dict):
|
||
|
|
continue
|
||
|
|
species = row.get('species_name') or row.get('species')
|
||
|
|
if species:
|
||
|
|
records.append({'species_name': str(species), 'species_role': str(row.get('species_role') or row.get('role') or 'reference_species')})
|
||
|
|
return records
|
||
|
|
|
||
|
|
|
||
|
|
def _methodology_batch_dirs(plan: dict[str, Any], *, source: str = '') -> list[Path]:
|
||
|
|
candidates: list[Path] = []
|
||
|
|
for value in (plan.get('source_batch_dir'), plan.get('batch_dir'), plan.get('methodology_batch_dir'), source):
|
||
|
|
if not value:
|
||
|
|
continue
|
||
|
|
path = Path(str(value)).expanduser()
|
||
|
|
if path.is_file():
|
||
|
|
path = path.parent
|
||
|
|
if path.exists() and path.is_dir():
|
||
|
|
candidates.append(path)
|
||
|
|
return _unique_paths(candidates)
|
||
|
|
|
||
|
|
|
||
|
|
def _species_records_from_species_summary_csv(path: Path) -> list[dict[str, str]]:
|
||
|
|
records: list[dict[str, str]] = []
|
||
|
|
if not path.exists():
|
||
|
|
return records
|
||
|
|
try:
|
||
|
|
with path.open('r', encoding='utf-8', newline='') as handle:
|
||
|
|
reader = csv.DictReader(handle)
|
||
|
|
for row in reader:
|
||
|
|
species = row.get('species_name')
|
||
|
|
canonical = _canonical_latin_species(species)
|
||
|
|
if not canonical:
|
||
|
|
continue
|
||
|
|
evidence = row.get('evidence_sentence') or ''
|
||
|
|
if evidence and not _species_evidence_mentions_species(canonical, evidence):
|
||
|
|
continue
|
||
|
|
if _species_mention_is_abbreviation_definition_only(canonical, evidence):
|
||
|
|
continue
|
||
|
|
records.append({'species_name': canonical, 'species_role': row.get('species_role') or 'reference_species'})
|
||
|
|
except Exception:
|
||
|
|
return records
|
||
|
|
return records
|
||
|
|
|
||
|
|
|
||
|
|
def _batch_species_text(batch_dir: Path) -> str:
|
||
|
|
chunks: list[str] = []
|
||
|
|
for name in ('review_template.csv', 'observations.json', 'alignment.json'):
|
||
|
|
path = batch_dir / name
|
||
|
|
if path.exists() and path.is_file():
|
||
|
|
try:
|
||
|
|
chunks.append(path.read_text(encoding='utf-8', errors='ignore'))
|
||
|
|
except Exception:
|
||
|
|
continue
|
||
|
|
return '\n'.join(chunks)
|
||
|
|
|
||
|
|
|
||
|
|
def _plan_species_text(plan: dict[str, Any]) -> str:
|
||
|
|
chunks: list[str] = []
|
||
|
|
|
||
|
|
def walk(value: Any, key: str = '') -> None:
|
||
|
|
if key in {'bbav5_prefill', 'data_requirements', 'methodology_completion_audit', 'reproduction_targets'}:
|
||
|
|
return
|
||
|
|
if isinstance(value, str):
|
||
|
|
if key not in {'plan_id', 'paper_id', 'source_batch_dir', 'batch_dir', 'methodology_batch_dir'}:
|
||
|
|
chunks.append(value)
|
||
|
|
elif isinstance(value, list):
|
||
|
|
for item in value:
|
||
|
|
walk(item, key)
|
||
|
|
elif isinstance(value, dict):
|
||
|
|
for child_key, child_value in value.items():
|
||
|
|
walk(child_value, str(child_key))
|
||
|
|
|
||
|
|
walk(plan)
|
||
|
|
return '\n'.join(chunks)
|
||
|
|
|
||
|
|
|
||
|
|
def _species_records_from_text(text: str, *, default_role: str) -> list[dict[str, str]]:
|
||
|
|
value = str(text or '')
|
||
|
|
if not value:
|
||
|
|
return []
|
||
|
|
aliases = {**COMMON_PLANT_SPECIES_ALIASES, **_species_alias_definitions(value)}
|
||
|
|
evidence_text = _strip_species_abbreviation_definitions(value)
|
||
|
|
records: list[dict[str, str]] = []
|
||
|
|
for alias, species in aliases.items():
|
||
|
|
if _contains_species_alias(evidence_text, alias) and _text_species_record_has_method_context(species, default_role, evidence_text):
|
||
|
|
records.append({'species_name': species, 'species_role': default_role})
|
||
|
|
for species in _unique(list(aliases.values())):
|
||
|
|
if _contains_species_alias(evidence_text, species) and _text_species_record_has_method_context(species, default_role, evidence_text):
|
||
|
|
records.append({'species_name': species, 'species_role': default_role})
|
||
|
|
return records
|
||
|
|
|
||
|
|
|
||
|
|
def _species_alias_definitions(text: str) -> dict[str, str]:
|
||
|
|
aliases: dict[str, str] = {}
|
||
|
|
pattern = re.compile(r'\b([A-Z]\.\s*[a-z][A-Za-z-]*)\s*:\s*([A-Z][a-z]+(?:\s+[a-z][a-z.-]+)+)')
|
||
|
|
for alias, species in pattern.findall(str(text or '')):
|
||
|
|
canonical = _canonical_latin_species(species)
|
||
|
|
if canonical:
|
||
|
|
aliases[re.sub(r'\s+', ' ', alias.strip())] = canonical
|
||
|
|
return aliases
|
||
|
|
|
||
|
|
|
||
|
|
def _strip_species_abbreviation_definitions(text: str) -> str:
|
||
|
|
return re.sub(
|
||
|
|
r'\b[A-Z]\.\s*[a-z][A-Za-z-]*\s*:\s*[A-Z][a-z]+(?:\s+[a-z][a-z.-]+)+(?:\s+[A-Za-z().-]+)*[;,.]?',
|
||
|
|
' ',
|
||
|
|
str(text or ''),
|
||
|
|
)
|
||
|
|
|
||
|
|
|
||
|
|
def _species_evidence_mentions_species(species: str, text: str) -> bool:
|
||
|
|
canonical = _canonical_latin_species(species)
|
||
|
|
if not canonical:
|
||
|
|
return False
|
||
|
|
stripped = _strip_species_abbreviation_definitions(str(text or ''))
|
||
|
|
if _contains_species_alias(stripped, canonical):
|
||
|
|
return True
|
||
|
|
for alias, alias_species in COMMON_PLANT_SPECIES_ALIASES.items():
|
||
|
|
if _species_key(alias_species) == _species_key(canonical) and _contains_species_alias(stripped, alias):
|
||
|
|
return True
|
||
|
|
for alias in COMMON_SPECIES_NAME_ALIASES.get(canonical, []):
|
||
|
|
if re.search(rf'(?<![A-Za-z0-9]){re.escape(alias)}(?![A-Za-z0-9])', stripped, re.I):
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
def _species_mention_is_abbreviation_definition_only(species: str, text: str) -> bool:
|
||
|
|
value = str(text or '')
|
||
|
|
if len(re.findall(r'\b[A-Z]\.\s*[a-z][A-Za-z-]*\s*:', value)) < 2:
|
||
|
|
return False
|
||
|
|
return not _species_evidence_mentions_species(species, value)
|
||
|
|
|
||
|
|
|
||
|
|
def _contains_species_alias(text: str, alias: str) -> bool:
|
||
|
|
compact = re.escape(re.sub(r'\s+', ' ', str(alias or '').strip()))
|
||
|
|
compact = compact.replace(r'\ ', r'\s*')
|
||
|
|
return bool(re.search(rf'(?<![A-Za-z0-9]){compact}(?![A-Za-z0-9])', str(text or '')))
|
||
|
|
|
||
|
|
|
||
|
|
def _canonical_latin_species(value: Any) -> str:
|
||
|
|
species = re.sub(r'\s+', ' ', str(value or '').strip().strip(' .,:;()[]{}'))
|
||
|
|
species = re.sub(r'\s+L\.?$', '', species).strip()
|
||
|
|
if not species:
|
||
|
|
return ''
|
||
|
|
if re.match(r'^[A-Z]\.\s*[a-z][a-z.-]+$', species):
|
||
|
|
return COMMON_PLANT_SPECIES_ALIASES.get(re.sub(r'\s+', ' ', species), '')
|
||
|
|
if not re.match(r'^[A-Z][a-z]+(?:\s+[a-z][a-z.-]+)+$', species):
|
||
|
|
return ''
|
||
|
|
words = species.split()
|
||
|
|
if len(words) < 2:
|
||
|
|
return ''
|
||
|
|
return f'{words[0]} {words[1]}'
|
||
|
|
|
||
|
|
|
||
|
|
def _normalize_species_role(value: Any) -> str:
|
||
|
|
role = str(value or '').strip().lower()
|
||
|
|
if 'target' in role or role in {'target', 'query'}:
|
||
|
|
return 'target'
|
||
|
|
if 'comparative' in role or 'comparison' in role:
|
||
|
|
return 'comparative'
|
||
|
|
return 'reference'
|
||
|
|
|
||
|
|
|
||
|
|
def _complete_species_requirements(
|
||
|
|
requirements: Any,
|
||
|
|
*,
|
||
|
|
targets: list[str],
|
||
|
|
references: list[str],
|
||
|
|
comparatives: list[str],
|
||
|
|
) -> list[dict[str, Any]]:
|
||
|
|
result = [
|
||
|
|
copy.deepcopy(req)
|
||
|
|
for req in requirements
|
||
|
|
if isinstance(req, dict) and str(req.get('requirement_type') or '') != 'whole_genome_annotation'
|
||
|
|
] if isinstance(requirements, list) else []
|
||
|
|
existing = {(_species_key(req.get('species_name')), str(req.get('requirement_type') or '')) for req in result}
|
||
|
|
|
||
|
|
def add(species: str, role: str) -> None:
|
||
|
|
key = (_species_key(species), 'whole_genome_annotation')
|
||
|
|
if not species or key in existing:
|
||
|
|
return
|
||
|
|
result.append({
|
||
|
|
'species_name': species,
|
||
|
|
'bbav5_role': role,
|
||
|
|
'requirement_type': 'whole_genome_annotation',
|
||
|
|
'required_files': ['gff', 'genome_or_cds_or_pep'],
|
||
|
|
'source': 'methodology_species_completion',
|
||
|
|
})
|
||
|
|
existing.add(key)
|
||
|
|
|
||
|
|
for species in targets:
|
||
|
|
add(species, 'target')
|
||
|
|
for species in references:
|
||
|
|
add(species, 'reference')
|
||
|
|
for species in comparatives:
|
||
|
|
add(species, 'comparative')
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _unique_species_records(records: list[dict[str, str]]) -> list[dict[str, str]]:
|
||
|
|
result: list[dict[str, str]] = []
|
||
|
|
seen: set[tuple[str, str]] = set()
|
||
|
|
for record in records:
|
||
|
|
species = _canonical_latin_species(record.get('species_name'))
|
||
|
|
role = _normalize_species_role(record.get('species_role'))
|
||
|
|
if not species:
|
||
|
|
continue
|
||
|
|
key = (_species_key(species), role)
|
||
|
|
if key in seen:
|
||
|
|
continue
|
||
|
|
result.append({'species_name': species, 'species_role': role})
|
||
|
|
seen.add(key)
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _unique_candidates(candidates: list[EvidenceCandidate]) -> list[EvidenceCandidate]:
|
||
|
|
result: list[EvidenceCandidate] = []
|
||
|
|
seen: set[tuple[str, str, str, str, str]] = set()
|
||
|
|
for candidate in candidates:
|
||
|
|
if not candidate.value:
|
||
|
|
continue
|
||
|
|
key = (
|
||
|
|
candidate.object_type,
|
||
|
|
_normalize_candidate_value(candidate.value),
|
||
|
|
candidate.role,
|
||
|
|
candidate.source_file,
|
||
|
|
candidate.source_field,
|
||
|
|
)
|
||
|
|
if key in seen:
|
||
|
|
continue
|
||
|
|
result.append(candidate)
|
||
|
|
seen.add(key)
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _evidence_excerpt(text: str, needle: Any, *, window: int = 240) -> str:
|
||
|
|
value = str(text or '')
|
||
|
|
query = str(needle or '')
|
||
|
|
if not value:
|
||
|
|
return ''
|
||
|
|
if not query:
|
||
|
|
return value[:window].strip()
|
||
|
|
match = re.search(re.escape(query), value, re.I)
|
||
|
|
if not match:
|
||
|
|
return value[:window].strip()
|
||
|
|
start = max(0, match.start() - window // 3)
|
||
|
|
end = min(len(value), match.end() + (window - (match.end() - start)))
|
||
|
|
return re.sub(r'\s+', ' ', value[start:end]).strip()
|
||
|
|
|
||
|
|
|
||
|
|
def _sentence_excerpt(text: str, needle: Any) -> str:
|
||
|
|
value = str(text or '')
|
||
|
|
query = str(needle or '')
|
||
|
|
match = re.search(re.escape(query), value, re.I) if query else None
|
||
|
|
if not match:
|
||
|
|
return _evidence_excerpt(value, query)
|
||
|
|
start_candidates = [value.rfind(mark, 0, match.start()) for mark in ('.', '\n', ';')]
|
||
|
|
start = max(start_candidates)
|
||
|
|
start = 0 if start < 0 else start + 1
|
||
|
|
end_candidates = [idx for idx in (value.find(mark, match.end()) for mark in ('.', '\n', ';')) if idx >= 0]
|
||
|
|
end = min(end_candidates) if end_candidates else min(len(value), match.end() + 180)
|
||
|
|
return re.sub(r'\s+', ' ', value[start:end]).strip()
|
||
|
|
|
||
|
|
|
||
|
|
def _unique_paths(paths: list[Path]) -> list[Path]:
|
||
|
|
result: list[Path] = []
|
||
|
|
seen: set[str] = set()
|
||
|
|
for path in paths:
|
||
|
|
key = str(path.resolve())
|
||
|
|
if key not in seen:
|
||
|
|
result.append(path)
|
||
|
|
seen.add(key)
|
||
|
|
return result
|
||
|
|
|
||
|
|
|
||
|
|
def _species_key(value: Any) -> str:
|
||
|
|
return _normalize_species_alias(str(value or ''))
|
||
|
|
|
||
|
|
|
||
|
|
def is_methodology_execution_intent(text: str) -> bool:
|
||
|
|
value = str(text or '').strip()
|
||
|
|
if not value:
|
||
|
|
return False
|
||
|
|
has_action = bool(re.search(r'(按|按照|使用|继续|执行|运行|复现|run|execute|continue|reproduce|use)', value, re.I))
|
||
|
|
has_methodology = bool(re.search(r'(方法学|文献方法|已保存(?:的)?(?:方法学)?方案|保存(?:的)?方案|文献方案|methodology|literature\s+method|saved\s+plan)', value, re.I))
|
||
|
|
return has_action and has_methodology
|
||
|
|
|
||
|
|
|
||
|
|
def _infer_gene_family_from_plan(plan: dict[str, Any]) -> str:
|
||
|
|
for section in ('evidence', 'stages', 'method_components'):
|
||
|
|
for row in plan.get(section) or []:
|
||
|
|
if not isinstance(row, dict):
|
||
|
|
continue
|
||
|
|
if row.get('family'):
|
||
|
|
return _normalize_gene_family_token(row.get('family'))
|
||
|
|
text = ' '.join(str(value or '') for value in row.values())
|
||
|
|
family = _infer_gene_family_from_text(text)
|
||
|
|
if family:
|
||
|
|
return family
|
||
|
|
for key in ('paper_id', 'plan_id', 'source_batch_dir'):
|
||
|
|
value = str(plan.get(key) or '')
|
||
|
|
for match in re.finditer(r'(?<![A-Za-z0-9])(?:[A-Z][a-z]?\.)?([A-Z][A-Za-z0-9_-]{2,})(?![A-Za-z0-9])', value):
|
||
|
|
family = _normalize_gene_family_token(match.group(1))
|
||
|
|
if family and family.lower() not in {'plan', 'batch', 'methodology', 'job'}:
|
||
|
|
return family
|
||
|
|
return ''
|
||
|
|
|
||
|
|
|
||
|
|
def _normalize_gene_family_token(value: Any) -> str:
|
||
|
|
token = str(value or '').strip()
|
||
|
|
if not token:
|
||
|
|
return ''
|
||
|
|
token = token.strip(' .,:;()[]{}')
|
||
|
|
species_prefixed = re.match(r'^([A-Z][A-Za-z]{0,2})[-_]([A-Za-z0-9][A-Za-z0-9_-]{2,})$', token)
|
||
|
|
if species_prefixed:
|
||
|
|
token = species_prefixed.group(2)
|
||
|
|
token = re.sub(r'[^A-Za-z0-9]+', '', token)
|
||
|
|
if token.lower() == 'hdzip':
|
||
|
|
return 'HDZIP'
|
||
|
|
return token
|
||
|
|
|
||
|
|
|
||
|
|
def _infer_gene_family_from_text(text: str) -> str:
|
||
|
|
value = str(text or '')
|
||
|
|
if re.search(r'\bHD[-\s]?Zip\b', value, re.I):
|
||
|
|
return 'HDZIP'
|
||
|
|
for pattern in (
|
||
|
|
r'\b([A-Z][A-Za-z0-9]*(?:[- ][A-Z][A-Za-z0-9]*)?)\s+(?:gene\s+family|protein\s+family|protein\s+sequences?|proteins?)\b',
|
||
|
|
r'\b([A-Z][A-Za-z0-9]*(?:[- ][A-Z][A-Za-z0-9]*)?)\s+family\b',
|
||
|
|
):
|
||
|
|
for match in re.finditer(pattern, value):
|
||
|
|
family = _normalize_gene_family_token(match.group(1))
|
||
|
|
if family and family.lower() not in {'known', 'candidate', 'target', 'reference'}:
|
||
|
|
return family
|
||
|
|
return ''
|
||
|
|
|
||
|
|
|
||
|
|
def _infer_required_pfams_from_plan(plan: dict[str, Any]) -> str:
|
||
|
|
pfams: list[str] = []
|
||
|
|
def add_from(value: Any) -> None:
|
||
|
|
if isinstance(value, str):
|
||
|
|
for hit in re.findall(r'PF\d{5}', value):
|
||
|
|
if hit not in pfams:
|
||
|
|
pfams.append(hit)
|
||
|
|
elif isinstance(value, list):
|
||
|
|
for item in value:
|
||
|
|
add_from(item)
|
||
|
|
elif isinstance(value, dict):
|
||
|
|
for item in value.values():
|
||
|
|
add_from(item)
|
||
|
|
add_from(plan.get('method_components'))
|
||
|
|
add_from(plan.get('stages'))
|
||
|
|
add_from(plan.get('evidence'))
|
||
|
|
return ','.join(pfams)
|
||
|
|
|
||
|
|
|
||
|
|
def methodology_plan_readiness(
|
||
|
|
plan: dict[str, Any],
|
||
|
|
*,
|
||
|
|
job_id: str,
|
||
|
|
input_root: Path,
|
||
|
|
workspace_root: Path,
|
||
|
|
) -> dict[str, Any]:
|
||
|
|
plan = complete_methodology_plan_for_execution(plan)
|
||
|
|
input_dir = Path(input_root) / job_id if job_id else Path(input_root)
|
||
|
|
final_dir = Path(workspace_root) / job_id / 'final' if job_id else Path(workspace_root)
|
||
|
|
requirements = plan.get('data_requirements') if isinstance(plan.get('data_requirements'), list) else []
|
||
|
|
statuses = []
|
||
|
|
missing = []
|
||
|
|
for req in requirements:
|
||
|
|
if not isinstance(req, dict):
|
||
|
|
continue
|
||
|
|
status = _requirement_status(req, input_dir, final_dir)
|
||
|
|
statuses.append(status)
|
||
|
|
if status['status'] != 'ready':
|
||
|
|
missing.append(status)
|
||
|
|
agent_params = methodology_agent_params(plan)
|
||
|
|
suggested_stages = []
|
||
|
|
if any(item.get('requirement_type') == 'whole_genome_annotation' for item in missing):
|
||
|
|
suggested_stages.append('genome_prepare')
|
||
|
|
requested = agent_params.get('requestedMethodologyStages') or []
|
||
|
|
for stage in requested:
|
||
|
|
if stage not in suggested_stages:
|
||
|
|
suggested_stages.append(stage)
|
||
|
|
config_path = input_dir / 'config.xlsx'
|
||
|
|
samples_path = input_dir / 'samples.yaml'
|
||
|
|
return {
|
||
|
|
'success': True,
|
||
|
|
'job_id': job_id,
|
||
|
|
'plan_id': plan.get('plan_id', ''),
|
||
|
|
'ready': len(missing) == 0 and config_path.exists(),
|
||
|
|
'config_exists': config_path.exists(),
|
||
|
|
'samples_exists': samples_path.exists(),
|
||
|
|
'requirements': statuses,
|
||
|
|
'missing_requirements': missing,
|
||
|
|
'agent_params': agent_params,
|
||
|
|
'suggested_stages': suggested_stages,
|
||
|
|
'manual_gates': plan.get('manual_gates') or [],
|
||
|
|
'completion_audit': plan.get('methodology_completion_audit') or {},
|
||
|
|
'completion_summary': _methodology_completion_summary(plan),
|
||
|
|
'reproduction_targets': plan.get('reproduction_targets') or [],
|
||
|
|
'message': '方法学方案数据已就绪' if len(missing) == 0 and config_path.exists() else '方法学方案仍需补齐数据或确认 config.xlsx',
|
||
|
|
}
|
||
|
|
|
||
|
|
|
||
|
|
def load_bound_methodology_plan(input_root: Path, job_id: str) -> dict[str, Any] | None:
|
||
|
|
if not job_id:
|
||
|
|
return None
|
||
|
|
path = bound_methodology_plan_path(input_root, job_id)
|
||
|
|
if not path.exists():
|
||
|
|
return None
|
||
|
|
return load_methodology_plan(path)
|
||
|
|
|
||
|
|
|
||
|
|
def _requirement_status(req: dict[str, Any], input_dir: Path, final_dir: Path) -> dict[str, Any]:
|
||
|
|
species = str(req.get('species_name') or '')
|
||
|
|
species_for_files = _canonical_species_for_files(species, input_dir)
|
||
|
|
req_type = str(req.get('requirement_type') or '')
|
||
|
|
if req_type == 'whole_genome_annotation':
|
||
|
|
has_gff = _has_matching_file(input_dir, species_for_files, GFF_EXTENSIONS) or _has_matching_file(final_dir, species_for_files, GFF_EXTENSIONS)
|
||
|
|
has_seq = _has_matching_file(input_dir, species_for_files, WHOLE_GENOME_EXTENSIONS) or _has_matching_file(final_dir, species_for_files, WHOLE_GENOME_EXTENSIONS)
|
||
|
|
missing = []
|
||
|
|
if not has_gff:
|
||
|
|
missing.append('gff')
|
||
|
|
if not has_seq:
|
||
|
|
missing.append('genome_or_cds_or_pep')
|
||
|
|
return {
|
||
|
|
**req,
|
||
|
|
'status': 'ready' if not missing else 'missing',
|
||
|
|
'missing_files': missing,
|
||
|
|
'matched': {'gff': has_gff, 'sequence': has_seq},
|
||
|
|
'resolved_species_code': species_for_files,
|
||
|
|
}
|
||
|
|
if req_type == 'known_family_protein':
|
||
|
|
has_family = _has_matching_file(input_dir, species_for_files, WHOLE_GENOME_EXTENSIONS, family_hint=True) or _has_matching_file(final_dir, species_for_files, WHOLE_GENOME_EXTENSIONS, family_hint=True)
|
||
|
|
return {
|
||
|
|
**req,
|
||
|
|
'status': 'ready' if has_family else 'missing',
|
||
|
|
'missing_files': [] if has_family else ['family_pep_fasta'],
|
||
|
|
'matched': {'family_pep_fasta': has_family},
|
||
|
|
'resolved_species_code': species_for_files,
|
||
|
|
}
|
||
|
|
return {**req, 'status': 'unknown', 'missing_files': list(req.get('required_files') or []), 'resolved_species_code': species_for_files}
|
||
|
|
|
||
|
|
|
||
|
|
def _canonical_species_for_files(species: str, input_dir: Path) -> str:
|
||
|
|
value = str(species or '').strip()
|
||
|
|
if not value:
|
||
|
|
return value
|
||
|
|
aliases = _species_aliases_from_config(input_dir)
|
||
|
|
normalized = _normalize_species_alias(value)
|
||
|
|
return aliases.get(normalized) or value
|
||
|
|
|
||
|
|
|
||
|
|
def _species_aliases_from_config(input_dir: Path) -> dict[str, str]:
|
||
|
|
config_path = Path(input_dir) / 'config.xlsx'
|
||
|
|
if not config_path.exists():
|
||
|
|
return {}
|
||
|
|
try:
|
||
|
|
import pandas as pd
|
||
|
|
df = pd.read_excel(config_path).fillna('')
|
||
|
|
except Exception:
|
||
|
|
return {}
|
||
|
|
aliases: dict[str, str] = {}
|
||
|
|
|
||
|
|
def add(alias: Any, code: str) -> None:
|
||
|
|
key = _normalize_species_alias(str(alias or ''))
|
||
|
|
if key and code:
|
||
|
|
aliases[key] = code
|
||
|
|
|
||
|
|
for _, row in df.iterrows():
|
||
|
|
code = str(row.get('Species_Code') or row.get('Species') or '').strip()
|
||
|
|
if not code:
|
||
|
|
continue
|
||
|
|
add(code, code)
|
||
|
|
for key in ('Species', 'Species_Name', 'Scientific_Name', 'Latin_Name', 'species_name', 'scientific_name', 'latin_name'):
|
||
|
|
add(row.get(key), code)
|
||
|
|
filename = str(row.get('Original_File_Name') or '').strip()
|
||
|
|
if filename:
|
||
|
|
stem = Path(filename).name
|
||
|
|
stem = re.sub(r'\.(?:gz|zip|fa|fasta|faa|fna|ffn|gff3?|gtf|cds|pep|all|chr|toplevel|dna|protein)+$', '', stem, flags=re.I)
|
||
|
|
parts = [part for part in re.split(r'[._-]+', stem) if part]
|
||
|
|
if len(parts) >= 2:
|
||
|
|
add(' '.join(parts[:2]), code)
|
||
|
|
add('_'.join(parts[:2]), code)
|
||
|
|
return aliases
|
||
|
|
|
||
|
|
|
||
|
|
def _normalize_species_alias(value: str) -> str:
|
||
|
|
return re.sub(r'[^A-Za-z0-9]+', ' ', str(value or '')).strip().lower()
|
||
|
|
|
||
|
|
|
||
|
|
def _has_matching_file(root: Path, species: str, extensions: tuple[str, ...], *, family_hint: bool = False) -> bool:
|
||
|
|
if not root.exists():
|
||
|
|
return False
|
||
|
|
tokens = _species_tokens(species)
|
||
|
|
for path in root.rglob('*'):
|
||
|
|
if not path.is_file() or path.name.startswith('.'):
|
||
|
|
continue
|
||
|
|
lower = path.name.lower()
|
||
|
|
if not lower.endswith(extensions):
|
||
|
|
continue
|
||
|
|
if tokens and not any(token in lower for token in tokens):
|
||
|
|
if family_hint and any(word in lower for word in ('family', 'known', 'ref', 'reference')):
|
||
|
|
return True
|
||
|
|
continue
|
||
|
|
return True
|
||
|
|
return False
|
||
|
|
|
||
|
|
|
||
|
|
def _species_tokens(species: str) -> list[str]:
|
||
|
|
words = [part for part in re.split(r'[^A-Za-z0-9]+', species.lower()) if part]
|
||
|
|
tokens = set(words)
|
||
|
|
if len(words) >= 2:
|
||
|
|
tokens.add(words[0][0] + words[1])
|
||
|
|
tokens.add(words[0][:2] + words[1][:2])
|
||
|
|
return [token for token in tokens if len(token) >= 2]
|
||
|
|
|
||
|
|
|
||
|
|
def _scope_mode(prefill: dict[str, Any]) -> str:
|
||
|
|
refs = _list(prefill.get('referenceSpecies'))
|
||
|
|
comps = _list(prefill.get('comparativeSpecies'))
|
||
|
|
if refs and comps:
|
||
|
|
return 'target_plus_reference_plus_comparative'
|
||
|
|
if refs:
|
||
|
|
return 'target_plus_reference'
|
||
|
|
if comps:
|
||
|
|
return 'target_plus_comparative'
|
||
|
|
return 'target_only'
|
||
|
|
|
||
|
|
|
||
|
|
def _list(value: Any) -> list[str]:
|
||
|
|
if isinstance(value, list):
|
||
|
|
values = value
|
||
|
|
elif isinstance(value, str):
|
||
|
|
values = re.split(r'[,;，；|]+', value)
|
||
|
|
else:
|
||
|
|
values = []
|
||
|
|
return _unique([str(item).strip() for item in values if str(item).strip()])
|
||
|
|
|
||
|
|
|
||
|
|
def _unique(values: list[str]) -> list[str]:
|
||
|
|
result = []
|
||
|
|
seen = set()
|
||
|
|
for value in values:
|
||
|
|
if value and value not in seen:
|
||
|
|
result.append(value)
|
||
|
|
seen.add(value)
|
||
|
|
return result
|