355 lines
16 KiB
Python
Executable File
355 lines
16 KiB
Python
Executable File
import json
|
|
from pathlib import Path
|
|
|
|
from gfa_core.methodology_bridge import (
|
|
bind_methodology_plan,
|
|
complete_methodology_plan_for_execution,
|
|
load_bound_methodology_plan,
|
|
methodology_agent_params,
|
|
methodology_plan_readiness,
|
|
)
|
|
|
|
|
|
def _plan():
|
|
return {
|
|
'plan_id': 'methodology_plan_hdzip',
|
|
'paper_id': 'si_hdzip',
|
|
'gene_family': 'HD-Zip',
|
|
'bbav5_prefill': {
|
|
'geneFamily': 'HD-Zip',
|
|
'targetSpecies': ['Sesamum indicum'],
|
|
'identifyBlastReferenceSpecies': ['Arabidopsis thaliana'],
|
|
'referenceSpecies': ['Arabidopsis thaliana'],
|
|
'comparativeSpecies': ['Oryza sativa'],
|
|
'requiredPfams': 'PF00046,PF02183',
|
|
'scopeMode': 'target_plus_reference_plus_comparative',
|
|
},
|
|
'data_requirements': [
|
|
{'species_name': 'Sesamum indicum', 'bbav5_role': 'target', 'requirement_type': 'whole_genome_annotation'},
|
|
{'species_name': 'Arabidopsis thaliana', 'bbav5_role': 'identify_blast_reference', 'requirement_type': 'known_family_protein'},
|
|
],
|
|
'execution_routes': [{'route_id': 'methodology_default', 'stage_order': ['genome_prepare', 'identify', 'phylogeny', 'reference_classification']}],
|
|
}
|
|
|
|
|
|
def test_bind_and_readiness_reports_missing_data(tmp_path):
|
|
input_root = tmp_path / 'input'
|
|
workspace_root = tmp_path / 'workspace'
|
|
result = bind_methodology_plan(_plan(), job_id='job1', input_root=input_root, workspace_root=workspace_root, source='unit')
|
|
assert result['success'] is True
|
|
loaded = load_bound_methodology_plan(input_root, 'job1')
|
|
assert loaded['plan_id'] == 'methodology_plan_hdzip'
|
|
|
|
readiness = methodology_plan_readiness(loaded, job_id='job1', input_root=input_root, workspace_root=workspace_root)
|
|
assert readiness['ready'] is False
|
|
assert {row['requirement_type'] for row in readiness['missing_requirements']} == {'whole_genome_annotation', 'known_family_protein'}
|
|
assert readiness['agent_params']['comparativeSpecies'] == ['Oryza sativa']
|
|
assert readiness['agent_params']['identifyBlastReferenceSpecies'] == ['Arabidopsis thaliana']
|
|
|
|
|
|
def test_readiness_accepts_role_files_and_preserves_agent_params(tmp_path):
|
|
input_root = tmp_path / 'input'
|
|
workspace_root = tmp_path / 'workspace'
|
|
job_dir = input_root / 'job2'
|
|
job_dir.mkdir(parents=True)
|
|
(job_dir / 'Sesamum_indicum.gff3').write_text('##gff-version 3\n', encoding='utf-8')
|
|
(job_dir / 'Sesamum_indicum.pep.fasta').write_text('>a\nMA\n', encoding='utf-8')
|
|
(job_dir / 'Arabidopsis_thaliana.gff3').write_text('##gff-version 3\n', encoding='utf-8')
|
|
(job_dir / 'Arabidopsis_thaliana.HD-Zip.family.pep.fasta').write_text('>at\nMA\n', encoding='utf-8')
|
|
(job_dir / 'Oryza_sativa.gff3').write_text('##gff-version 3\n', encoding='utf-8')
|
|
(job_dir / 'Oryza_sativa.pep.fasta').write_text('>os\nMA\n', encoding='utf-8')
|
|
(job_dir / 'config.xlsx').write_bytes(b'placeholder')
|
|
|
|
readiness = methodology_plan_readiness(_plan(), job_id='job2', input_root=input_root, workspace_root=workspace_root)
|
|
assert readiness['ready'] is True
|
|
assert readiness['missing_requirements'] == []
|
|
params = methodology_agent_params(_plan())
|
|
assert params['scopeMode'] == 'target_plus_reference_plus_comparative'
|
|
assert params['requestedMethodologyStages'][-1] == 'reference_classification'
|
|
json.dumps(params)
|
|
|
|
|
|
def test_methodology_family_inference_uses_method_text_not_species_prefixed_paper_id():
|
|
plan = {
|
|
'plan_id': 'methodology_plan_SI-HDZIP_methodology_batch_SI-HDZIP_d9d6a3f1d47b',
|
|
'paper_id': 'SI-HDZIP',
|
|
'gene_family': '',
|
|
'bbav5_prefill': {
|
|
'targetSpecies': ['Si'],
|
|
'referenceSpecies': ['At'],
|
|
'scopeMode': 'target_plus_reference',
|
|
},
|
|
'stages': [
|
|
{
|
|
'evidence_sentence': (
|
|
'HMM profile of the homeodomain and the leucine zipper domain '
|
|
'were used to identify known HD-Zip protein sequences.'
|
|
),
|
|
'bbav5_stage_id': 'identify',
|
|
}
|
|
],
|
|
'execution_routes': [{'route_id': 'methodology_default', 'stage_order': ['identify', 'phylogeny', 'combo']}],
|
|
}
|
|
|
|
params = methodology_agent_params(plan)
|
|
|
|
assert params['geneFamily'] == 'HDZIP'
|
|
assert params['prefix'] == 'Si.HDZIP'
|
|
|
|
|
|
def test_methodology_species_completion_uses_batch_evidence_without_abbreviation_only_false_positive(tmp_path):
|
|
batch_dir = tmp_path / 'methodology_batch_Ft_spl'
|
|
batch_dir.mkdir()
|
|
(batch_dir / 'species_summary.csv').write_text(
|
|
'paper_id,family,species_name,species_role,evidence_source,observation_id,normalized_stage,evidence_sentence\n'
|
|
'Ft_SPL_2019_BMC,SPL,Arabidopsis thaliana,reference_species,method_observation,obs1,identify,A. thaliana proteins were used.\n'
|
|
'Ft_SPL_2019_BMC,SPL,Oryza sativa,reference_species,method_observation,obs2,phylogeny,O. sativa was included.\n',
|
|
encoding='utf-8',
|
|
)
|
|
(batch_dir / 'review_template.csv').write_text(
|
|
'The SPL protein sequences (A. thaliana, B. vulgaris, G. max, '
|
|
'S. lycopersicum, V. vinifera and O. sativa) for N-J phylogenetic trees '
|
|
'were downloaded from the UniProt database. '
|
|
'A. thaliana: Arabidopsis thaliana; B. vulgaris: Beta vulgaris; '
|
|
'G. max: Glycine max; O. sativa: Oryza sativa; '
|
|
'S. lycopersicum: Solanum lycopersicum; V. vinifera: Vitis vinifera; '
|
|
'Z. mays: Zea mays L;\n',
|
|
encoding='utf-8',
|
|
)
|
|
plan = {
|
|
'plan_id': 'methodology_plan_ft_spl',
|
|
'paper_id': 'Ft_SPL_2019_BMC',
|
|
'gene_family': 'SPL',
|
|
'source_batch_dir': str(batch_dir),
|
|
'bbav5_prefill': {'targetSpecies': ['Fagopyrum tataricum']},
|
|
'data_requirements': [
|
|
{'species_name': 'Fagopyrum tataricum', 'bbav5_role': 'target', 'requirement_type': 'whole_genome_annotation'},
|
|
],
|
|
'execution_routes': [{'route_id': 'methodology_default', 'stage_order': ['identify', 'phylogeny']}],
|
|
}
|
|
|
|
params = methodology_agent_params(plan)
|
|
refs = set(params['referenceSpecies'])
|
|
|
|
assert 'Arabidopsis thaliana' in refs
|
|
assert 'Oryza sativa' in refs
|
|
assert 'Beta vulgaris' in refs
|
|
assert 'Glycine max' in refs
|
|
assert 'Solanum lycopersicum' in refs
|
|
assert 'Vitis vinifera' in refs
|
|
assert 'Zea mays' not in refs
|
|
|
|
readiness = methodology_plan_readiness(plan, job_id='job_ft', input_root=tmp_path / 'input', workspace_root=tmp_path / 'workspace')
|
|
missing_species = {row['species_name'] for row in readiness['missing_requirements']}
|
|
assert {'Fagopyrum tataricum', 'Arabidopsis thaliana', 'Oryza sativa', 'Beta vulgaris'} <= missing_species
|
|
assert 'Zea mays' not in missing_species
|
|
|
|
|
|
def test_methodology_species_completion_corrects_bad_prefill_roles_from_batch_evidence(tmp_path):
|
|
batch_dir = tmp_path / 'methodology_batch_kq_spl'
|
|
batch_dir.mkdir()
|
|
(batch_dir / 'species_summary.csv').write_text(
|
|
'paper_id,family,species_name,species_role,evidence_source,observation_id,normalized_stage,evidence_sentence\n'
|
|
'kq.spl,SPL,Fagopyrum tataricum,target_species,title,,,Genome-wide identification of SPL genes in Tartary Buckwheat\n'
|
|
'kq.spl,SPL,Zea mays,target_species,method_observation,obs_qpcr,qRT-PCR,'
|
|
'Using tartary buckwheat histone H3 as an internal reference gene; '
|
|
'Z. mays: Zea mays L; Results: 24 Fagopyrum tataricum SPL genes were identified.\n',
|
|
encoding='utf-8',
|
|
)
|
|
(batch_dir / 'review_template.csv').write_text(
|
|
'The SPL protein sequences (A. thaliana, B. vulgaris, G. max, '
|
|
'S. lycopersicum, V. vinifera and O. sativa) for N-J phylogenetic trees '
|
|
'were downloaded from the UniProt database. '
|
|
'A. thaliana: Arabidopsis thaliana; Z. mays: Zea mays L;\n',
|
|
encoding='utf-8',
|
|
)
|
|
bad_targets = [
|
|
'Beta vulgaris',
|
|
'Fagopyrum tataricum',
|
|
'Glycine max',
|
|
'Solanum lycopersicum',
|
|
'Vitis vinifera',
|
|
'Zea mays',
|
|
]
|
|
plan = {
|
|
'plan_id': 'methodology_plan_kq_spl_bad_roles',
|
|
'paper_id': 'kq.spl',
|
|
'gene_family': 'SPL',
|
|
'source_batch_dir': str(batch_dir),
|
|
'bbav5_prefill': {
|
|
'targetSpecies': bad_targets,
|
|
'referenceSpecies': [],
|
|
'comparativeSpecies': [],
|
|
'scopeMode': 'target_only',
|
|
},
|
|
'data_requirements': [
|
|
{'species_name': species, 'bbav5_role': 'target', 'requirement_type': 'whole_genome_annotation'}
|
|
for species in bad_targets
|
|
],
|
|
}
|
|
|
|
completed = complete_methodology_plan_for_execution(plan)
|
|
params = methodology_agent_params(completed)
|
|
requirement_roles = {
|
|
row['species_name']: row['bbav5_role']
|
|
for row in completed['data_requirements']
|
|
if row.get('requirement_type') == 'whole_genome_annotation'
|
|
}
|
|
|
|
assert params['targetSpecies'] == ['Fagopyrum tataricum']
|
|
assert set(params['referenceSpecies']) == {
|
|
'Arabidopsis thaliana',
|
|
'Beta vulgaris',
|
|
'Glycine max',
|
|
'Oryza sativa',
|
|
'Solanum lycopersicum',
|
|
'Vitis vinifera',
|
|
}
|
|
assert 'Zea mays' not in params['targetSpecies']
|
|
assert 'Zea mays' not in params['referenceSpecies']
|
|
assert requirement_roles['Fagopyrum tataricum'] == 'target'
|
|
assert requirement_roles['Arabidopsis thaliana'] == 'reference'
|
|
assert requirement_roles['Oryza sativa'] == 'reference'
|
|
assert 'Zea mays' not in requirement_roles
|
|
|
|
|
|
def test_methodology_completion_paradigm_covers_stages_pfams_and_audit():
|
|
plan = {
|
|
'plan_id': 'methodology_plan_spl_sparse',
|
|
'paper_id': 'Ft_SPL_2019_BMC',
|
|
'gene_family': 'SPL',
|
|
'bbav5_prefill': {'targetSpecies': ['Fagopyrum tataricum']},
|
|
'method_components': [
|
|
{
|
|
'normalized_stage': 'gene_family_identification',
|
|
'evidence_sentence': 'The SBP domain profile PF03110 was downloaded from Pfam.',
|
|
},
|
|
{
|
|
'normalized_stage': 'promoter_analysis',
|
|
'evidence_sentence': 'Cis-elements in promoter sequences were analyzed.',
|
|
},
|
|
],
|
|
}
|
|
|
|
completed = complete_methodology_plan_for_execution(plan)
|
|
params = methodology_agent_params(completed)
|
|
audit = completed['methodology_completion_audit']
|
|
|
|
assert params['requiredPfams'] == 'PF03110'
|
|
assert completed['execution_routes'][0]['stage_order'] == ['identify', 'promoter']
|
|
assert audit['paradigm_version'] == 'methodology_evidence_completion.v2'
|
|
assert 'pfam_evidence_completion' in audit['applied_rules']
|
|
assert 'stage_evidence_completion' in audit['applied_rules']
|
|
assert any(row['object_type'] == 'domain_requirement' for row in audit['records'])
|
|
assert any(row['object_type'] == 'stage' and row['value'] == 'promoter' and row['evidence_text'] for row in audit['records'])
|
|
assert 'stage' in audit['candidate_types']
|
|
|
|
|
|
def test_methodology_completion_adds_accession_gate_figure_targets_and_readiness_audit(tmp_path):
|
|
plan = {
|
|
'plan_id': 'methodology_plan_expression',
|
|
'paper_id': 'expr_paper',
|
|
'gene_family': 'SPL',
|
|
'bbav5_prefill': {'targetSpecies': ['Fagopyrum tataricum']},
|
|
'method_components': [
|
|
{
|
|
'normalized_stage': 'expression_analysis',
|
|
'evidence_sentence': (
|
|
'RNA-seq raw data were downloaded from NCBI SRA under BioProject PRJNA530826. '
|
|
'Figure 2: Expression heatmap of SPL genes under salt stress. '
|
|
'Figure 7: Protein interaction network of SPL genes. '
|
|
'Normalized expression matrix GSE12345 with FPKM values was used. '
|
|
'The SPL protein sequences were downloaded from the UniProt database. '
|
|
'Reference-only identifier PRJNA999999 was listed in a citation note.'
|
|
),
|
|
}
|
|
],
|
|
}
|
|
|
|
completed = complete_methodology_plan_for_execution(plan)
|
|
gates = completed['manual_gates']
|
|
targets = completed['reproduction_targets']
|
|
data_requirements = completed['data_requirements']
|
|
audit = completed['methodology_completion_audit']
|
|
|
|
assert [gate['accession'] for gate in gates] == ['PRJNA530826']
|
|
assert {req.get('accession') for req in data_requirements if req.get('requirement_type') == 'expression_matrix'} == {'GSE12345'}
|
|
assert {target['label'] for target in targets} == {'Figure 2', 'Figure 7'}
|
|
figure2 = next(target for target in targets if target['label'] == 'Figure 2')
|
|
figure7 = next(target for target in targets if target['label'] == 'Figure 7')
|
|
assert figure2['status'] == 'planning_target_only'
|
|
assert figure2['suggested_stage'] == 'expression'
|
|
assert figure2['coverage_status'] == 'covered_by_stage'
|
|
assert figure7['suggested_stage'] == 'ppi'
|
|
assert figure7['coverage_status'] == 'needs_method_inference_or_default_tool'
|
|
assert 'accession_manual_gate_completion' in audit['applied_rules']
|
|
assert 'expression_matrix_requirement_completion' in audit['applied_rules']
|
|
assert 'figure_table_target_completion' in audit['applied_rules']
|
|
assert audit['accepted_candidate_count'] >= 3
|
|
assert audit['rejected_candidate_count'] >= 2
|
|
assert any(row['value'] == 'PRJNA999999' for row in audit['rejected_candidates'])
|
|
assert any(row['object_type'] == 'reference_database' and row['value'] == 'UniProt' for row in audit['rejected_candidates'])
|
|
|
|
readiness = methodology_plan_readiness(completed, job_id='job_expr', input_root=tmp_path / 'input', workspace_root=tmp_path / 'workspace')
|
|
assert readiness['completion_audit']['paradigm_version'] == 'methodology_evidence_completion.v2'
|
|
assert {target['label'] for target in readiness['reproduction_targets']} == {'Figure 2', 'Figure 7'}
|
|
assert readiness['manual_gates'][0]['accession'] == 'PRJNA530826'
|
|
assert readiness['completion_summary']['manual_gate_count'] == 1
|
|
assert readiness['completion_summary']['reproduction_target_count'] == 2
|
|
assert readiness['completion_summary']['covered_reproduction_target_count'] == 1
|
|
|
|
|
|
def test_methodology_completion_is_idempotent_and_preserves_audit_records():
|
|
plan = {
|
|
'plan_id': 'methodology_plan_idempotent',
|
|
'paper_id': 'idem_paper',
|
|
'gene_family': 'SPL',
|
|
'bbav5_prefill': {'targetSpecies': ['Fagopyrum tataricum']},
|
|
'method_components': [
|
|
{
|
|
'normalized_stage': 'gene_family_identification',
|
|
'evidence_sentence': 'The SBP domain profile PF03110 was used for identification.',
|
|
},
|
|
{
|
|
'normalized_stage': 'motif_analysis',
|
|
'evidence_sentence': 'Figure 3: Conserved motif distribution of SPL proteins.',
|
|
},
|
|
],
|
|
}
|
|
|
|
once = complete_methodology_plan_for_execution(plan)
|
|
twice = complete_methodology_plan_for_execution(once)
|
|
once_audit = once['methodology_completion_audit']
|
|
twice_audit = twice['methodology_completion_audit']
|
|
|
|
assert twice['bbav5_prefill']['requiredPfams'] == once['bbav5_prefill']['requiredPfams']
|
|
assert twice['execution_routes'] == once['execution_routes']
|
|
assert twice['reproduction_targets'] == once['reproduction_targets']
|
|
assert twice_audit['completion_run_count'] == 2
|
|
assert twice_audit['records_preserved_from_previous_completion'] == len(once_audit['records'])
|
|
assert len(twice_audit['records']) == len(once_audit['records'])
|
|
|
|
|
|
def test_subfamily_pipeline_scope_keeps_comparative_species():
|
|
from gfa_core.mas.agents.planning_v2 import PlanningAgentV2
|
|
|
|
agent = object.__new__(PlanningAgentV2)
|
|
stages = [
|
|
{'id': 'phylogeny', 'params': {}},
|
|
{'id': 'reference_classification', 'params': {}},
|
|
]
|
|
params = {
|
|
'prefix': 'Sesamum.HD-Zip',
|
|
'geneFamily': 'HD-Zip',
|
|
'targetSpecies': ['Sesamum indicum'],
|
|
'referenceSpecies': ['Arabidopsis thaliana'],
|
|
'comparativeSpecies': ['Oryza sativa'],
|
|
'subfamilyPipeline': True,
|
|
}
|
|
agent._apply_reference_classification_evidence_stage_details(stages, params)
|
|
|
|
for stage in stages:
|
|
stage_params = stage['params']
|
|
assert stage_params['comparativeSpecies'] == ['Oryza sativa']
|
|
assert stage_params['scopeMode'] == 'target_plus_reference_plus_comparative'
|