364 lines
17 KiB
Python
Executable File
364 lines
17 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
'''Build an interactive HTML review page for PlantGeneBench-100 questions.'''
|
||
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import json
|
||
from collections import Counter
|
||
from datetime import datetime
|
||
from html import escape
|
||
from pathlib import Path
|
||
from typing import Any
|
||
|
||
|
||
def load_jsonl(path: Path) -> list[dict[str, Any]]:
|
||
rows = []
|
||
with path.open('r', encoding='utf-8') as handle:
|
||
for lineno, line in enumerate(handle, start=1):
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
try:
|
||
rows.append(json.loads(line))
|
||
except json.JSONDecodeError as exc:
|
||
raise SystemExit(f'{path}:{lineno}: invalid JSON: {exc}') from exc
|
||
return rows
|
||
|
||
|
||
def pretty(value: Any) -> str:
|
||
if value in (None, '', [], {}):
|
||
return ''
|
||
return json.dumps(value, ensure_ascii=False, indent=2, sort_keys=True)
|
||
|
||
|
||
def inline_list(value: Any) -> str:
|
||
if not value:
|
||
return ''
|
||
if isinstance(value, list):
|
||
return ', '.join(str(v) for v in value)
|
||
return str(value)
|
||
|
||
|
||
def item_html(task: dict[str, Any], gold_row: dict[str, Any], index: int) -> str:
|
||
gold = gold_row.get('gold') or {}
|
||
task_id = task.get('task_id', '')
|
||
layer = task.get('layer') or task.get('category') or 'unknown'
|
||
difficulty = task.get('difficulty', '')
|
||
stages = gold.get('target_stages') or gold.get('expected_stages') or []
|
||
plan = gold.get('execution_plan') or gold.get('expected_execution_plan') or []
|
||
entities = gold.get('entities') or {}
|
||
params = gold.get('parameters') or gold.get('required_params') or {}
|
||
artifacts = gold.get('artifacts') or {}
|
||
must_not = gold.get('must_not_do') or []
|
||
must_include = gold.get('must_include') or []
|
||
can_execute = gold.get('can_execute')
|
||
support_status = gold.get('support_status')
|
||
boundary_action = gold.get('boundary_action')
|
||
scoring_profile = gold_row.get('scoring_profile') or task.get('scoring_profile') or []
|
||
full_task = pretty(task)
|
||
full_gold = pretty(gold)
|
||
must_not_html = ''.join(f'<li>{escape(str(x))}</li>' for x in must_not) or '<li class="muted">无</li>'
|
||
must_include_html = ''.join(f'<li>{escape(str(x))}</li>' for x in must_include) or '<li class="muted">无</li>'
|
||
return f'''
|
||
<section class="card" data-task-id="{escape(task_id)}" data-layer="{escape(layer)}" data-difficulty="{escape(difficulty)}">
|
||
<div class="card-head">
|
||
<div>
|
||
<h2><span class="idx">{index:03d}</span> {escape(task_id)}</h2>
|
||
<p class="meta"><span>{escape(layer)}</span><span>{escape(difficulty)}</span><span>scoring: {escape(inline_list(scoring_profile))}</span></p>
|
||
</div>
|
||
<a class="anchor" href="#{escape(task_id)}" id="{escape(task_id)}">#</a>
|
||
</div>
|
||
|
||
<div class="question-block">
|
||
<h3>题目正文</h3>
|
||
<p class="question">{escape(task.get('user_query', ''))}</p>
|
||
</div>
|
||
|
||
<div class="grid">
|
||
<div class="panel">
|
||
<h3>Gold 摘要</h3>
|
||
<dl>
|
||
<dt>Intent</dt><dd><code>{escape(str(gold.get('intent') or ''))}</code></dd>
|
||
<dt>Analysis type</dt><dd><code>{escape(str(gold.get('analysis_type') or ''))}</code></dd>
|
||
<dt>Entities</dt><dd><pre>{escape(pretty(entities))}</pre></dd>
|
||
<dt>Target stages</dt><dd><code>{escape(inline_list(stages))}</code></dd>
|
||
<dt>Execution plan</dt><dd><code>{escape(inline_list(plan))}</code></dd>
|
||
<dt>Parameters</dt><dd><pre>{escape(pretty(params))}</pre></dd>
|
||
<dt>Artifacts</dt><dd><pre>{escape(pretty(artifacts))}</pre></dd>
|
||
<dt>Can execute</dt><dd><code>{escape(str(can_execute))}</code></dd>
|
||
<dt>Support status</dt><dd><code>{escape(str(support_status))}</code></dd>
|
||
<dt>Boundary action</dt><dd><code>{escape(str(boundary_action))}</code></dd>
|
||
</dl>
|
||
</div>
|
||
|
||
<div class="panel review-panel">
|
||
<h3>修订意见</h3>
|
||
<label>题目是否需要修订
|
||
<select data-field="status" data-task="{escape(task_id)}">
|
||
<option value="">未选择</option>
|
||
<option value="keep">保留</option>
|
||
<option value="revise_minor">小修</option>
|
||
<option value="revise_major">大修</option>
|
||
<option value="replace">替换</option>
|
||
<option value="remove">删除</option>
|
||
</select>
|
||
</label>
|
||
<label>题目修订意见
|
||
<textarea data-field="question_note" data-task="{escape(task_id)}" placeholder="例如：题面缺少输入文件；物种/家族不清楚；难度不合理；更适合作为边界控制题……"></textarea>
|
||
</label>
|
||
<label>建议改写后的题目
|
||
<textarea data-field="revised_question" data-task="{escape(task_id)}" placeholder="可直接写新的题目正文。"></textarea>
|
||
</label>
|
||
<label>Gold / 评分标准修订意见
|
||
<textarea data-field="gold_note" data-task="{escape(task_id)}" placeholder="例如：target_stages 应改为 identify,motif；must-not-do 不合理；can_execute 应为 false……"></textarea>
|
||
</label>
|
||
<label>优先级
|
||
<select data-field="priority" data-task="{escape(task_id)}">
|
||
<option value="">未选择</option>
|
||
<option value="low">low</option>
|
||
<option value="medium">medium</option>
|
||
<option value="high">high</option>
|
||
</select>
|
||
</label>
|
||
</div>
|
||
</div>
|
||
|
||
<div class="notes">
|
||
<div>
|
||
<h3>Must-not-do</h3>
|
||
<ul>{must_not_html}</ul>
|
||
</div>
|
||
<div>
|
||
<h3>Must-include</h3>
|
||
<ul>{must_include_html}</ul>
|
||
</div>
|
||
</div>
|
||
|
||
<details>
|
||
<summary>查看 rationale / input_context / 完整 JSON</summary>
|
||
<h4>Rationale</h4>
|
||
<pre>{escape(str(gold.get('rationale') or ''))}</pre>
|
||
<h4>Input context</h4>
|
||
<pre>{escape(pretty(task.get('input_context') or {}))}</pre>
|
||
<h4>Full task JSON</h4>
|
||
<pre>{escape(full_task)}</pre>
|
||
<h4>Full gold JSON</h4>
|
||
<pre>{escape(full_gold)}</pre>
|
||
</details>
|
||
</section>
|
||
'''
|
||
|
||
|
||
def build_html(tasks: list[dict[str, Any]], gold_rows: list[dict[str, Any]], task_path: Path, gold_path: Path) -> str:
|
||
gold_by_id = {row['task_id']: row for row in gold_rows}
|
||
layers = Counter(task.get('layer') or task.get('category') or 'unknown' for task in tasks)
|
||
difficulties = Counter(task.get('difficulty', 'unknown') for task in tasks)
|
||
cards = []
|
||
for index, task in enumerate(tasks, start=1):
|
||
task_id = task.get('task_id')
|
||
gold_row = gold_by_id.get(task_id) or {'task_id': task_id, 'gold': {}, 'layer': task.get('layer')}
|
||
cards.append(item_html(task, gold_row, index))
|
||
data_json = json.dumps([task.get('task_id') for task in tasks], ensure_ascii=False)
|
||
generated = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
|
||
layer_options = ''.join(f'<option value="{escape(k)}">{escape(k)} ({v})</option>' for k, v in sorted(layers.items()))
|
||
diff_options = ''.join(f'<option value="{escape(k)}">{escape(k)} ({v})</option>' for k, v in sorted(difficulties.items()))
|
||
layer_summary = escape(', '.join(f'{k}={v}' for k, v in sorted(layers.items())))
|
||
diff_summary = escape(', '.join(f'{k}={v}' for k, v in sorted(difficulties.items())))
|
||
cards_html = ''.join(cards)
|
||
return f'''<!doctype html>
|
||
<html lang="zh-CN">
|
||
<head>
|
||
<meta charset="utf-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||
<title>PlantGeneBench-100 题库修订审阅</title>
|
||
<style>
|
||
:root {{ color-scheme: light; --bg:#f5f6f8; --panel:#ffffff; --ink:#1e2329; --muted:#667085; --line:#d9dee7; --accent:#1f6feb; }}
|
||
* {{ box-sizing:border-box; }}
|
||
body {{ margin:0; background:var(--bg); color:var(--ink); font-family:Arial,"Microsoft YaHei",sans-serif; line-height:1.55; }}
|
||
header {{ position:sticky; top:0; z-index:10; background:#fff; border-bottom:1px solid var(--line); padding:14px 22px; box-shadow:0 2px 10px rgba(16,24,40,.05); }}
|
||
h1 {{ margin:0 0 6px; font-size:22px; }}
|
||
.meta, .muted {{ color:var(--muted); font-size:13px; }}
|
||
.toolbar {{ display:flex; flex-wrap:wrap; gap:10px; align-items:center; margin-top:10px; }}
|
||
.toolbar input, .toolbar select, button {{ height:34px; border:1px solid var(--line); border-radius:6px; background:#fff; padding:0 10px; }}
|
||
.toolbar input {{ min-width:260px; }}
|
||
button {{ cursor:pointer; color:#111827; }}
|
||
button.primary {{ background:var(--accent); color:#fff; border-color:var(--accent); }}
|
||
main {{ max-width:1280px; margin:20px auto 64px; padding:0 18px; }}
|
||
.summary {{ display:grid; grid-template-columns:repeat(4,minmax(0,1fr)); gap:10px; margin-bottom:16px; }}
|
||
.summary div {{ background:#fff; border:1px solid var(--line); border-radius:8px; padding:10px 12px; }}
|
||
.card {{ background:var(--panel); border:1px solid var(--line); border-radius:8px; margin:14px 0; padding:16px; }}
|
||
.card-head {{ display:flex; justify-content:space-between; gap:12px; border-bottom:1px solid #edf0f5; padding-bottom:10px; margin-bottom:12px; }}
|
||
h2 {{ margin:0; font-size:20px; }}
|
||
h3 {{ margin:0 0 8px; font-size:15px; }}
|
||
h4 {{ margin:14px 0 6px; }}
|
||
.idx {{ color:var(--muted); font-weight:400; }}
|
||
.anchor {{ color:var(--muted); text-decoration:none; font-size:18px; }}
|
||
.meta span {{ display:inline-block; margin-right:8px; padding:2px 7px; background:#f2f4f7; border-radius:999px; }}
|
||
.question-block {{ background:#f8fafc; border:1px solid #e6eaf0; border-radius:8px; padding:12px; margin-bottom:12px; }}
|
||
.question {{ margin:0; font-size:16px; white-space:pre-wrap; }}
|
||
.grid {{ display:grid; grid-template-columns:minmax(0,1.08fr) minmax(360px,.92fr); gap:14px; align-items:start; }}
|
||
.panel {{ border:1px solid #e6eaf0; border-radius:8px; padding:12px; background:#fff; }}
|
||
dl {{ display:grid; grid-template-columns:140px minmax(0,1fr); gap:6px 10px; margin:0; }}
|
||
dt {{ color:var(--muted); }}
|
||
dd {{ margin:0; min-width:0; overflow-wrap:anywhere; }}
|
||
code {{ background:#eef4ff; padding:1px 5px; border-radius:4px; }}
|
||
pre {{ white-space:pre-wrap; overflow:auto; background:#f8fafc; border:1px solid #e6eaf0; border-radius:6px; padding:8px; margin:4px 0; max-height:360px; }}
|
||
.review-panel label {{ display:block; font-size:13px; color:#344054; margin:10px 0; }}
|
||
.review-panel textarea, .review-panel select {{ display:block; width:100%; margin-top:4px; border:1px solid var(--line); border-radius:6px; padding:8px; font:inherit; background:#fff; }}
|
||
.review-panel textarea {{ min-height:82px; resize:vertical; }}
|
||
.notes {{ display:grid; grid-template-columns:1fr 1fr; gap:14px; margin:12px 0; }}
|
||
.notes > div {{ border:1px solid #e6eaf0; border-radius:8px; padding:12px; background:#fff; }}
|
||
ul {{ margin:0; padding-left:20px; }}
|
||
details {{ margin-top:12px; }}
|
||
summary {{ cursor:pointer; color:var(--accent); }}
|
||
.hidden {{ display:none !important; }}
|
||
.dirty {{ outline:2px solid rgba(31,111,235,.18); }}
|
||
@media (max-width:900px) {{ .summary, .grid, .notes {{ grid-template-columns:1fr; }} header {{ position:static; }} .toolbar input {{ min-width:0; width:100%; }} }}
|
||
</style>
|
||
</head>
|
||
<body>
|
||
<header>
|
||
<h1>PlantGeneBench-100 题库修订审阅</h1>
|
||
<div class="meta">Generated: {escape(generated)} | tasks=<code>{escape(str(task_path))}</code> | gold=<code>{escape(str(gold_path))}</code></div>
|
||
<div class="toolbar">
|
||
<input id="search" type="search" placeholder="搜索 ID、题面、stage、gold...">
|
||
<select id="layer"><option value="">全部 layer</option>{layer_options}</select>
|
||
<select id="difficulty"><option value="">全部难度</option>{diff_options}</select>
|
||
<select id="reviewStatus"><option value="">全部修订状态</option><option value="filled">已填写意见</option><option value="empty">未填写意见</option></select>
|
||
<button id="save" class="primary">保存到浏览器</button>
|
||
<button id="exportJson">导出 JSON</button>
|
||
<button id="exportTsv">导出 TSV</button>
|
||
<button id="clear">清空意见</button>
|
||
</div>
|
||
</header>
|
||
<main>
|
||
<section class="summary">
|
||
<div><b>题目数</b><br>{len(tasks)}</div>
|
||
<div><b>Layer</b><br>{layer_summary}</div>
|
||
<div><b>难度</b><br>{diff_summary}</div>
|
||
<div><b>提示</b><br>输入会自动保存在浏览器 localStorage；正式提交请导出 JSON 或 TSV。</div>
|
||
</section>
|
||
{cards_html}
|
||
</main>
|
||
<script>
|
||
const TASK_IDS = {data_json};
|
||
const STORAGE_KEY = 'plantgenebench_100_v1_review_notes';
|
||
function loadNotes() {{
|
||
try {{ return JSON.parse(localStorage.getItem(STORAGE_KEY) || '{{}}'); }} catch(e) {{ return {{}}; }}
|
||
}}
|
||
function collectNotes() {{
|
||
const notes = loadNotes();
|
||
document.querySelectorAll('[data-task][data-field]').forEach(el => {{
|
||
const id = el.dataset.task;
|
||
const field = el.dataset.field;
|
||
notes[id] = notes[id] || {{ task_id:id }};
|
||
notes[id][field] = el.value || '';
|
||
}});
|
||
return notes;
|
||
}}
|
||
function saveNotes() {{
|
||
const notes = collectNotes();
|
||
localStorage.setItem(STORAGE_KEY, JSON.stringify(notes));
|
||
updateSummaryState();
|
||
}}
|
||
function restoreNotes() {{
|
||
const notes = loadNotes();
|
||
document.querySelectorAll('[data-task][data-field]').forEach(el => {{
|
||
const row = notes[el.dataset.task] || {{}};
|
||
el.value = row[el.dataset.field] || '';
|
||
}});
|
||
updateSummaryState();
|
||
}}
|
||
function hasNote(id) {{
|
||
const n = collectNotes()[id] || {{}};
|
||
return ['status','question_note','revised_question','gold_note','priority'].some(k => (n[k] || '').trim());
|
||
}}
|
||
function updateSummaryState() {{
|
||
document.querySelectorAll('.card').forEach(card => {{
|
||
card.classList.toggle('dirty', hasNote(card.dataset.taskId));
|
||
}});
|
||
}}
|
||
function applyFilters() {{
|
||
const q = document.getElementById('search').value.trim().toLowerCase();
|
||
const layer = document.getElementById('layer').value;
|
||
const diff = document.getElementById('difficulty').value;
|
||
const status = document.getElementById('reviewStatus').value;
|
||
document.querySelectorAll('.card').forEach(card => {{
|
||
const text = card.innerText.toLowerCase();
|
||
let ok = true;
|
||
if (q && !text.includes(q)) ok = false;
|
||
if (layer && card.dataset.layer !== layer) ok = false;
|
||
if (diff && card.dataset.difficulty !== diff) ok = false;
|
||
if (status === 'filled' && !hasNote(card.dataset.taskId)) ok = false;
|
||
if (status === 'empty' && hasNote(card.dataset.taskId)) ok = false;
|
||
card.classList.toggle('hidden', !ok);
|
||
}});
|
||
}}
|
||
function download(filename, content, mime) {{
|
||
const blob = new Blob([content], {{ type:mime }});
|
||
const url = URL.createObjectURL(blob);
|
||
const a = document.createElement('a');
|
||
a.href = url; a.download = filename; document.body.appendChild(a); a.click(); a.remove();
|
||
URL.revokeObjectURL(url);
|
||
}}
|
||
function exportJson() {{
|
||
const notes = collectNotes();
|
||
const rows = TASK_IDS.map(id => notes[id] || {{task_id:id}}).filter(row => Object.entries(row).some(([k,v]) => k !== 'task_id' && String(v || '').trim()));
|
||
download('plantgenebench_100_v1_revision_notes.json', JSON.stringify(rows, null, 2), 'application/json;charset=utf-8');
|
||
}}
|
||
function tsvEscape(v) {{ return String(v || '').replace(/ /g, ' ').replace(/
|
||
?
|
||
/g, ' '); }}
|
||
function exportTsv() {{
|
||
const notes = collectNotes();
|
||
const fields = ['task_id','status','priority','question_note','revised_question','gold_note'];
|
||
const rows = [fields.join(' ')];
|
||
TASK_IDS.forEach(id => {{
|
||
const row = notes[id] || {{task_id:id}};
|
||
if (fields.some(f => f !== 'task_id' && String(row[f] || '').trim())) rows.push(fields.map(f => tsvEscape(row[f] || (f === 'task_id' ? id : ''))).join(' '));
|
||
}});
|
||
download('plantgenebench_100_v1_revision_notes.tsv', rows.join('
|
||
') + '
|
||
', 'text/tab-separated-values;charset=utf-8');
|
||
}}
|
||
function clearNotes() {{
|
||
if (!confirm('确认清空当前浏览器中保存的所有修订意见？')) return;
|
||
localStorage.removeItem(STORAGE_KEY);
|
||
restoreNotes();
|
||
applyFilters();
|
||
}}
|
||
document.querySelectorAll('[data-task][data-field]').forEach(el => {{
|
||
el.addEventListener('input', () => {{ saveNotes(); applyFilters(); }});
|
||
el.addEventListener('change', () => {{ saveNotes(); applyFilters(); }});
|
||
}});
|
||
['search','layer','difficulty','reviewStatus'].forEach(id => document.getElementById(id).addEventListener('input', applyFilters));
|
||
document.getElementById('save').addEventListener('click', saveNotes);
|
||
document.getElementById('exportJson').addEventListener('click', exportJson);
|
||
document.getElementById('exportTsv').addEventListener('click', exportTsv);
|
||
document.getElementById('clear').addEventListener('click', clearNotes);
|
||
restoreNotes();
|
||
applyFilters();
|
||
</script>
|
||
</body>
|
||
</html>
|
||
'''
|
||
|
||
|
||
def run() -> int:
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
parser.add_argument('--tasks', type=Path, default=Path('benchmarks/plantgenebench/tasks/plantgenebench_100_v1.full.jsonl'))
|
||
parser.add_argument('--gold', type=Path, default=Path('benchmarks/plantgenebench/gold/plantgenebench_100_v1.full.gold.jsonl'))
|
||
parser.add_argument('--out', type=Path, default=Path('benchmarks/plantgenebench/plantgenebench_100_v1_review.html'))
|
||
args = parser.parse_args()
|
||
tasks = load_jsonl(args.tasks)
|
||
gold_rows = load_jsonl(args.gold)
|
||
html = build_html(tasks, gold_rows, args.tasks, args.gold)
|
||
args.out.parent.mkdir(parents=True, exist_ok=True)
|
||
args.out.write_text(html, encoding='utf-8')
|
||
print(f'Wrote {len(tasks)} review items to {args.out}')
|
||
return 0
|
||
|
||
|
||
if __name__ == '__main__':
|
||
raise SystemExit(run())
|