113 lines
3.9 KiB
Python
113 lines
3.9 KiB
Python
"""Batch-judge all completed runs of an experiment with the LLM judge.
|
|
|
|
Usage: python judge_all.py <experiment_id> [judge_model] [concurrency]
|
|
|
|
For every done run: the judge decides detection + false alarms (semantic,
|
|
level-agnostic), while line coverage is computed rule-based from parsed
|
|
findings. Idempotent: runs whose result already carries a judge verdict
|
|
(judge_verdict key in parsed_findings) are skipped.
|
|
"""
|
|
|
|
import asyncio
|
|
import sys
|
|
import uuid
|
|
|
|
from app.analysis.judge import (
|
|
build_judge_prompt,
|
|
coverage_from_verdict,
|
|
parse_verdict,
|
|
verdict_to_metrics,
|
|
)
|
|
from app.analysis.parser import parse_output
|
|
from app.db import SessionLocal, init_db
|
|
from app.model_adapters.factory import create_adapter
|
|
from app.models import ExperimentRun
|
|
|
|
|
|
async def main() -> None:
|
|
experiment_id = uuid.UUID(sys.argv[1])
|
|
judge_model = sys.argv[2] if len(sys.argv) > 2 else "deepseek"
|
|
concurrency = int(sys.argv[3]) if len(sys.argv) > 3 else 5
|
|
|
|
init_db()
|
|
db = SessionLocal()
|
|
adapter = create_adapter(judge_model)
|
|
|
|
runs = (
|
|
db.query(ExperimentRun)
|
|
.filter(ExperimentRun.experiment_id == experiment_id, ExperimentRun.status == "done")
|
|
.all()
|
|
)
|
|
todo = []
|
|
for run in runs:
|
|
if not run.result or not run.result.raw_output:
|
|
continue
|
|
pf = run.result.parsed_findings
|
|
if isinstance(pf, dict) and pf.get("judge_verdict"):
|
|
continue
|
|
todo.append(run)
|
|
print(f"{len(todo)} runs to judge (judge={judge_model}, concurrency={concurrency})")
|
|
|
|
semaphore = asyncio.Semaphore(concurrency)
|
|
|
|
async def judge_one(run):
|
|
async with semaphore:
|
|
gt_list = [
|
|
{
|
|
"defect_type": d.defect_type,
|
|
"line_start": d.line_start,
|
|
"line_end": d.line_end,
|
|
"description": d.description,
|
|
"reference_fix": d.reference_fix,
|
|
}
|
|
for d in run.sample.defects
|
|
]
|
|
gt = gt_list[0] if gt_list else {}
|
|
prompt = build_judge_prompt(run.result.raw_output, gt)
|
|
last_err = None
|
|
for _ in range(3):
|
|
try:
|
|
resp = await adapter.chat(prompt, {"temperature": 0.0, "max_tokens": 2048})
|
|
verdict = parse_verdict(resp.text)
|
|
if verdict:
|
|
break
|
|
except Exception as e: # noqa: BLE001
|
|
last_err = e
|
|
await asyncio.sleep(2)
|
|
else:
|
|
print(f"WARN judge failed for run {run.id}: {last_err or 'unparseable verdict'}")
|
|
return None
|
|
|
|
metrics = verdict_to_metrics(verdict, n_ground_truth=len(gt_list) or 1)
|
|
coverage = coverage_from_verdict(verdict, gt_list)
|
|
level = run.template_version.template.level
|
|
findings = parse_output(run.result.raw_output, level)
|
|
|
|
run.result.detection_rate = metrics["detection_rate"]
|
|
run.result.false_positive_rate = metrics["false_positive_rate"]
|
|
run.result.coverage_rate = coverage
|
|
run.result.parsed_findings = {
|
|
"judge_model": judge_model,
|
|
"judge_verdict": verdict,
|
|
"findings": [
|
|
{
|
|
"defect_type": f.defect_type,
|
|
"line_start": f.line_start,
|
|
"line_end": f.line_end,
|
|
"description": f.description,
|
|
}
|
|
for f in findings
|
|
],
|
|
}
|
|
db.commit()
|
|
return verdict
|
|
|
|
results = await asyncio.gather(*(judge_one(run) for run in todo))
|
|
done = sum(1 for r in results if r)
|
|
detected = sum(1 for r in results if r and r["detected"])
|
|
print(f"judged {done}/{len(todo)}; detected {detected}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|