first commit

2026-09-19 12:54:45 +08:00
commit 6fc5b64077
126 changed files with 8601 additions and 0 deletions
@@ -0,0 +1,111 @@
+"""LLM-as-judge evaluation of review outputs against Ground Truth.
+
+Rule-based matching (position hit + type match) cannot score L1 outputs,
+which deliberately contain no line numbers or defect-type labels. Following
+the cross-evaluation methodology of Liang et al., a fixed judge model
+(temperature 0, reasoning disabled) decides whether a review output
+semantically identifies each injected defect, and counts false alarms.
+
+The judge verdict is stored alongside rule-based metrics so both remain
+auditable.
+"""
+
+import json
+import re
+from typing import Any, Dict, List, Optional
+
+JUDGE_PROMPT = """You are the judge in an automated code-review experiment. Exactly one defect was deliberately injected into the code under review.
+
+## Injected defect (Ground Truth)
+- Type: {defect_type}
+- Location: lines {line_start}-{line_end} of the presented diff
+- Description: {description}
+- Reference fix: {reference_fix}
+
+## Review output under evaluation
+\"\"\"
+{raw_output}
+\"\"\"
+
+Tasks:
+1. Decide whether the review output correctly identifies the injected defect — i.e. it points out essentially the same problem, even if the wording, defect label, or line numbers differ or are absent.
+2. Count how many DISTINCT additional problems the review claims that are clearly NOT the injected defect (false alarms). Ignore stylistic nits that are part of describing the injected defect.
+3. If the review output mentions any line number(s) for the defect it identifies, list them; otherwise use null.
+
+Answer with JSON only, no other text:
+{{"detected": true or false, "false_alarms": <integer>, "lines_reported": [<integer>, ...] or null, "reason": "<one short sentence>"}}"""
+
+
+def build_judge_prompt(raw_output: str, gt: Dict[str, Any]) -> str:
+    return JUDGE_PROMPT.format(
+        defect_type=gt["defect_type"],
+        line_start=gt.get("line_start"),
+        line_end=gt.get("line_end"),
+        description=gt.get("description") or "",
+        reference_fix=gt.get("reference_fix") or "",
+        raw_output=(raw_output or "").strip()[:12000],
+    )
+
+
+def parse_verdict(text: str) -> Optional[Dict[str, Any]]:
+    """Extract the JSON verdict from the judge's reply."""
+    if not text:
+        return None
+    match = re.search(r"\{.*\}", text, re.DOTALL)
+    if not match:
+        return None
+    try:
+        data = json.loads(match.group(0))
+    except json.JSONDecodeError:
+        return None
+    if "detected" not in data:
+        return None
+    lines = data.get("lines_reported")
+    if isinstance(lines, list):
+        lines = [int(x) for x in lines if isinstance(x, (int, float))]
+    else:
+        lines = None
+    return {
+        "detected": bool(data["detected"]),
+        "false_alarms": int(data.get("false_alarms") or 0),
+        "lines_reported": lines or None,
+        "reason": str(data.get("reason") or ""),
+    }
+
+
+def verdict_to_metrics(verdict: Dict[str, Any], n_ground_truth: int = 1) -> Dict[str, float]:
+    """Convert a judge verdict into detection/false-positive rates.
+
+    detection_rate: share of injected defects identified (0 or 1 per run).
+    false_positive_rate: false alarms / (false alarms + hits), matching the
+    thesis definition "proportion of reported problems that hit no injected
+    defect".
+    """
+    hits = 1 if verdict["detected"] else 0
+    fa = max(0, verdict["false_alarms"])
+    detection_rate = hits / n_ground_truth if n_ground_truth else 0.0
+    total_reported = hits + fa
+    fpr = fa / total_reported if total_reported else 0.0
+    return {
+        "detection_rate": round(detection_rate, 4),
+        "false_positive_rate": round(fpr, 4),
+    }
+
+
+def coverage_from_verdict(verdict: Dict[str, Any], ground_truth: List[Dict[str, Any]], tolerance: int = 3) -> Optional[float]:
+    """Line coverage from judge-extracted line numbers.
+
+    Returns None when the review reported no line numbers (expected for L1),
+    so coverage stays NULL instead of polluting aggregates with zeros.
+    """
+    lines = verdict.get("lines_reported")
+    if not lines:
+        return None
+    targets = [(gt["line_start"], gt.get("line_end") or gt["line_start"]) for gt in ground_truth if gt.get("line_start") is not None]
+    if not targets:
+        return None
+    hits = 0
+    for gt_start, gt_end in targets:
+        if any(gt_start - tolerance <= ln <= gt_end + tolerance for ln in lines):
+            hits += 1
+    return round(hits / len(targets), 4)