"""Generate the Likert-5 scoring sheet for L3 fix-suggestion actionability. Reads the 108 L3 runs (3 models x 12 samples x 3 repeats) from the experiment database and produces a workbook with: Sheet 1 封面: title, key facts, sheet index, notes Sheet 2 评分说明: the 1-5 rubric and scoring guidelines Sheet 3 打分表: one row per L3 output, dropdown-validated score column """ import uuid from pathlib import Path from openpyxl import Workbook from openpyxl.styles import Alignment, Border, Font, PatternFill, Side from openpyxl.utils import get_column_letter from openpyxl.worksheet.datavalidation import DataValidation from app.db import SessionLocal, init_db from app.models import ExperimentRun EXPERIMENT_ID = "42a452df-6091-47c3-8632-a9ec29815aaa" OUTPUT = Path(r"C:\Users\eeymoo\Documents\毕业论文\实验结果\L3修复建议_李克特评分表.xlsx") GREY_FILL = PatternFill(start_color="F5F5F5", end_color="F5F5F5", fill_type="solid") HEADER_FILL = PatternFill(start_color="333333", end_color="333333", fill_type="solid") BLUE_FILL = PatternFill(start_color="E6F0FA", end_color="E6F0FA", fill_type="solid") THIN = Side(style="thin", color="D0D0D0") BORDER = Border(left=THIN, right=THIN, top=THIN, bottom=THIN) def collect_l3_runs(): init_db() db = SessionLocal() runs = ( db.query(ExperimentRun) .filter(ExperimentRun.experiment_id == uuid.UUID(EXPERIMENT_ID), ExperimentRun.status == "done") .all() ) l3 = [r for r in runs if r.template_version.template.level == "L3"] l3.sort(key=lambda r: (r.model_id, r.sample.language, str(r.sample_id), r.repeat_index)) return l3 def style_header(ws, row, cols): for col in cols: c = ws.cell(row=row, column=col) c.font = Font(bold=True, color="FFFFFF", size=11) c.fill = HEADER_FILL c.alignment = Alignment(horizontal="center", vertical="center") def build_cover(wb, n_rows): ws = wb.active ws.title = "封面" ws.sheet_view.showGridLines = False ws.column_dimensions["A"].width = 3 for col, w in zip("BCDEF", (22, 26, 26, 26, 26)): ws.column_dimensions[col].width = w ws.merge_cells("B2:F2") ws["B2"] = "L3 修复建议可操作性评分表(李克特五级量表)" ws["B2"].font = Font(size=18, bold=True) ws["B2"].alignment = Alignment(horizontal="center", vertical="center") ws.row_dimensions[2].height = 38 ws.merge_cells("B3:F3") ws["B3"] = "基于大模型的智能代码审查提示词优化研究 · 实验 full-factorial-v1" ws["B3"].font = Font(size=11, color="666666") ws["B3"].alignment = Alignment(horizontal="center") ws["B5"] = "关键信息" ws["B5"].font = Font(bold=True, size=12) facts = [ ("待评分输出", f"{n_rows} 条(3 模型 × 12 样本 × 3 次重复的 L3 级输出)"), ("评分维度", "建议可操作性(李克特 1–5 级)"), ("评分对象", "模型给出的修复建议,对照 Ground Truth 与参考修复"), ("数据来源", "实验数据库 promptcr.db(experiment: full-factorial-v1)"), ] r = 6 for k, v in facts: ws.cell(row=r, column=2, value=k).font = Font(bold=True) ws.cell(row=r, column=2).fill = BLUE_FILL ws.cell(row=r, column=2).border = BORDER ws.merge_cells(start_row=r, start_column=3, end_row=r, end_column=6) ws.cell(row=r, column=3, value=v).border = BORDER for col in range(3, 7): ws.cell(row=r, column=col).border = BORDER r += 1 ws["B11"] = "工作表索引" ws["B11"].font = Font(bold=True, size=12) index = [ ("评分说明", "李克特 1–5 级判据与评分注意事项(打钩前必读)"), ("打分表", f"{n_rows} 条 L3 输出逐条打分,评分列为下拉选择 1–5"), ] r = 12 for name, desc in index: ws.cell(row=r, column=2, value=name).font = Font(bold=True, color="0066CC") ws.cell(row=r, column=2).border = BORDER ws.merge_cells(start_row=r, start_column=3, end_row=r, end_column=6) ws.cell(row=r, column=3, value=desc) for col in range(3, 7): ws.cell(row=r, column=col).border = BORDER r += 1 ws["B15"] = "备注" ws["B15"].font = Font(bold=True, size=12) notes = [ "1. 评分前请先阅读「评分说明」工作表,保持前后判据一致。", "2. run_id 与实验数据库中的实验单元一一对应,便于回溯原始输出与指标。", "3. 评分完成后,将按模型 × 级别做描述统计与模糊综合评判(参考亓莱滨方法)。", "4. 如中途打断,可分多次填写,评分列留空视为未评。", ] r = 16 for note in notes: ws.merge_cells(start_row=r, start_column=2, end_row=r, end_column=6) ws.cell(row=r, column=2, value=note).font = Font(size=10, color="666666") r += 1 def build_rubric(wb): ws = wb.create_sheet("评分说明") ws.sheet_view.showGridLines = False ws.column_dimensions["A"].width = 3 ws.column_dimensions["B"].width = 8 ws.column_dimensions["C"].width = 18 ws.column_dimensions["D"].width = 80 ws.merge_cells("B2:D2") ws["B2"] = "建议可操作性 · 李克特五级量表判据" ws["B2"].font = Font(size=15, bold=True) ws["B2"].alignment = Alignment(horizontal="center", vertical="center") ws.row_dimensions[2].height = 30 header_row = 4 ws.cell(row=header_row, column=2, value="分值") ws.cell(row=header_row, column=3, value="等级") ws.cell(row=header_row, column=4, value="判据") style_header(ws, header_row, (2, 3, 4)) rubric = [ (5, "完全可操作", "建议可直接应用:能针对预埋缺陷给出正确的修复方式与具体代码/步骤,无需修改即可采纳。"), (4, "基本可操作", "建议方向正确、内容具体,仅需少量人工调整(如修正细节、补全边界)即可应用。"), (3, "部分可操作", "建议方向基本正确,但存在明显缺漏或不够具体,需较多人工补充、验证后才能实施。"), (2, "可操作性差", "建议笼统模糊(如仅说「建议检查逻辑」「注意空指针」),或与预埋缺陷仅部分相关,难以直接指导修复。"), (1, "不可操作", "建议与预埋缺陷无关、明显错误、自相矛盾,或未给出任何修复建议。"), ] r = header_row + 1 for score, label, desc in rubric: ws.cell(row=r, column=2, value=score).alignment = Alignment(horizontal="center", vertical="center") ws.cell(row=r, column=2).font = Font(bold=True, size=12, color="0066CC") ws.cell(row=r, column=3, value=label).font = Font(bold=True) ws.cell(row=r, column=3).alignment = Alignment(vertical="center") cell = ws.cell(row=r, column=4, value=desc) cell.alignment = Alignment(wrap_text=True, vertical="center") for col in (2, 3, 4): ws.cell(row=r, column=col).border = BORDER if score % 2 == 1: if ws.cell(row=r, column=col).fill.start_color.rgb in (None, "00000000"): ws.cell(row=r, column=col).fill = GREY_FILL ws.row_dimensions[r].height = 42 r += 1 r += 1 ws.cell(row=r, column=2, value="评分注意事项").font = Font(bold=True, size=12) notes = [ "1. 仅评价修复建议的可操作性,不评价缺陷检出是否正确(检出正确性由检出率指标衡量)。", "2. 若一条输出包含多个问题的建议,以针对预埋缺陷(Ground Truth 列所示)的那条建议为评分对象。", "3. 对照「参考修复」判断建议的正确性,但措辞不要求一致,语义等价即可。", "4. 若模型未检出预埋缺陷(建议全部指向其他问题),评 1 分。", "5. 评分时保持标准前后一致;建议先抽 5 条试评,校准后再正式评分。", ] r += 1 for note in notes: ws.merge_cells(start_row=r, start_column=2, end_row=r, end_column=4) cell = ws.cell(row=r, column=2, value=note) cell.font = Font(size=10, color="666666") cell.alignment = Alignment(wrap_text=True, vertical="center") ws.row_dimensions[r].height = 28 r += 1 def build_scoring(wb, runs): ws = wb.create_sheet("打分表") ws.sheet_view.showGridLines = False headers = [ ("序号", 6), ("run_id", 34), ("模型", 10), ("语言", 12), ("仓库", 10), ("commit", 10), ("重复", 6), ("Ground Truth 缺陷", 36), ("参考修复", 36), ("模型 L3 输出(含修复建议)", 70), ("评分(1-5)", 10), ("备注", 14), ] header_row = 2 for i, (title, width) in enumerate(headers, start=2): col = get_column_letter(i) ws.column_dimensions[col].width = width ws.cell(row=header_row, column=i, value=title) ws.column_dimensions["A"].width = 2 style_header(ws, header_row, range(2, 2 + len(headers))) ws.row_dimensions[header_row].height = 24 ws.freeze_panes = "B3" dv = DataValidation(type="list", formula1='"1,2,3,4,5"', allow_blank=True, showDropDown=False) dv.error = "请选择 1-5 的整数分值" dv.errorTitle = "无效评分" ws.add_data_validation(dv) r = header_row + 1 for idx, run in enumerate(runs, start=1): gt = run.sample.defects[0] if run.sample.defects else None values = [ idx, str(run.id), run.model_id, run.sample.language, run.sample.repo, run.sample.commit_sha[:7], run.repeat_index, (gt.description or "") if gt else "", (gt.reference_fix or "") if gt else "", run.result.raw_output or "", None, None, ] for i, v in enumerate(values, start=2): cell = ws.cell(row=r, column=i, value=v) cell.border = BORDER if i in (8, 9, 10, 11): # GT / 参考修复 / 模型输出 → wrap cell.alignment = Alignment(wrap_text=True, vertical="top") else: cell.alignment = Alignment(horizontal="center", vertical="top") ws.cell(row=r, column=12).fill = BLUE_FILL # 评分列高亮 dv.add(ws.cell(row=r, column=12)) ws.row_dimensions[r].height = 110 r += 1 ws.auto_filter.ref = f"B{header_row}:M{r - 1}" def main(): runs = collect_l3_runs() print(f"L3 runs: {len(runs)}") wb = Workbook() build_cover(wb, len(runs)) build_rubric(wb) build_scoring(wb, runs) OUTPUT.parent.mkdir(parents=True, exist_ok=True) wb.save(OUTPUT) print("saved:", OUTPUT) # 校验:重开文件确认结构 from openpyxl import load_workbook wb2 = load_workbook(OUTPUT) assert wb2.sheetnames == ["封面", "评分说明", "打分表"], wb2.sheetnames ws = wb2["打分表"] assert ws.max_row == len(runs) + 2, (ws.max_row, len(runs)) assert ws["L2"].value == "评分(1-5)" print("verify ok: sheets =", wb2.sheetnames, "| data rows =", ws.max_row - 2) if __name__ == "__main__": main()