eeymoo/PromptCR-Lab
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
eeymoo
2026-09-19 12:54:45 +08:00
commit 6fc5b64077
126 changed files with 8601 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/make_scoring_sheet.py
+261
View File
@@ -0,0 +1,261 @@
"""Generate the Likert-5 scoring sheet for L3 fix-suggestion actionability.
Reads the 108 L3 runs (3 models x 12 samples x 3 repeats) from the
experiment database and produces a workbook with:
Sheet 1 封面: title, key facts, sheet index, notes
Sheet 2 评分说明: the 1-5 rubric and scoring guidelines
Sheet 3 打分表: one row per L3 output, dropdown-validated score column
"""
import uuid
from pathlib import Path
from openpyxl import Workbook
from openpyxl.styles import Alignment, Border, Font, PatternFill, Side
from openpyxl.utils import get_column_letter
from openpyxl.worksheet.datavalidation import DataValidation
from app.db import SessionLocal, init_db
from app.models import ExperimentRun
EXPERIMENT_ID = "42a452df-6091-47c3-8632-a9ec29815aaa"
OUTPUT = Path(r"C:\Users\eeymoo\Documents\毕业论文\实验结果\L3修复建议_李克特评分表.xlsx")
GREY_FILL = PatternFill(start_color="F5F5F5", end_color="F5F5F5", fill_type="solid")
HEADER_FILL = PatternFill(start_color="333333", end_color="333333", fill_type="solid")
BLUE_FILL = PatternFill(start_color="E6F0FA", end_color="E6F0FA", fill_type="solid")
THIN = Side(style="thin", color="D0D0D0")
BORDER = Border(left=THIN, right=THIN, top=THIN, bottom=THIN)
def collect_l3_runs():
init_db()
db = SessionLocal()
runs = (
db.query(ExperimentRun)
.filter(ExperimentRun.experiment_id == uuid.UUID(EXPERIMENT_ID), ExperimentRun.status == "done")
.all()
)
l3 = [r for r in runs if r.template_version.template.level == "L3"]
l3.sort(key=lambda r: (r.model_id, r.sample.language, str(r.sample_id), r.repeat_index))
return l3
def style_header(ws, row, cols):
for col in cols:
c = ws.cell(row=row, column=col)
c.font = Font(bold=True, color="FFFFFF", size=11)
c.fill = HEADER_FILL
c.alignment = Alignment(horizontal="center", vertical="center")
def build_cover(wb, n_rows):
ws = wb.active
ws.title = "封面"
ws.sheet_view.showGridLines = False
ws.column_dimensions["A"].width = 3
for col, w in zip("BCDEF", (22, 26, 26, 26, 26)):
ws.column_dimensions[col].width = w
ws.merge_cells("B2:F2")
ws["B2"] = "L3 修复建议可操作性评分表（李克特五级量表）"
ws["B2"].font = Font(size=18, bold=True)
ws["B2"].alignment = Alignment(horizontal="center", vertical="center")
ws.row_dimensions[2].height = 38
ws.merge_cells("B3:F3")
ws["B3"] = "基于大模型的智能代码审查提示词优化研究 · 实验 full-factorial-v1"
ws["B3"].font = Font(size=11, color="666666")
ws["B3"].alignment = Alignment(horizontal="center")
ws["B5"] = "关键信息"
ws["B5"].font = Font(bold=True, size=12)
facts = [
("待评分输出", f"{n_rows} 条（3 模型 × 12 样本 × 3 次重复的 L3 级输出）"),
("评分维度", "建议可操作性（李克特 1–5 级）"),
("评分对象", "模型给出的修复建议，对照 Ground Truth 与参考修复"),
("数据来源", "实验数据库 promptcr.db（experiment: full-factorial-v1）"),
]
r = 6
for k, v in facts:
ws.cell(row=r, column=2, value=k).font = Font(bold=True)
ws.cell(row=r, column=2).fill = BLUE_FILL
ws.cell(row=r, column=2).border = BORDER
ws.merge_cells(start_row=r, start_column=3, end_row=r, end_column=6)
ws.cell(row=r, column=3, value=v).border = BORDER
for col in range(3, 7):
ws.cell(row=r, column=col).border = BORDER
r += 1
ws["B11"] = "工作表索引"
ws["B11"].font = Font(bold=True, size=12)
index = [
("评分说明", "李克特 1–5 级判据与评分注意事项（打钩前必读）"),
("打分表", f"{n_rows} 条 L3 输出逐条打分，评分列为下拉选择 1–5"),
]
r = 12
for name, desc in index:
ws.cell(row=r, column=2, value=name).font = Font(bold=True, color="0066CC")
ws.cell(row=r, column=2).border = BORDER
ws.merge_cells(start_row=r, start_column=3, end_row=r, end_column=6)
ws.cell(row=r, column=3, value=desc)
for col in range(3, 7):
ws.cell(row=r, column=col).border = BORDER
r += 1
ws["B15"] = "备注"
ws["B15"].font = Font(bold=True, size=12)
notes = [
"1. 评分前请先阅读「评分说明」工作表，保持前后判据一致。",
"2. run_id 与实验数据库中的实验单元一一对应，便于回溯原始输出与指标。",
"3. 评分完成后，将按模型 × 级别做描述统计与模糊综合评判（参考亓莱滨方法）。",
"4. 如中途打断，可分多次填写，评分列留空视为未评。",
]
r = 16
for note in notes:
ws.merge_cells(start_row=r, start_column=2, end_row=r, end_column=6)
ws.cell(row=r, column=2, value=note).font = Font(size=10, color="666666")
r += 1
def build_rubric(wb):
ws = wb.create_sheet("评分说明")
ws.sheet_view.showGridLines = False
ws.column_dimensions["A"].width = 3
ws.column_dimensions["B"].width = 8
ws.column_dimensions["C"].width = 18
ws.column_dimensions["D"].width = 80
ws.merge_cells("B2:D2")
ws["B2"] = "建议可操作性 · 李克特五级量表判据"
ws["B2"].font = Font(size=15, bold=True)
ws["B2"].alignment = Alignment(horizontal="center", vertical="center")
ws.row_dimensions[2].height = 30
header_row = 4
ws.cell(row=header_row, column=2, value="分值")
ws.cell(row=header_row, column=3, value="等级")
ws.cell(row=header_row, column=4, value="判据")
style_header(ws, header_row, (2, 3, 4))
rubric = [
(5, "完全可操作", "建议可直接应用：能针对预埋缺陷给出正确的修复方式与具体代码/步骤，无需修改即可采纳。"),
(4, "基本可操作", "建议方向正确、内容具体，仅需少量人工调整（如修正细节、补全边界）即可应用。"),
(3, "部分可操作", "建议方向基本正确，但存在明显缺漏或不够具体，需较多人工补充、验证后才能实施。"),
(2, "可操作性差", "建议笼统模糊（如仅说「建议检查逻辑」「注意空指针」），或与预埋缺陷仅部分相关，难以直接指导修复。"),
(1, "不可操作", "建议与预埋缺陷无关、明显错误、自相矛盾，或未给出任何修复建议。"),
]
r = header_row + 1
for score, label, desc in rubric:
ws.cell(row=r, column=2, value=score).alignment = Alignment(horizontal="center", vertical="center")
ws.cell(row=r, column=2).font = Font(bold=True, size=12, color="0066CC")
ws.cell(row=r, column=3, value=label).font = Font(bold=True)
ws.cell(row=r, column=3).alignment = Alignment(vertical="center")
cell = ws.cell(row=r, column=4, value=desc)
cell.alignment = Alignment(wrap_text=True, vertical="center")
for col in (2, 3, 4):
ws.cell(row=r, column=col).border = BORDER
if score % 2 == 1:
if ws.cell(row=r, column=col).fill.start_color.rgb in (None, "00000000"):
ws.cell(row=r, column=col).fill = GREY_FILL
ws.row_dimensions[r].height = 42
r += 1
r += 1
ws.cell(row=r, column=2, value="评分注意事项").font = Font(bold=True, size=12)
notes = [
"1. 仅评价修复建议的可操作性，不评价缺陷检出是否正确（检出正确性由检出率指标衡量）。",
"2. 若一条输出包含多个问题的建议，以针对预埋缺陷（Ground Truth 列所示）的那条建议为评分对象。",
"3. 对照「参考修复」判断建议的正确性，但措辞不要求一致，语义等价即可。",
"4. 若模型未检出预埋缺陷（建议全部指向其他问题），评 1 分。",
"5. 评分时保持标准前后一致；建议先抽 5 条试评，校准后再正式评分。",
]
r += 1
for note in notes:
ws.merge_cells(start_row=r, start_column=2, end_row=r, end_column=4)
cell = ws.cell(row=r, column=2, value=note)
cell.font = Font(size=10, color="666666")
cell.alignment = Alignment(wrap_text=True, vertical="center")
ws.row_dimensions[r].height = 28
r += 1
def build_scoring(wb, runs):
ws = wb.create_sheet("打分表")
ws.sheet_view.showGridLines = False
headers = [
("序号", 6), ("run_id", 34), ("模型", 10), ("语言", 12), ("仓库", 10),
("commit", 10), ("重复", 6), ("Ground Truth 缺陷", 36), ("参考修复", 36),
("模型 L3 输出（含修复建议）", 70), ("评分(1-5)", 10), ("备注", 14),
]
header_row = 2
for i, (title, width) in enumerate(headers, start=2):
col = get_column_letter(i)
ws.column_dimensions[col].width = width
ws.cell(row=header_row, column=i, value=title)
ws.column_dimensions["A"].width = 2
style_header(ws, header_row, range(2, 2 + len(headers)))
ws.row_dimensions[header_row].height = 24
ws.freeze_panes = "B3"
dv = DataValidation(type="list", formula1='"1,2,3,4,5"', allow_blank=True, showDropDown=False)
dv.error = "请选择 1-5 的整数分值"
dv.errorTitle = "无效评分"
ws.add_data_validation(dv)
r = header_row + 1
for idx, run in enumerate(runs, start=1):
gt = run.sample.defects[0] if run.sample.defects else None
values = [
idx,
str(run.id),
run.model_id,
run.sample.language,
run.sample.repo,
run.sample.commit_sha[:7],
run.repeat_index,
(gt.description or "") if gt else "",
(gt.reference_fix or "") if gt else "",
run.result.raw_output or "",
None,
None,
]
for i, v in enumerate(values, start=2):
cell = ws.cell(row=r, column=i, value=v)
cell.border = BORDER
if i in (8, 9, 10, 11): # GT / 参考修复 / 模型输出 → wrap
cell.alignment = Alignment(wrap_text=True, vertical="top")
else:
cell.alignment = Alignment(horizontal="center", vertical="top")
ws.cell(row=r, column=12).fill = BLUE_FILL # 评分列高亮
dv.add(ws.cell(row=r, column=12))
ws.row_dimensions[r].height = 110
r += 1
ws.auto_filter.ref = f"B{header_row}:M{r - 1}"
def main():
runs = collect_l3_runs()
print(f"L3 runs: {len(runs)}")
wb = Workbook()
build_cover(wb, len(runs))
build_rubric(wb)
build_scoring(wb, runs)
OUTPUT.parent.mkdir(parents=True, exist_ok=True)
wb.save(OUTPUT)
print("saved:", OUTPUT)
# 校验：重开文件确认结构
from openpyxl import load_workbook
wb2 = load_workbook(OUTPUT)
assert wb2.sheetnames == ["封面", "评分说明", "打分表"], wb2.sheetnames
ws = wb2["打分表"]
assert ws.max_row == len(runs) + 2, (ws.max_row, len(runs))
assert ws["L2"].value == "评分(1-5)"
print("verify ok: sheets =", wb2.sheetnames, "| data rows =", ws.max_row - 2)
if __name__ == "__main__":
main()