From 6fc5b6407707d137fa54d7340876c9a4bd27fac6 Mon Sep 17 00:00:00 2001 From: eeymoo Date: Sat, 19 Sep 2026 12:54:45 +0800 Subject: [PATCH] first commit --- .gitignore | 95 ++ README.md | 112 ++ backend/Dockerfile | 21 + backend/alembic.ini | 40 + backend/alembic/README | 1 + backend/alembic/env.py | 61 + backend/alembic/script.py.mako | 26 + backend/alembic/versions/0001_initial.py | 121 ++ backend/analysis_export.py | 171 +++ backend/app/__init__.py | 0 backend/app/analysis/__init__.py | 0 backend/app/analysis/charts.py | 80 + backend/app/analysis/judge.py | 111 ++ backend/app/analysis/likert.py | 54 + backend/app/analysis/parser.py | 129 ++ backend/app/analysis/service.py | 185 +++ backend/app/analysis/stability.py | 58 + backend/app/analysis/statistics.py | 36 + backend/app/api/__init__.py | 0 backend/app/api/main.py | 24 + backend/app/api/router.py | 280 ++++ backend/app/cli/__init__.py | 0 backend/app/cli/main.py | 135 ++ backend/app/config.py | 41 + backend/app/dataset/__init__.py | 0 backend/app/dataset/builder.py | 128 ++ backend/app/dataset/diff_extractor.py | 94 ++ backend/app/dataset/git_parser.py | 121 ++ backend/app/dataset/rules/base.py | 30 + backend/app/dataset/rules/java/__init__.py | 0 .../app/dataset/rules/java/boundary_error.py | 57 + backend/app/dataset/rules/java/concurrency.py | 69 + .../app/dataset/rules/java/logic_operator.py | 52 + .../app/dataset/rules/java/null_pointer.py | 75 + .../app/dataset/rules/java/resource_leak.py | 50 + .../app/dataset/rules/javascript/__init__.py | 0 .../rules/javascript/boundary_error.py | 63 + .../dataset/rules/javascript/concurrency.py | 60 + .../rules/javascript/logic_operator.py | 61 + .../dataset/rules/javascript/null_pointer.py | 76 + .../dataset/rules/javascript/resource_leak.py | 60 + backend/app/dataset/rules/python/__init__.py | 0 .../app/dataset/rules/python/_custom_demo.py | 47 + .../dataset/rules/python/boundary_error.py | 50 + .../app/dataset/rules/python/concurrency.py | 52 + .../dataset/rules/python/logic_operator.py | 44 + .../app/dataset/rules/python/null_pointer.py | 65 + .../app/dataset/rules/python/resource_leak.py | 57 + backend/app/dataset/rules/registry.py | 49 + backend/app/db.py | 24 + backend/app/experiments/__init__.py | 4 + backend/app/experiments/matrix.py | 81 + backend/app/experiments/runner.py | 112 ++ backend/app/model_adapters/__init__.py | 0 backend/app/model_adapters/base.py | 93 ++ backend/app/model_adapters/factory.py | 48 + backend/app/model_adapters/providers.py | 57 + backend/app/models/__init__.py | 21 + backend/app/models/database.py | 146 ++ backend/app/prompts/__init__.py | 0 backend/app/prompts/defaults.py | 84 ++ backend/app/prompts/service.py | 113 ++ backend/fill_scores.py | 23 + backend/judge_all.py | 112 ++ backend/l3_runs.json | 1298 +++++++++++++++++ backend/l3_scores.json | 434 ++++++ backend/make_scoring_sheet.py | 261 ++++ backend/pytest.ini | 5 + backend/run_by_model.py | 68 + backend/tests/conftest.py | 19 + backend/tests/test_analysis.py | 28 + backend/tests/test_analysis_service.py | 61 + backend/tests/test_api.py | 42 + backend/tests/test_charts.py | 21 + backend/tests/test_experiments.py | 46 + backend/tests/test_git_parser.py | 40 + backend/tests/test_model_adapters.py | 26 + backend/tests/test_prompts.py | 36 + backend/tests/test_rules.py | 44 + backend/tests/test_runner.py | 62 + backend/tests/test_stability.py | 51 + docker-compose.yml | 49 + frontend/.gitignore | 24 + frontend/.oxlintrc.json | 8 + frontend/Dockerfile | 18 + frontend/README.md | 32 + frontend/index.html | 13 + frontend/nginx.conf | 18 + frontend/package.json | 41 + frontend/postcss.config.js | 6 + frontend/public/favicon.svg | 1 + frontend/public/icons.svg | 24 + frontend/src/App.css | 184 +++ frontend/src/App.test.tsx | 10 + frontend/src/App.tsx | 64 + frontend/src/assets/hero.png | Bin 0 -> 13057 bytes frontend/src/assets/react.svg | 1 + frontend/src/assets/vite.svg | 1 + frontend/src/components/ui/button.tsx | 53 + frontend/src/hooks/useApi.ts | 64 + frontend/src/index.css | 59 + frontend/src/main.tsx | 10 + frontend/src/pages/AnalysisPage.tsx | 163 +++ frontend/src/pages/DatasetPage.tsx | 124 ++ frontend/src/pages/ExperimentPage.tsx | 196 +++ frontend/src/pages/MonitorPage.tsx | 125 ++ frontend/src/test/setup.ts | 1 + frontend/tailwind.config.js | 50 + frontend/tsconfig.app.json | 33 + frontend/tsconfig.json | 7 + frontend/tsconfig.node.json | 23 + frontend/vite.config.ts | 25 + frontend/vitest.config.ts | 18 + .../init-promptcr-platform/.openspec.yaml | 2 + .../changes/init-promptcr-platform/design.md | 71 + .../init-promptcr-platform/proposal.md | 39 + .../specs/analysis-metrics/spec.md | 59 + .../specs/backend-access/spec.md | 42 + .../specs/dataset-management/spec.md | 49 + .../specs/experiment-orchestration/spec.md | 45 + .../specs/model-adapters/spec.md | 38 + .../specs/prompt-strategies/spec.md | 35 + .../specs/web-frontend/spec.md | 53 + .../changes/init-promptcr-platform/tasks.md | 70 + openspec/config.yaml | 1 + requirements.txt | 51 + 126 files changed, 8601 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 backend/Dockerfile create mode 100644 backend/alembic.ini create mode 100644 backend/alembic/README create mode 100644 backend/alembic/env.py create mode 100644 backend/alembic/script.py.mako create mode 100644 backend/alembic/versions/0001_initial.py create mode 100644 backend/analysis_export.py create mode 100644 backend/app/__init__.py create mode 100644 backend/app/analysis/__init__.py create mode 100644 backend/app/analysis/charts.py create mode 100644 backend/app/analysis/judge.py create mode 100644 backend/app/analysis/likert.py create mode 100644 backend/app/analysis/parser.py create mode 100644 backend/app/analysis/service.py create mode 100644 backend/app/analysis/stability.py create mode 100644 backend/app/analysis/statistics.py create mode 100644 backend/app/api/__init__.py create mode 100644 backend/app/api/main.py create mode 100644 backend/app/api/router.py create mode 100644 backend/app/cli/__init__.py create mode 100644 backend/app/cli/main.py create mode 100644 backend/app/config.py create mode 100644 backend/app/dataset/__init__.py create mode 100644 backend/app/dataset/builder.py create mode 100644 backend/app/dataset/diff_extractor.py create mode 100644 backend/app/dataset/git_parser.py create mode 100644 backend/app/dataset/rules/base.py create mode 100644 backend/app/dataset/rules/java/__init__.py create mode 100644 backend/app/dataset/rules/java/boundary_error.py create mode 100644 backend/app/dataset/rules/java/concurrency.py create mode 100644 backend/app/dataset/rules/java/logic_operator.py create mode 100644 backend/app/dataset/rules/java/null_pointer.py create mode 100644 backend/app/dataset/rules/java/resource_leak.py create mode 100644 backend/app/dataset/rules/javascript/__init__.py create mode 100644 backend/app/dataset/rules/javascript/boundary_error.py create mode 100644 backend/app/dataset/rules/javascript/concurrency.py create mode 100644 backend/app/dataset/rules/javascript/logic_operator.py create mode 100644 backend/app/dataset/rules/javascript/null_pointer.py create mode 100644 backend/app/dataset/rules/javascript/resource_leak.py create mode 100644 backend/app/dataset/rules/python/__init__.py create mode 100644 backend/app/dataset/rules/python/_custom_demo.py create mode 100644 backend/app/dataset/rules/python/boundary_error.py create mode 100644 backend/app/dataset/rules/python/concurrency.py create mode 100644 backend/app/dataset/rules/python/logic_operator.py create mode 100644 backend/app/dataset/rules/python/null_pointer.py create mode 100644 backend/app/dataset/rules/python/resource_leak.py create mode 100644 backend/app/dataset/rules/registry.py create mode 100644 backend/app/db.py create mode 100644 backend/app/experiments/__init__.py create mode 100644 backend/app/experiments/matrix.py create mode 100644 backend/app/experiments/runner.py create mode 100644 backend/app/model_adapters/__init__.py create mode 100644 backend/app/model_adapters/base.py create mode 100644 backend/app/model_adapters/factory.py create mode 100644 backend/app/model_adapters/providers.py create mode 100644 backend/app/models/__init__.py create mode 100644 backend/app/models/database.py create mode 100644 backend/app/prompts/__init__.py create mode 100644 backend/app/prompts/defaults.py create mode 100644 backend/app/prompts/service.py create mode 100644 backend/fill_scores.py create mode 100644 backend/judge_all.py create mode 100644 backend/l3_runs.json create mode 100644 backend/l3_scores.json create mode 100644 backend/make_scoring_sheet.py create mode 100644 backend/pytest.ini create mode 100644 backend/run_by_model.py create mode 100644 backend/tests/conftest.py create mode 100644 backend/tests/test_analysis.py create mode 100644 backend/tests/test_analysis_service.py create mode 100644 backend/tests/test_api.py create mode 100644 backend/tests/test_charts.py create mode 100644 backend/tests/test_experiments.py create mode 100644 backend/tests/test_git_parser.py create mode 100644 backend/tests/test_model_adapters.py create mode 100644 backend/tests/test_prompts.py create mode 100644 backend/tests/test_rules.py create mode 100644 backend/tests/test_runner.py create mode 100644 backend/tests/test_stability.py create mode 100644 docker-compose.yml create mode 100644 frontend/.gitignore create mode 100644 frontend/.oxlintrc.json create mode 100644 frontend/Dockerfile create mode 100644 frontend/README.md create mode 100644 frontend/index.html create mode 100644 frontend/nginx.conf create mode 100644 frontend/package.json create mode 100644 frontend/postcss.config.js create mode 100644 frontend/public/favicon.svg create mode 100644 frontend/public/icons.svg create mode 100644 frontend/src/App.css create mode 100644 frontend/src/App.test.tsx create mode 100644 frontend/src/App.tsx create mode 100644 frontend/src/assets/hero.png create mode 100644 frontend/src/assets/react.svg create mode 100644 frontend/src/assets/vite.svg create mode 100644 frontend/src/components/ui/button.tsx create mode 100644 frontend/src/hooks/useApi.ts create mode 100644 frontend/src/index.css create mode 100644 frontend/src/main.tsx create mode 100644 frontend/src/pages/AnalysisPage.tsx create mode 100644 frontend/src/pages/DatasetPage.tsx create mode 100644 frontend/src/pages/ExperimentPage.tsx create mode 100644 frontend/src/pages/MonitorPage.tsx create mode 100644 frontend/src/test/setup.ts create mode 100644 frontend/tailwind.config.js create mode 100644 frontend/tsconfig.app.json create mode 100644 frontend/tsconfig.json create mode 100644 frontend/tsconfig.node.json create mode 100644 frontend/vite.config.ts create mode 100644 frontend/vitest.config.ts create mode 100644 openspec/changes/init-promptcr-platform/.openspec.yaml create mode 100644 openspec/changes/init-promptcr-platform/design.md create mode 100644 openspec/changes/init-promptcr-platform/proposal.md create mode 100644 openspec/changes/init-promptcr-platform/specs/analysis-metrics/spec.md create mode 100644 openspec/changes/init-promptcr-platform/specs/backend-access/spec.md create mode 100644 openspec/changes/init-promptcr-platform/specs/dataset-management/spec.md create mode 100644 openspec/changes/init-promptcr-platform/specs/experiment-orchestration/spec.md create mode 100644 openspec/changes/init-promptcr-platform/specs/model-adapters/spec.md create mode 100644 openspec/changes/init-promptcr-platform/specs/prompt-strategies/spec.md create mode 100644 openspec/changes/init-promptcr-platform/specs/web-frontend/spec.md create mode 100644 openspec/changes/init-promptcr-platform/tasks.md create mode 100644 openspec/config.yaml create mode 100644 requirements.txt diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..1e21c9b --- /dev/null +++ b/.gitignore @@ -0,0 +1,95 @@ +# Environment variables and secrets +.env +.env.* +*.pem +*.key +*.p12 +*.pfx + +# Python +__pycache__/ +*.py[cod] +*$py.class +*.so +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST +.pytest_cache/ +.mypy_cache/ +.dmypy.json +dmypy.json +.ruff_cache/ +.venv/ +venv/ +ENV/ +env/ + +# Node.js +node_modules/ +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +package-lock.json +yarn.lock +pnpm-lock.yaml + +# Frontend build +dist/ +dist-ssr/ +*.local +.vite/ + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Logs +logs/ +*.log + +# Databases +*.sqlite +*.sqlite3 +*.db + +# Docker +.dockerignore + +# Coverage +htmlcov/ +.coverage +.coverage.* +coverage.xml +*.cover + +# Openspec local metadata +.opencode/ + +# 克隆的实验目标仓库（体量大，不入库） +data/repos/ + +# 运行残留 +*.pid diff --git a/README.md b/README.md new file mode 100644 index 0000000..f1e9895 --- /dev/null +++ b/README.md @@ -0,0 +1,112 @@ +# PromptCR-Lab + +多模型 × 多提示词策略代码审查实验支撑平台。 + +## 功能 + +- **数据集管理**：从 Git 仓库筛选 commit、提取 Unidiff、按可插拔规则预埋缺陷、生成 Ground Truth。 +- **模型接入**：DeepSeek / Kimi / 通义千问统一 OpenAI 兼容适配器，内置并发控制与指数退避重试。 +- **提示词策略**：L1/L2/L3 三级 Jinja2 模板，版本化管理，旧版本只读。 +- **实验调度**：3 模型 × N 级 × 12 commit × 3 重复全因子矩阵，asyncio 异步调度，失败重试 + 断点续跑。 +- **结果分析**：检出率 / 误报率 / 覆盖率 / 李克特五级量表 / 输出稳定性；统计分析与论文图表。 +- **接入层**：FastAPI RESTful API + Typer CLI。 +- **Web 前端**：React 18 + Tailwind CSS 3 + shadcn/ui + Vite + ECharts。 +- **部署**：`docker compose up` 一键拉起 PostgreSQL + 后端 + 前端（Nginx）。 + +## 安装 + +### 本地开发 + +```bash +# 后端 +cd backend +python3 -m pip install -r ../requirements.txt +DATABASE_URL="postgresql+psycopg://postgres:postgres@localhost:5432/promptcr" \ + uvicorn app.api.main:app --reload + +# 前端 +cd frontend +npm install +npm run dev +``` + +### Docker + +```bash +cp .env.example .env +# 编辑 .env 填入模型 API 密钥 +docker compose up --build +``` + +访问 http://localhost + +## 配置 + +在 `.env` 中配置： + +```env +DATABASE_URL=postgresql+psycopg://postgres:postgres@db:5432/promptcr +DEEPSEEK_API_KEY=your_key +KIMI_API_KEY=your_key +QWEN_API_KEY=your_key +``` + +密钥仅通过环境变量读取，仓库中无硬编码。 + +## 断点续跑 + +实验运行中断后，重新调用： + +```bash +# CLI +python3 -m app.cli.main run-experiment + +# API +POST /api/experiments//run +``` + +调度器会自动扫描 `pending` / `failed`（未超重试）单元继续执行，已完成的 `run` 不会重复调用。 + +## 数据库 ER 说明 + +核心表： + +- `samples`：commit 样本（repo, commit_sha, language, diff, 变更前后上下文） +- `defects`：预埋缺陷 Ground Truth（类型、位置行号、参考修复） +- `prompt_templates` / `prompt_template_versions`：提示词模板与版本（旧版本只读） +- `experiments`：实验批次配置快照 +- `experiment_runs`：实验单元（唯一 run_id、状态、重试次数、采样参数快照） +- `results`：单元结果（原始输出、token、耗时、解析出的指出项、五维指标、李克特分） + +## 测试 + +```bash +# 后端 +cd backend +python3 -m pytest + +# 前端 +cd frontend +npm run test +``` + +## 项目结构 + +``` +. +├── backend/ # Python 后端 (FastAPI + SQLAlchemy + asyncio) +│ ├── app/ +│ │ ├── api/ # RESTful API +│ │ ├── cli/ # Typer CLI +│ │ ├── dataset/ # Git 解析 / diff 提取 / 缺陷规则 +│ │ ├── model_adapters/ # LLM 适配器 +│ │ ├── prompts/ # 模板管理 +│ │ ├── experiments/ # 矩阵生成 / 异步调度 +│ │ └── analysis/ # 指标 / 统计 / 图表 +│ └── tests/ +├── frontend/ # React SPA +├── docker-compose.yml +├── backend/Dockerfile +├── frontend/Dockerfile +└── requirements.txt +``` diff --git a/backend/Dockerfile b/backend/Dockerfile new file mode 100644 index 0000000..0657d2e --- /dev/null +++ b/backend/Dockerfile @@ -0,0 +1,21 @@ +FROM python:3.11-slim + +WORKDIR /app + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PYTHONPATH=/app + +RUN apt-get update && apt-get install -y --no-install-recommends \ + git \ + libpq-dev \ + gcc \ + && rm -rf /var/lib/apt/lists/* + +COPY requirements.txt /app/requirements.txt +RUN pip install --no-cache-dir -r requirements.txt + +COPY backend /app/backend +COPY backend/alembic.ini /app/alembic.ini + +CMD ["sh", "-c", "alembic upgrade head && uvicorn app.api.main:app --host 0.0.0.0 --port 8000"] diff --git a/backend/alembic.ini b/backend/alembic.ini new file mode 100644 index 0000000..371d60d --- /dev/null +++ b/backend/alembic.ini @@ -0,0 +1,40 @@ +[alembic] +script_location = alembic +prepend_sys_path = . +version_path_separator = os + +[post_write_hooks] + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/backend/alembic/README b/backend/alembic/README new file mode 100644 index 0000000..a23d4fb --- /dev/null +++ b/backend/alembic/README @@ -0,0 +1 @@ +Generic single-database configuration with an async dbapi. diff --git a/backend/alembic/env.py b/backend/alembic/env.py new file mode 100644 index 0000000..a150900 --- /dev/null +++ b/backend/alembic/env.py @@ -0,0 +1,61 @@ +from logging.config import fileConfig + +from sqlalchemy import engine_from_config, pool + +from alembic import context + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from app.config import get_settings +from app.db import Base +from app.models import database # noqa: F401 + +settings = get_settings() +config = context.config + +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +target_metadata = Base.metadata + + +def get_url(): + return settings.database_url + + +def run_migrations_offline() -> None: + url = get_url() + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + configuration = config.get_section(config.config_ini_section, {}) + configuration["sqlalchemy.url"] = get_url() + connectable = engine_from_config( + configuration, + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/backend/alembic/script.py.mako b/backend/alembic/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/backend/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/backend/alembic/versions/0001_initial.py b/backend/alembic/versions/0001_initial.py new file mode 100644 index 0000000..41b2b72 --- /dev/null +++ b/backend/alembic/versions/0001_initial.py @@ -0,0 +1,121 @@ +"""initial + +Revision ID: 0001 +Revises: +Create Date: 2026-08-02 00:00:00.000000 + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision: str = "0001" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_table( + "samples", + sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column("repo", sa.String(255), nullable=False), + sa.Column("commit_sha", sa.String(40), nullable=False), + sa.Column("language", sa.String(50), nullable=False), + sa.Column("diff", sa.Text, nullable=False), + sa.Column("before_context", sa.Text, nullable=True), + sa.Column("after_context", sa.Text, nullable=True), + sa.Column("created_at", sa.DateTime, nullable=False), + ) + + op.create_table( + "defects", + sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column("sample_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("samples.id"), nullable=False), + sa.Column("defect_type", sa.String(100), nullable=False), + sa.Column("language", sa.String(50), nullable=False), + sa.Column("line_start", sa.Integer, nullable=True), + sa.Column("line_end", sa.Integer, nullable=True), + sa.Column("description", sa.Text, nullable=True), + sa.Column("reference_fix", sa.Text, nullable=True), + sa.Column("created_at", sa.DateTime, nullable=False), + ) + + op.create_table( + "prompt_templates", + sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column("strategy_id", sa.String(100), nullable=False), + sa.Column("level", sa.String(20), nullable=False), + sa.Column("created_at", sa.DateTime, nullable=False), + sa.UniqueConstraint("strategy_id", "level", name="uix_strategy_level"), + ) + + op.create_table( + "prompt_template_versions", + sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column("template_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("prompt_templates.id"), nullable=False), + sa.Column("version_number", sa.Integer, nullable=False), + sa.Column("body", sa.Text, nullable=False), + sa.Column("variables_schema", sa.JSON, nullable=False), + sa.Column("created_at", sa.DateTime, nullable=False), + ) + + op.create_table( + "experiments", + sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column("name", sa.String(255), nullable=False), + sa.Column("models", sa.JSON, nullable=False), + sa.Column("levels", sa.JSON, nullable=False), + sa.Column("sample_ids", sa.JSON, nullable=False), + sa.Column("repeats", sa.Integer, nullable=False), + sa.Column("sampling_params", sa.JSON, nullable=False), + sa.Column("status", sa.String(20), nullable=False), + sa.Column("created_at", sa.DateTime, nullable=False), + ) + + op.create_table( + "experiment_runs", + sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column("run_id", sa.String(64), unique=True, nullable=False), + sa.Column("experiment_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("experiments.id"), nullable=False), + sa.Column("sample_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("samples.id"), nullable=False), + sa.Column("model_id", sa.String(100), nullable=False), + sa.Column("template_version_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("prompt_template_versions.id"), nullable=False), + sa.Column("repeat_index", sa.Integer, nullable=False), + sa.Column("status", sa.String(20), nullable=False), + sa.Column("retry_count", sa.Integer, nullable=False), + sa.Column("sampling_params", sa.JSON, nullable=False), + sa.Column("started_at", sa.DateTime, nullable=True), + sa.Column("completed_at", sa.DateTime, nullable=True), + sa.Column("created_at", sa.DateTime, nullable=False), + ) + + op.create_table( + "results", + sa.Column("id", postgresql.UUID(as_uuid=True), primary_key=True), + sa.Column("run_id", postgresql.UUID(as_uuid=True), sa.ForeignKey("experiment_runs.id"), nullable=False, unique=True), + sa.Column("raw_output", sa.Text, nullable=True), + sa.Column("token_usage", sa.JSON, nullable=True), + sa.Column("latency_ms", sa.Float, nullable=True), + sa.Column("parsed_findings", sa.JSON, nullable=True), + sa.Column("detection_rate", sa.Float, nullable=True), + sa.Column("false_positive_rate", sa.Float, nullable=True), + sa.Column("coverage_rate", sa.Float, nullable=True), + sa.Column("stability_score", sa.Float, nullable=True), + sa.Column("likert_score", sa.Integer, nullable=True), + sa.Column("created_at", sa.DateTime, nullable=False), + sa.Column("updated_at", sa.DateTime, nullable=False), + ) + + +def downgrade() -> None: + op.drop_table("results") + op.drop_table("experiment_runs") + op.drop_table("experiments") + op.drop_table("prompt_template_versions") + op.drop_table("prompt_templates") + op.drop_table("defects") + op.drop_table("samples") diff --git a/backend/analysis_export.py b/backend/analysis_export.py new file mode 100644 index 0000000..6c5509e --- /dev/null +++ b/backend/analysis_export.py @@ -0,0 +1,171 @@ +"""Post-experiment analysis: stability, compliance, ANOVA, charts, export. + +Usage: python analysis_export.py + +- stability_score: mean pairwise token-Jaccard of the 3 repeat outputs per + (model, level, sample) group, written back to each result row. +- line-number compliance: share of runs whose review reported line numbers. +- ANOVA + paired t-tests on detection_rate across model-level groups. +- Charts (heatmap / boxplot / grouped bar) saved as PNG. +- group_summary.csv + raw run-level export for thesis chapter 5. +""" + +import base64 +import csv +import json +import re +import sys +import uuid +from collections import defaultdict +from pathlib import Path +from statistics import mean + +from app.analysis.service import AnalysisService +from app.db import SessionLocal, init_db +from app.models import ExperimentRun + + +def token_set(text: str) -> set: + return set(re.findall(r"[a-zA-Z_][a-zA-Z_0-9]*", (text or "").lower())) + + +def jaccard(a: set, b: set) -> float: + union = a | b + return len(a & b) / len(union) if union else 1.0 + + +def main() -> None: + experiment_id = sys.argv[1] + out_dir = Path(sys.argv[2]) + out_dir.mkdir(parents=True, exist_ok=True) + + init_db() + db = SessionLocal() + svc = AnalysisService(db) + + runs = ( + db.query(ExperimentRun) + .filter(ExperimentRun.experiment_id == uuid.UUID(experiment_id), ExperimentRun.status == "done") + .all() + ) + + # --- stability: token-Jaccard across repeats --- + groups = defaultdict(list) + for run in runs: + level = run.template_version.template.level + groups[(run.model_id, level, str(run.sample_id))].append(run) + + for key, group_runs in groups.items(): + texts = [r.result.raw_output for r in group_runs if r.result and r.result.raw_output] + if len(texts) < 2: + continue + sets = [token_set(t) for t in texts] + pairs = [(i, j) for i in range(len(sets)) for j in range(i + 1, len(sets))] + score = mean([jaccard(sets[i], sets[j]) for i, j in pairs]) + for r in group_runs: + if r.result: + r.result.stability_score = round(score, 4) + db.commit() + + # --- per model-level summary --- + summary = defaultdict(lambda: defaultdict(list)) + compliance = defaultdict(lambda: [0, 0]) + for run in runs: + level = run.template_version.template.level + key = (run.model_id, level) + res = run.result + if not res: + continue + if res.detection_rate is not None: + summary[key]["detection_rate"].append(res.detection_rate) + if res.false_positive_rate is not None: + summary[key]["false_positive_rate"].append(res.false_positive_rate) + if res.coverage_rate is not None: + summary[key]["coverage_rate"].append(res.coverage_rate) + if res.stability_score is not None: + summary[key]["stability"].append(res.stability_score) + pf = res.parsed_findings or {} + verdict = pf.get("judge_verdict") if isinstance(pf, dict) else None + if verdict: + compliance[key][1] += 1 + if verdict.get("lines_reported"): + compliance[key][0] += 1 + + rows = [] + for (model, level) in sorted(summary): + g = summary[(model, level)] + c = compliance[(model, level)] + row = { + "model": model, + "level": level, + "n": len(g["detection_rate"]), + "detection_rate": round(mean(g["detection_rate"]), 4) if g["detection_rate"] else None, + "false_positive_rate": round(mean(g["false_positive_rate"]), 4) if g["false_positive_rate"] else None, + "coverage_rate": round(mean(g["coverage_rate"]), 4) if g["coverage_rate"] else None, + "coverage_n": len(g["coverage_rate"]), + "stability": round(mean(g["stability"]), 4) if g["stability"] else None, + "line_compliance": round(c[0] / c[1], 4) if c[1] else None, + } + rows.append(row) + print(row) + + with open(out_dir / "group_summary.csv", "w", newline="", encoding="utf-8-sig") as f: + writer = csv.DictWriter(f, fieldnames=list(rows[0].keys())) + writer.writeheader() + writer.writerows(rows) + + # --- per-language breakdown --- + lang_summary = defaultdict(list) + for run in runs: + if run.result and run.result.detection_rate is not None: + lang_summary[(run.model_id, run.sample.language)].append(run.result.detection_rate) + lang_rows = [ + {"model": m, "language": lang, "n": len(v), "detection_rate": round(mean(v), 4)} + for (m, lang), v in sorted(lang_summary.items()) + ] + with open(out_dir / "language_summary.csv", "w", newline="", encoding="utf-8-sig") as f: + writer = csv.DictWriter(f, fieldnames=["model", "language", "n", "detection_rate"]) + writer.writeheader() + writer.writerows(lang_rows) + + # --- ANOVA + paired t-tests --- + stats = {"anova": svc.run_anova(experiment_id), "paired_t_tests": {}} + models = sorted({r.model_id for r in runs}) + levels = sorted({r.template_version.template.level for r in runs}) + for m in models: + for a, b in [("L1", "L2"), ("L2", "L3"), ("L1", "L3")]: + if a in levels and b in levels: + key = f"{m}:{a} vs {b}" + stats["paired_t_tests"][key] = svc.run_paired_t_test(f"{m}-{a}", f"{m}-{b}", experiment_id) + with open(out_dir / "statistics.json", "w", encoding="utf-8") as f: + json.dump(stats, f, ensure_ascii=False, indent=2) + print("ANOVA:", stats["anova"]) + + # --- charts --- + charts = svc.generate_charts(experiment_id) + for name, b64 in charts.items(): + try: + (out_dir / f"{name}.png").write_bytes(base64.b64decode(b64)) + print("chart saved:", name) + except Exception as e: # noqa: BLE001 + print(f"chart {name} failed: {e}") + + # --- raw run-level export --- + with open(out_dir / "runs.csv", "w", newline="", encoding="utf-8-sig") as f: + writer = csv.writer(f) + writer.writerow(["run_id", "model", "level", "language", "sample_id", "repeat", + "detection_rate", "false_positive_rate", "coverage_rate", + "stability_score", "latency_ms", "total_tokens"]) + for run in runs: + res = run.result + writer.writerow([ + str(run.id), run.model_id, run.template_version.template.level, + run.sample.language, str(run.sample_id), run.repeat_index, + res.detection_rate, res.false_positive_rate, res.coverage_rate, + res.stability_score, round(res.latency_ms or 0), (res.token_usage or {}).get("total_tokens"), + ]) + print("exports written to", out_dir) + + +if __name__ == "__main__": + main() diff --git a/backend/app/__init__.py b/backend/app/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/analysis/__init__.py b/backend/app/analysis/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/analysis/charts.py b/backend/app/analysis/charts.py new file mode 100644 index 0000000..0e76efc --- /dev/null +++ b/backend/app/analysis/charts.py @@ -0,0 +1,80 @@ +"""Matplotlib-based chart generation for paper figures.""" + +import base64 +from io import BytesIO +from typing import Dict, List + +import matplotlib +import matplotlib.pyplot as plt +import numpy as np + +matplotlib.rcParams["font.sans-serif"] = ["DejaVu Sans"] +matplotlib.rcParams["axes.unicode_minus"] = False + + +def _to_base64(fig: matplotlib.figure.Figure) -> str: + buf = BytesIO() + fig.savefig(buf, format="png", dpi=150, bbox_inches="tight") + buf.seek(0) + return base64.b64encode(buf.read()).decode("utf-8") + + +def heatmap(data: Dict[str, Dict[str, float]], title: str = "Heatmap") -> str: + """Generate a heatmap from a nested dict (rows × columns).""" + rows = list(data.keys()) + cols = sorted({c for row in data.values() for c in row.keys()}) + matrix = np.array([[data[row].get(col, 0.0) for col in cols] for row in rows]) + + fig, ax = plt.subplots(figsize=(8, 6)) + im = ax.imshow(matrix, cmap="YlOrRd", aspect="auto") + ax.set_xticks(np.arange(len(cols))) + ax.set_yticks(np.arange(len(rows))) + ax.set_xticklabels(cols) + ax.set_yticklabels(rows) + ax.set_title(title) + + for i in range(len(rows)): + for j in range(len(cols)): + text = ax.text(j, i, f"{matrix[i, j]:.2f}", ha="center", va="center", color="black") + + fig.colorbar(im, ax=ax) + encoded = _to_base64(fig) + plt.close(fig) + return encoded + + +def boxplot(groups: Dict[str, List[float]], title: str = "Boxplot") -> str: + fig, ax = plt.subplots(figsize=(8, 6)) + labels = list(groups.keys()) + values = [groups[label] for label in labels] + ax.boxplot(values) + ax.set_xticklabels(labels) + ax.set_title(title) + ax.set_ylabel("Score") + encoded = _to_base64(fig) + plt.close(fig) + return encoded + + +def grouped_bar( + data: Dict[str, Dict[str, float]], + title: str = "Grouped Bar Chart", +) -> str: + fig, ax = plt.subplots(figsize=(10, 6)) + categories = list(data.keys()) + subcategories = sorted({sc for row in data.values() for sc in row.keys()}) + x = np.arange(len(categories)) + width = 0.8 / len(subcategories) + + for idx, subcat in enumerate(subcategories): + values = [data[cat].get(subcat, 0.0) for cat in categories] + ax.bar(x + idx * width, values, width, label=subcat) + + ax.set_xticks(x + width * (len(subcategories) - 1) / 2) + ax.set_xticklabels(categories) + ax.set_ylabel("Score") + ax.set_title(title) + ax.legend() + encoded = _to_base64(fig) + plt.close(fig) + return encoded diff --git a/backend/app/analysis/judge.py b/backend/app/analysis/judge.py new file mode 100644 index 0000000..82a22db --- /dev/null +++ b/backend/app/analysis/judge.py @@ -0,0 +1,111 @@ +"""LLM-as-judge evaluation of review outputs against Ground Truth. + +Rule-based matching (position hit + type match) cannot score L1 outputs, +which deliberately contain no line numbers or defect-type labels. Following +the cross-evaluation methodology of Liang et al., a fixed judge model +(temperature 0, reasoning disabled) decides whether a review output +semantically identifies each injected defect, and counts false alarms. + +The judge verdict is stored alongside rule-based metrics so both remain +auditable. +""" + +import json +import re +from typing import Any, Dict, List, Optional + +JUDGE_PROMPT = """You are the judge in an automated code-review experiment. Exactly one defect was deliberately injected into the code under review. + +## Injected defect (Ground Truth) +- Type: {defect_type} +- Location: lines {line_start}-{line_end} of the presented diff +- Description: {description} +- Reference fix: {reference_fix} + +## Review output under evaluation +\"\"\" +{raw_output} +\"\"\" + +Tasks: +1. Decide whether the review output correctly identifies the injected defect — i.e. it points out essentially the same problem, even if the wording, defect label, or line numbers differ or are absent. +2. Count how many DISTINCT additional problems the review claims that are clearly NOT the injected defect (false alarms). Ignore stylistic nits that are part of describing the injected defect. +3. If the review output mentions any line number(s) for the defect it identifies, list them; otherwise use null. + +Answer with JSON only, no other text: +{{"detected": true or false, "false_alarms": , "lines_reported": [, ...] or null, "reason": ""}}""" + + +def build_judge_prompt(raw_output: str, gt: Dict[str, Any]) -> str: + return JUDGE_PROMPT.format( + defect_type=gt["defect_type"], + line_start=gt.get("line_start"), + line_end=gt.get("line_end"), + description=gt.get("description") or "", + reference_fix=gt.get("reference_fix") or "", + raw_output=(raw_output or "").strip()[:12000], + ) + + +def parse_verdict(text: str) -> Optional[Dict[str, Any]]: + """Extract the JSON verdict from the judge's reply.""" + if not text: + return None + match = re.search(r"\{.*\}", text, re.DOTALL) + if not match: + return None + try: + data = json.loads(match.group(0)) + except json.JSONDecodeError: + return None + if "detected" not in data: + return None + lines = data.get("lines_reported") + if isinstance(lines, list): + lines = [int(x) for x in lines if isinstance(x, (int, float))] + else: + lines = None + return { + "detected": bool(data["detected"]), + "false_alarms": int(data.get("false_alarms") or 0), + "lines_reported": lines or None, + "reason": str(data.get("reason") or ""), + } + + +def verdict_to_metrics(verdict: Dict[str, Any], n_ground_truth: int = 1) -> Dict[str, float]: + """Convert a judge verdict into detection/false-positive rates. + + detection_rate: share of injected defects identified (0 or 1 per run). + false_positive_rate: false alarms / (false alarms + hits), matching the + thesis definition "proportion of reported problems that hit no injected + defect". + """ + hits = 1 if verdict["detected"] else 0 + fa = max(0, verdict["false_alarms"]) + detection_rate = hits / n_ground_truth if n_ground_truth else 0.0 + total_reported = hits + fa + fpr = fa / total_reported if total_reported else 0.0 + return { + "detection_rate": round(detection_rate, 4), + "false_positive_rate": round(fpr, 4), + } + + +def coverage_from_verdict(verdict: Dict[str, Any], ground_truth: List[Dict[str, Any]], tolerance: int = 3) -> Optional[float]: + """Line coverage from judge-extracted line numbers. + + Returns None when the review reported no line numbers (expected for L1), + so coverage stays NULL instead of polluting aggregates with zeros. + """ + lines = verdict.get("lines_reported") + if not lines: + return None + targets = [(gt["line_start"], gt.get("line_end") or gt["line_start"]) for gt in ground_truth if gt.get("line_start") is not None] + if not targets: + return None + hits = 0 + for gt_start, gt_end in targets: + if any(gt_start - tolerance <= ln <= gt_end + tolerance for ln in lines): + hits += 1 + return round(hits / len(targets), 4) diff --git a/backend/app/analysis/likert.py b/backend/app/analysis/likert.py new file mode 100644 index 0000000..d720a45 --- /dev/null +++ b/backend/app/analysis/likert.py @@ -0,0 +1,54 @@ +"""Likert scale scoring storage and aggregation.""" + +from typing import Dict, List + +from sqlalchemy.orm import Session + +from app.models import Result + + +def save_likert_score(db: Session, run_id: str, score: int) -> Result: + if not 1 <= score <= 5: + raise ValueError("Likert score must be between 1 and 5") + result = db.query(Result).filter_by(run_id=run_id).first() + if not result: + raise ValueError(f"Result not found for run {run_id}") + result.likert_score = score + db.commit() + db.refresh(result) + return result + + +def aggregate_likert_by_model_and_level(db: Session) -> Dict[str, Dict[str, Dict[str, float]]]: + """Aggregate Likert scores by model_id and level. + + Returns mean and frequency distribution per (model, level). + """ + rows = ( + db.query(Result, ExperimentRun) + .join(ExperimentRun, Result.run_id == ExperimentRun.id) + .all() + ) + + grouped: Dict[str, Dict[str, List[int]]] = {} + for result, run in rows: + if result.likert_score is None: + continue + key_model = run.model_id + key_level = run.template_version.template.level + grouped.setdefault(key_model, {}).setdefault(key_level, []).append(result.likert_score) + + output = {} + for model, levels in grouped.items(): + output[model] = {} + for level, scores in levels.items(): + total = len(scores) + output[model][level] = { + "mean": round(sum(scores) / total, 2) if total else 0.0, + "count": total, + "distribution": {i: scores.count(i) for i in range(1, 6)}, + } + return output + + +from app.models import ExperimentRun # noqa: E402 diff --git a/backend/app/analysis/parser.py b/backend/app/analysis/parser.py new file mode 100644 index 0000000..791ca1a --- /dev/null +++ b/backend/app/analysis/parser.py @@ -0,0 +1,129 @@ +"""Parse model outputs and compare against Ground Truth.""" + +import re +from dataclasses import dataclass +from typing import Dict, List, Optional + + +@dataclass +class Finding: + defect_type: str + line_start: Optional[int] + line_end: Optional[int] + description: str + + +def parse_output(output: str, level: str) -> List[Finding]: + """Parse model output into structured findings. + + Does not guess: if output is empty or unparseable, returns empty list. + """ + if not output or not output.strip(): + return [] + + findings = [] + if level == "L1": + # Expect bullet list of defect types or short descriptions + for line in output.splitlines(): + line = line.strip() + if not line: + continue + if line.startswith(("-", "*", "•", "1.", "2.", "3.")): + item = re.sub(r"^[-*•0-9.\s]+", "", line) + findings.append( + Finding( + defect_type=item.split(":", 1)[0].strip(), + line_start=None, + line_end=None, + description=item, + ) + ) + else: + # Parse L2/L3 structured blocks + current: Dict[str, str] = {} + for raw in output.splitlines(): + line = raw.strip() + if line.startswith(("-", "*", "•")): + if current: + findings.append(_build_finding(current)) + current = {} + key, _, value = line.lstrip("-*• ").partition(":") + current[key.strip().lower()] = value.strip() + elif line and current: + key, _, value = line.partition(":") + current[key.strip().lower()] = value.strip() + if current: + findings.append(_build_finding(current)) + + return findings + + +def _build_finding(fields: Dict[str, str]) -> Finding: + defect_type = fields.get("type", "unknown") + lines = fields.get("lines", "") + line_start, line_end = None, None + if lines: + parts = re.split(r"[-,\s]+", lines) + try: + line_start = int(parts[0]) + line_end = int(parts[-1]) if len(parts) > 1 else line_start + except ValueError: + pass + description = fields.get("explanation", fields.get("fix", "")) + return Finding(defect_type, line_start, line_end, description) + + +def compare_findings( + findings: List[Finding], + ground_truth: List[dict], + line_tolerance: int = 3, +) -> Dict[str, float]: + """Compare parsed findings to Ground Truth defects. + + Returns detection_rate, false_positive_rate, coverage_rate. + """ + if not ground_truth: + return {"detection_rate": 0.0, "false_positive_rate": 0.0, "coverage_rate": 0.0} + + detected = set() + false_positives = 0 + + for finding in findings: + matched = False + for gt in ground_truth: + type_match = finding.defect_type.lower() in gt["defect_type"].lower() or gt[ + "defect_type" + ].lower() in finding.defect_type.lower() + line_match = False + if finding.line_start is not None and gt.get("line_start") is not None: + gt_start = gt["line_start"] + gt_end = gt.get("line_end", gt_start) + if ( + min(finding.line_start, finding.line_end or finding.line_start) - line_tolerance + <= gt_end + and max(finding.line_start, finding.line_end or finding.line_start) + + line_tolerance + >= gt_start + ): + line_match = True + if type_match or line_match: + matched = True + detected.add(gt.get("id", id(gt))) + break + if not matched: + false_positives += 1 + + tp = len(detected) + fp = false_positives + fn = len(ground_truth) - tp + + detection_rate = tp / len(ground_truth) + precision = tp / (tp + fp) if (tp + fp) > 0 else 0.0 + false_positive_rate = 1.0 - precision + coverage_rate = tp / len(ground_truth) + + return { + "detection_rate": round(detection_rate, 4), + "false_positive_rate": round(false_positive_rate, 4), + "coverage_rate": round(coverage_rate, 4), + } diff --git a/backend/app/analysis/service.py b/backend/app/analysis/service.py new file mode 100644 index 0000000..d23df28 --- /dev/null +++ b/backend/app/analysis/service.py @@ -0,0 +1,185 @@ +"""Analysis service: compute metrics and produce charts/JSON for frontend.""" + +from typing import Any, Dict, List +from uuid import UUID + +from sqlalchemy.orm import Session + +from app.analysis.charts import boxplot, grouped_bar, heatmap +from app.analysis.likert import aggregate_likert_by_model_and_level +from app.analysis.parser import compare_findings, parse_output +from app.analysis.statistics import anova, descriptive_stats, paired_t_test +from app.models import Experiment, ExperimentRun, Result + + +class AnalysisService: + def __init__(self, db: Session): + self.db = db + + def get_experiment_results(self, experiment_id: str) -> List[Dict[str, Any]]: + runs = ( + self.db.query(ExperimentRun) + .filter_by(experiment_id=UUID(experiment_id)) + .all() + ) + return [ + { + "run_id": str(run.id), + "model_id": run.model_id, + "level": run.template_version.template.level, + "sample_id": str(run.sample_id), + "repeat_index": run.repeat_index, + "status": run.status, + "result": self._serialize_result(run.result) if run.result else None, + } + for run in runs + ] + + def _serialize_result(self, result: Result) -> Dict[str, Any]: + return { + "raw_output": result.raw_output, + "token_usage": result.token_usage, + "latency_ms": result.latency_ms, + "parsed_findings": result.parsed_findings, + "detection_rate": result.detection_rate, + "false_positive_rate": result.false_positive_rate, + "coverage_rate": result.coverage_rate, + "stability_score": result.stability_score, + "likert_score": result.likert_score, + } + + def compute_metrics_for_run(self, run_id: str) -> Dict[str, Any]: + run = self.db.query(ExperimentRun).filter_by(id=UUID(run_id)).first() + if not run or not run.result: + return {"error": "Run or result not found"} + + level = run.template_version.template.level + findings = parse_output(run.result.raw_output or "", level) + gt = [ + { + "id": str(d.id), + "defect_type": d.defect_type, + "line_start": d.line_start, + "line_end": d.line_end, + } + for d in run.sample.defects + ] + metrics = compare_findings(findings, gt) + + run.result.parsed_findings = [self._finding_to_dict(f) for f in findings] + run.result.detection_rate = metrics["detection_rate"] + run.result.false_positive_rate = metrics["false_positive_rate"] + run.result.coverage_rate = metrics["coverage_rate"] + self.db.commit() + + return { + "run_id": run_id, + "findings": [self._finding_to_dict(f) for f in findings], + **metrics, + } + + def _finding_to_dict(self, finding) -> Dict[str, Any]: + return { + "defect_type": finding.defect_type, + "line_start": finding.line_start, + "line_end": finding.line_end, + "description": finding.description, + } + + def aggregate_metrics(self, experiment_id: str) -> Dict[str, Any]: + runs = ( + self.db.query(ExperimentRun) + .filter_by(experiment_id=UUID(experiment_id), status="done") + .all() + ) + detection_rates = [] + fp_rates = [] + coverage_rates = [] + for run in runs: + if not run.result: + continue + if run.result.detection_rate is not None: + detection_rates.append(run.result.detection_rate) + if run.result.false_positive_rate is not None: + fp_rates.append(run.result.false_positive_rate) + if run.result.coverage_rate is not None: + coverage_rates.append(run.result.coverage_rate) + + return { + "detection_rate": descriptive_stats(detection_rates), + "false_positive_rate": descriptive_stats(fp_rates), + "coverage_rate": descriptive_stats(coverage_rates), + } + + def likert_aggregation(self) -> Dict[str, Any]: + return aggregate_likert_by_model_and_level(self.db) + + def generate_charts(self, experiment_id: str) -> Dict[str, str]: + runs = ( + self.db.query(ExperimentRun) + .filter_by(experiment_id=UUID(experiment_id), status="done") + .all() + ) + heatmap_data: Dict[str, Dict[str, float]] = {} + box_groups: Dict[str, List[float]] = {} + bar_data: Dict[str, Dict[str, float]] = {} + + for run in runs: + if not run.result: + continue + model = run.model_id + level = run.template_version.template.level + dr = run.result.detection_rate or 0.0 + heatmap_data.setdefault(model, {}) + bar_data.setdefault(model, {}) + heatmap_data[model][level] = heatmap_data[model].get(level, 0.0) + dr + box_groups.setdefault(f"{model}-{level}", []).append(dr) + bar_data[model][level] = bar_data[model].get(level, 0.0) + dr + + # average heatmap and bar values + counts: Dict[str, Dict[str, int]] = {} + for run in runs: + if not run.result: + continue + model = run.model_id + level = run.template_version.template.level + counts.setdefault(model, {}).setdefault(level, 0) + counts[model][level] += 1 + for model in heatmap_data: + for level in heatmap_data[model]: + heatmap_data[model][level] /= counts[model][level] + bar_data[model][level] /= counts[model][level] + + return { + "heatmap": heatmap(heatmap_data, title="Detection Rate Heatmap"), + "boxplot": boxplot(box_groups, title="Detection Rate Distribution"), + "grouped_bar": grouped_bar(bar_data, title="Detection Rate by Model and Level"), + } + + def run_anova(self, experiment_id: str) -> Dict[str, Any]: + runs = ( + self.db.query(ExperimentRun) + .filter_by(experiment_id=UUID(experiment_id), status="done") + .all() + ) + groups: Dict[str, List[float]] = {} + for run in runs: + if not run.result or run.result.detection_rate is None: + continue + key = f"{run.model_id}-{run.template_version.template.level}" + groups.setdefault(key, []).append(run.result.detection_rate) + return anova(list(groups.values())) + + def run_paired_t_test(self, group_a_key: str, group_b_key: str, experiment_id: str) -> Dict[str, Any]: + runs = ( + self.db.query(ExperimentRun) + .filter_by(experiment_id=UUID(experiment_id), status="done") + .all() + ) + groups: Dict[str, List[float]] = {} + for run in runs: + if not run.result or run.result.detection_rate is None: + continue + key = f"{run.model_id}-{run.template_version.template.level}" + groups.setdefault(key, []).append(run.result.detection_rate) + return paired_t_test(groups.get(group_a_key, []), groups.get(group_b_key, [])) diff --git a/backend/app/analysis/stability.py b/backend/app/analysis/stability.py new file mode 100644 index 0000000..977dfbd --- /dev/null +++ b/backend/app/analysis/stability.py @@ -0,0 +1,58 @@ +"""Output stability metric using Jaccard similarity.""" + +from typing import Dict, List, Set + +from sqlalchemy.orm import Session + +from app.analysis.parser import Finding, parse_output +from app.models import ExperimentRun + + +def _finding_key(finding: Finding) -> str: + parts = [finding.defect_type.lower()] + if finding.line_start is not None: + parts.append(str(finding.line_start)) + if finding.line_end is not None: + parts.append(str(finding.line_end)) + return "|".join(parts) + + +def compute_stability_score(db: Session, experiment_id: str, model_id: str, level: str) -> float: + """Compute average pairwise Jaccard across three repeats for each sample.""" + from uuid import UUID + + runs = ( + db.query(ExperimentRun) + .filter_by(experiment_id=UUID(experiment_id), model_id=model_id) + .all() + ) + + by_sample: Dict[str, List[Set[str]]] = {} + for run in runs: + if run.template_version.template.level != level: + continue + if not run.result or not run.result.raw_output: + continue + sample_id = str(run.sample_id) + findings = set(_finding_key(f) for f in parse_output(run.result.raw_output, level)) + by_sample.setdefault(sample_id, []).append(findings) + + scores = [] + for sample_id, repeats in by_sample.items(): + if len(repeats) < 2: + continue + # pairwise Jaccard for up to 3 repeats + pairs = [(0, 1), (0, 2), (1, 2)] + pair_scores = [] + for i, j in pairs: + if i < len(repeats) and j < len(repeats): + a, b = repeats[i], repeats[j] + union = a | b + if not union: + pair_scores.append(1.0) + else: + pair_scores.append(len(a & b) / len(union)) + if pair_scores: + scores.append(sum(pair_scores) / len(pair_scores)) + + return round(sum(scores) / len(scores), 4) if scores else 0.0 diff --git a/backend/app/analysis/statistics.py b/backend/app/analysis/statistics.py new file mode 100644 index 0000000..86fa481 --- /dev/null +++ b/backend/app/analysis/statistics.py @@ -0,0 +1,36 @@ +"""Statistical analysis helpers.""" + +from typing import Dict, List, Optional + +import numpy as np +import pandas as pd +from scipy import stats + + +def descriptive_stats(values: List[float]) -> Dict[str, float]: + if not values: + return {"mean": 0.0, "std": 0.0, "min": 0.0, "max": 0.0, "median": 0.0} + arr = np.array(values, dtype=float) + return { + "mean": round(float(np.mean(arr)), 4), + "std": round(float(np.std(arr, ddof=1)), 4), + "min": round(float(np.min(arr)), 4), + "max": round(float(np.max(arr)), 4), + "median": round(float(np.median(arr)), 4), + } + + +def anova(groups: List[List[float]]) -> Dict[str, Optional[float]]: + """One-way ANOVA across groups.""" + if len(groups) < 2 or any(len(g) < 2 for g in groups): + return {"f_statistic": None, "p_value": None} + f_stat, p_value = stats.f_oneway(*groups) + return {"f_statistic": round(float(f_stat), 4), "p_value": round(float(p_value), 6)} + + +def paired_t_test(a: List[float], b: List[float]) -> Dict[str, Optional[float]]: + """Paired t-test between two samples.""" + if len(a) != len(b) or len(a) < 2: + return {"t_statistic": None, "p_value": None} + t_stat, p_value = stats.ttest_rel(a, b) + return {"t_statistic": round(float(t_stat), 4), "p_value": round(float(p_value), 6)} diff --git a/backend/app/api/__init__.py b/backend/app/api/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/api/main.py b/backend/app/api/main.py new file mode 100644 index 0000000..e54610c --- /dev/null +++ b/backend/app/api/main.py @@ -0,0 +1,24 @@ +"""FastAPI application.""" + +from contextlib import asynccontextmanager + +from fastapi import Depends, FastAPI, HTTPException +from sqlalchemy.orm import Session + +from app.api import router +from app.db import get_db, init_db + + +@asynccontextmanager +async def lifespan(app: FastAPI): + init_db() + yield + + +app = FastAPI(title="PromptCR-Lab API", lifespan=lifespan) +app.include_router(router.api_router, prefix="/api") + + +@app.get("/health") +def health_check(): + return {"status": "ok"} diff --git a/backend/app/api/router.py b/backend/app/api/router.py new file mode 100644 index 0000000..48a184a --- /dev/null +++ b/backend/app/api/router.py @@ -0,0 +1,280 @@ +"""FastAPI RESTful routers.""" + +from typing import Any, Dict, List, Optional +from uuid import UUID + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel, Field +from sqlalchemy.orm import Session + +from app.analysis.service import AnalysisService +from app.db import get_db +from app.dataset.builder import build_dataset_from_repo +from app.dataset.git_parser import select_candidates +from app.experiments.matrix import generate_full_factorial_matrix +from app.experiments.runner import ExperimentRunner +from app.model_adapters.factory import create_adapter, list_models +from app.models import Experiment, ExperimentRun, PromptTemplate, PromptTemplateVersion, Sample +from app.prompts.defaults import seed_default_templates +from app.prompts.service import PromptService + +api_router = APIRouter() + + +# ---- Dataset ---- + + +class DatasetBuildRequest(BaseModel): + repo_path: str + count: int = 12 + languages: List[str] = None + + +@api_router.post("/datasets/build") +def build_dataset(req: DatasetBuildRequest, db: Session = Depends(get_db)): + languages = req.languages or ["python", "java", "javascript"] + samples = build_dataset_from_repo(db, req.repo_path, req.count, languages) + return { + "count": len(samples), + "samples": [ + {"id": str(s.id), "repo": s.repo, "commit_sha": s.commit_sha, "language": s.language} + for s in samples + ], + } + + +@api_router.get("/datasets/samples") +def list_samples(db: Session = Depends(get_db)): + samples = db.query(Sample).all() + return [ + { + "id": str(s.id), + "repo": s.repo, + "commit_sha": s.commit_sha, + "language": s.language, + "defect_count": len(s.defects), + } + for s in samples + ] + + +@api_router.get("/datasets/samples/{sample_id}") +def get_sample(sample_id: str, db: Session = Depends(get_db)): + try: + sample = db.query(Sample).filter_by(id=UUID(sample_id)).first() + except ValueError: + raise HTTPException(status_code=400, detail="Invalid UUID") + if not sample: + raise HTTPException(status_code=404, detail="Sample not found") + return { + "id": str(sample.id), + "repo": sample.repo, + "commit_sha": sample.commit_sha, + "language": sample.language, + "diff": sample.diff, + "defects": [ + { + "id": str(d.id), + "defect_type": d.defect_type, + "line_start": d.line_start, + "line_end": d.line_end, + "reference_fix": d.reference_fix, + } + for d in sample.defects + ], + } + + +# ---- Prompts ---- + + +class PromptVersionCreate(BaseModel): + strategy_id: str + level: str + body: str + variables_schema: Optional[Dict[str, Any]] = None + + +@api_router.get("/prompts") +def list_prompts(db: Session = Depends(get_db)): + templates = db.query(PromptTemplate).all() + return [ + { + "id": str(t.id), + "strategy_id": t.strategy_id, + "level": t.level, + "version_count": len(t.versions), + } + for t in templates + ] + + +@api_router.get("/prompts/{strategy_id}/{level}/versions") +def list_prompt_versions(strategy_id: str, level: str, db: Session = Depends(get_db)): + service = PromptService(db) + return [ + { + "id": str(v.id), + "version_number": v.version_number, + "body": v.body, + "variables_schema": v.variables_schema, + "created_at": v.created_at.isoformat() if v.created_at else None, + } + for v in service.list_versions(strategy_id, level) + ] + + +@api_router.post("/prompts/versions") +def create_prompt_version(req: PromptVersionCreate, db: Session = Depends(get_db)): + service = PromptService(db) + version = service.create_version( + req.strategy_id, req.level, req.body, req.variables_schema + ) + return { + "id": str(version.id), + "version_number": version.version_number, + "template_id": str(version.template_id), + } + + +# ---- Experiments ---- + + +class ExperimentCreate(BaseModel): + name: str + models: List[str] + levels: List[str] + sample_ids: List[str] + repeats: int = 3 + sampling_params: Optional[Dict[str, Any]] = None + + +@api_router.post("/experiments") +def create_experiment(req: ExperimentCreate, db: Session = Depends(get_db)): + seed_default_templates(db) + experiment = generate_full_factorial_matrix( + db, + name=req.name, + models=req.models, + levels=req.levels, + sample_ids=req.sample_ids, + repeats=req.repeats, + sampling_params=req.sampling_params, + ) + return { + "id": str(experiment.id), + "name": experiment.name, + "status": experiment.status, + "run_count": len(experiment.runs), + } + + +@api_router.get("/experiments") +def list_experiments(db: Session = Depends(get_db)): + experiments = db.query(Experiment).all() + return [ + { + "id": str(e.id), + "name": e.name, + "status": e.status, + "run_count": len(e.runs), + } + for e in experiments + ] + + +@api_router.post("/experiments/{experiment_id}/run") +async def run_experiment(experiment_id: str, db: Session = Depends(get_db)): + runner = ExperimentRunner(db) + summary = await runner.run_experiment(experiment_id=experiment_id) + return summary + + +@api_router.get("/experiments/{experiment_id}/runs") +def get_experiment_runs(experiment_id: str, db: Session = Depends(get_db)): + try: + runs = db.query(ExperimentRun).filter_by(experiment_id=UUID(experiment_id)).all() + except ValueError: + raise HTTPException(status_code=400, detail="Invalid UUID") + return [ + { + "id": str(r.id), + "run_id": r.run_id, + "model_id": r.model_id, + "level": r.template_version.template.level, + "sample_id": str(r.sample_id), + "repeat_index": r.repeat_index, + "status": r.status, + "retry_count": r.retry_count, + } + for r in runs + ] + + +@api_router.get("/experiments/{experiment_id}/runs/{run_id}") +def get_run(run_id: str, db: Session = Depends(get_db)): + try: + run = db.query(ExperimentRun).filter_by(id=UUID(run_id)).first() + except ValueError: + raise HTTPException(status_code=400, detail="Invalid UUID") + if not run: + raise HTTPException(status_code=404, detail="Run not found") + return { + "id": str(run.id), + "run_id": run.run_id, + "model_id": run.model_id, + "level": run.template_version.template.level, + "sample_id": str(run.sample_id), + "status": run.status, + "raw_output": run.result.raw_output if run.result else None, + "latency_ms": run.result.latency_ms if run.result else None, + } + + +# ---- Analysis ---- + + +@api_router.post("/analysis/{run_id}/metrics") +def compute_run_metrics(run_id: str, db: Session = Depends(get_db)): + service = AnalysisService(db) + return service.compute_metrics_for_run(run_id) + + +@api_router.get("/analysis/{experiment_id}/aggregate") +def aggregate_metrics(experiment_id: str, db: Session = Depends(get_db)): + service = AnalysisService(db) + return service.aggregate_metrics(experiment_id) + + +@api_router.get("/analysis/{experiment_id}/charts") +def get_charts(experiment_id: str, db: Session = Depends(get_db)): + service = AnalysisService(db) + return service.generate_charts(experiment_id) + + +@api_router.get("/analysis/likert") +def likert_aggregation(db: Session = Depends(get_db)): + service = AnalysisService(db) + return service.likert_aggregation() + + +@api_router.post("/analysis/likert/{run_id}") +def save_likert(run_id: str, score: int, db: Session = Depends(get_db)): + from app.analysis.likert import save_likert_score + + result = save_likert_score(db, run_id, score) + return {"run_id": run_id, "likert_score": result.likert_score} + + +@api_router.get("/analysis/{experiment_id}/anova") +def experiment_anova(experiment_id: str, db: Session = Depends(get_db)): + service = AnalysisService(db) + return service.run_anova(experiment_id) + + +# ---- Models ---- + + +@api_router.get("/models") +def list_available_models(): + return {"models": list_models()} diff --git a/backend/app/cli/__init__.py b/backend/app/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/cli/main.py b/backend/app/cli/main.py new file mode 100644 index 0000000..9f58840 --- /dev/null +++ b/backend/app/cli/main.py @@ -0,0 +1,135 @@ +"""Typer CLI equivalent to the REST API.""" + +import uuid +from typing import List, Optional + +import typer +from sqlalchemy.orm import Session + +from app.api.router import ( + AnalysisService, + DatasetBuildRequest, + ExperimentCreate, + PromptService, + create_adapter, + generate_full_factorial_matrix, + seed_default_templates, +) +from app.db import SessionLocal, init_db +from app.experiments.runner import ExperimentRunner +from app.model_adapters.factory import list_models +from app.models import Sample + +app = typer.Typer(help="PromptCR-Lab command-line interface") + + +def get_db() -> Session: + init_db() + return SessionLocal() + + +@app.command() +def build_dataset( + repo_path: str, + count: int = typer.Option(12, "--count", "-c"), + languages: Optional[List[str]] = typer.Option(None, "--language", "-l"), +): + """Build dataset from a Git repository.""" + from app.dataset.builder import build_dataset_from_repo + + db = get_db() + languages = languages or ["python", "java", "javascript"] + samples = build_dataset_from_repo(db, repo_path, count, languages) + typer.echo(f"Created {len(samples)} samples") + + +@app.command() +def list_samples(): + """List all samples.""" + db = get_db() + samples = db.query(Sample).all() + for s in samples: + typer.echo(f"{s.id} {s.repo} {s.commit_sha} {s.language}") + + +@app.command() +def create_experiment( + name: str, + models: List[str] = typer.Option(..., "--model", "-m"), + levels: List[str] = typer.Option(..., "--level", "-l"), + sample_ids: List[str] = typer.Option(..., "--sample", "-s"), + repeats: int = typer.Option(3, "--repeats", "-r"), +): + """Create a full-factorial experiment.""" + db = get_db() + seed_default_templates(db) + experiment = generate_full_factorial_matrix( + db, + name=name, + models=models, + levels=levels, + sample_ids=sample_ids, + repeats=repeats, + ) + typer.echo(f"Created experiment {experiment.id} with {len(experiment.runs)} runs") + + +@app.command() +def run_experiment( + experiment_id: str, + concurrency: int = typer.Option(5, "--concurrency", "-c"), +): + """Run pending experiment units.""" + import asyncio + + db = get_db() + runner = ExperimentRunner(db, concurrency=concurrency) + summary = asyncio.run(runner.run_experiment(experiment_id=uuid.UUID(experiment_id))) + typer.echo(f"Total: {summary['total']}, Completed: {summary['completed']}") + + +@app.command() +def smoke( + model: str = typer.Option("deepseek", "--model", "-m"), + level: str = typer.Option("L1", "--level", "-l"), + sample_id: str = typer.Option(..., "--sample", "-s"), +): + """Run a 1×1×1×1 smoke test against a real model API.""" + import asyncio + + db = get_db() + seed_default_templates(db) + sample = db.query(Sample).filter_by(id=uuid.UUID(sample_id)).first() + if not sample: + typer.echo("Sample not found", err=True) + raise typer.Exit(1) + + prompt_service = PromptService(db) + prompt = prompt_service.render("code_review", None, {"language": sample.language, "diff": sample.diff}) + adapter = create_adapter(model) + + async def call(): + response = await adapter.chat(prompt) + typer.echo(response.text) + + asyncio.run(call()) + + +@app.command() +def list_models_cmd(): + """List supported model IDs.""" + for model in list_models(): + typer.echo(model) + + +@app.command() +def aggregate(experiment_id: str): + """Aggregate metrics for an experiment.""" + db = get_db() + service = AnalysisService(db) + # AnalysisService converts the id internally and expects a string. + typer.echo(service.aggregate_metrics(experiment_id)) + + +if __name__ == "__main__": + app() diff --git a/backend/app/config.py b/backend/app/config.py new file mode 100644 index 0000000..18604c5 --- /dev/null +++ b/backend/app/config.py @@ -0,0 +1,41 @@ +from functools import lru_cache +from typing import Optional + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + ) + + # Application + app_name: str = "PromptCR-Lab" + debug: bool = False + + # Database (psycopg2 preferred; fall back to psycopg3 if not built) + database_url: str = "postgresql+psycopg://postgres:postgres@localhost:5432/promptcr" + + # Model API keys (loaded from environment, never hardcoded) + deepseek_api_key: Optional[str] = None + deepseek_base_url: str = "https://api.deepseek.com/v1" + deepseek_model: str = "deepseek-chat" + + kimi_api_key: Optional[str] = None + kimi_base_url: str = "https://api.moonshot.cn/v1" + kimi_model: str = "moonshot-v1-8k" + + qwen_api_key: Optional[str] = None + qwen_base_url: str = "https://dashscope.aliyuncs.com/compatible-mode/v1" + qwen_model: str = "qwen-turbo" + + # Defaults + default_model_concurrency: int = 5 + default_max_retries: int = 3 + + +@lru_cache +def get_settings() -> Settings: + return Settings() diff --git a/backend/app/dataset/__init__.py b/backend/app/dataset/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/dataset/builder.py b/backend/app/dataset/builder.py new file mode 100644 index 0000000..86eeb8d --- /dev/null +++ b/backend/app/dataset/builder.py @@ -0,0 +1,128 @@ +"""Dataset builder: apply mutation rules to samples and persist Ground Truth.""" + +import random +from collections import Counter +from pathlib import Path +from typing import List, Optional + +from sqlalchemy.orm import Session + +from app.dataset.diff_extractor import _detect_language, extract_diff_bundle, bundle_to_sample_dict +from app.dataset.git_parser import select_candidates +from app.dataset.rules.registry import get_registry +from app.models import Sample, Defect + + +def build_dataset_from_repo( + db: Session, + repo_path: str | Path, + count: int = 12, + languages: Optional[List[str]] = None, +) -> List[Sample]: + """Build a dataset from a Git repository with injected defects. + + Only candidates that actually receive a mutation are kept: every sample + in the dataset must carry Ground Truth, so candidates whose changed files + match no rule are skipped and the scan continues until ``count`` mutated + samples have been collected. + """ + languages = languages or ["python", "java", "javascript"] + registry = get_registry() + candidates = select_candidates(repo_path, count=max(count * 5, 30), languages=languages) + + samples = [] + type_counts: Counter = Counter() + for candidate in candidates: + if len(samples) >= count: + break + bundle = extract_diff_bundle(repo_path, candidate.sha) + primary_lang = _infer_primary_language(bundle.after_files or bundle.before_files, languages) + sample_dict = bundle_to_sample_dict(bundle, primary_language=primary_lang) + sample = Sample(**sample_dict) + + # Collect every (file, rule) mutation opportunity for this commit. + # Rules parse and mutate a single source file, so each changed file + # of the primary language is tried individually; line numbers are + # shifted into the coordinate space of the joined source stored on + # the sample. Among all opportunities, prefer the defect type that is + # currently least represented in this build, so frequent patterns + # (e.g. `&&` swaps) do not dominate the dataset. + after_source = _join_source(bundle.after_files) + matches = [] + if primary_lang in registry.all_rules(): + rules = list(registry.rules_for(primary_lang)) + random.Random(candidate.sha).shuffle(rules) + line_offset = 0 + for path, content in (bundle.after_files or {}).items(): + if _detect_language(path) != primary_lang: + line_offset += len(content.splitlines()) + 2 + continue + for rule in rules: + m = rule.detect_and_mutate(content, filename=path) + if m: + m.line_start += line_offset + m.line_end += line_offset + matches.append((path, m)) + line_offset += len(content.splitlines()) + 2 + + if not matches: + # Skip candidates whose files match no rule: samples without + # Ground Truth are useless for the experiment. + continue + + rng = random.Random(candidate.sha) + min_count = min(type_counts[m.defect_type] for _, m in matches) + best = [(p, m) for p, m in matches if type_counts[m.defect_type] == min_count] + path, mutation = rng.choice(best) + type_counts[mutation.defect_type] += 1 + + files = dict(bundle.after_files) + files[path] = mutation.mutated_source + mutated_joined = _join_source(files) + mutation.description = f"{mutation.description} (file: {path})" + + sample.diff = _compute_diff_from_mutated(after_source, mutated_joined) + sample.after_context = {"mutated": mutated_joined} + defect = Defect( + sample=sample, + defect_type=mutation.defect_type, + language=mutation.language, + line_start=mutation.line_start, + line_end=mutation.line_end, + description=mutation.description, + reference_fix=mutation.reference_fix, + ) + sample.defects.append(defect) + + db.add(sample) + samples.append(sample) + + db.commit() + for sample in samples: + db.refresh(sample) + return samples + + +def _infer_primary_language(files: dict, languages: List[str]) -> str: + from app.dataset.diff_extractor import _detect_language + counts = {} + for path in files: + lang = _detect_language(path) + if lang in languages: + counts[lang] = counts.get(lang, 0) + 1 + if counts: + return max(counts, key=counts.get) + return languages[0] + + +def _join_source(files: dict) -> str: + return "\n\n".join(files.values()) + + +def _compute_diff_from_mutated(original: str, mutated: str) -> str: + """Produce a simple unified-diff-like string from original and mutated.""" + import difflib + + orig_lines = original.splitlines(keepends=True) + mut_lines = mutated.splitlines(keepends=True) + return "".join(difflib.unified_diff(orig_lines, mut_lines, lineterm="")) diff --git a/backend/app/dataset/diff_extractor.py b/backend/app/dataset/diff_extractor.py new file mode 100644 index 0000000..8dfdceb --- /dev/null +++ b/backend/app/dataset/diff_extractor.py @@ -0,0 +1,94 @@ +"""Unidiff extraction with before/after context.""" + +from dataclasses import dataclass +from pathlib import Path +from typing import Optional + +import git +from git import Repo + + +@dataclass +class DiffBundle: + repo: str + commit_sha: str + diff: str + before_files: dict + after_files: dict + + +def _detect_language(filename: str) -> str: + ext = Path(filename).suffix.lower() + mapping = { + ".py": "python", + ".java": "java", + ".js": "javascript", + ".ts": "javascript", + ".jsx": "javascript", + ".tsx": "javascript", + } + return mapping.get(ext, "unknown") + + +def extract_diff_bundle( + repo_path: str, + commit_sha: str, + parent_index: int = 0, +) -> DiffBundle: + """Extract unified diff and before/after file snapshots for a commit.""" + repo = Repo(str(repo_path)) + commit = repo.commit(commit_sha) + parents = commit.parents + if parents: + base = parents[parent_index] + diff = base.diff(commit, create_patch=True, unified=3) + else: + # Initial commit: diff against empty tree + diff = commit.diff(git.Git(repo).hash_object("/dev/null", t=None), create_patch=True, unified=3) + + diff_text = "\n".join(d.diff.decode("utf-8", errors="replace") for d in diff if d.diff) + + before_files = {} + after_files = {} + for d in diff: + a_path = d.a_path or d.b_path + b_path = d.b_path or d.a_path + if a_path and d.a_blob: + try: + before_files[a_path] = d.a_blob.data_stream.read().decode("utf-8", errors="replace") + except Exception: + before_files[a_path] = "" + if b_path and d.b_blob: + try: + after_files[b_path] = d.b_blob.data_stream.read().decode("utf-8", errors="replace") + except Exception: + after_files[b_path] = "" + + return DiffBundle( + repo=Path(repo_path).name, + commit_sha=commit_sha, + diff=diff_text, + before_files=before_files, + after_files=after_files, + ) + + +def bundle_to_sample_dict(bundle: DiffBundle, primary_language: Optional[str] = None) -> dict: + """Convert a DiffBundle into a dict matching the Sample schema.""" + if not primary_language: + # Infer from changed files + exts = {Path(p).suffix.lower() for p in bundle.after_files or bundle.before_files} + for ext, lang in {".py": "python", ".java": "java", ".js": "javascript"}.items(): + if ext in exts: + primary_language = lang + break + primary_language = primary_language or "unknown" + + return { + "repo": bundle.repo, + "commit_sha": bundle.commit_sha, + "language": primary_language, + "diff": bundle.diff, + "before_context": bundle.before_files, + "after_context": bundle.after_files, + } diff --git a/backend/app/dataset/git_parser.py b/backend/app/dataset/git_parser.py new file mode 100644 index 0000000..407119b --- /dev/null +++ b/backend/app/dataset/git_parser.py @@ -0,0 +1,121 @@ +"""Git repository parsing and commit candidate selection.""" + +from dataclasses import dataclass +from pathlib import Path +from typing import List, Optional + +from git import Repo + + +@dataclass +class CommitCandidate: + repo: str + sha: str + message: str + author: str + date: str + stats: dict + files: List[dict] + + +def list_commits( + repo_path: str | Path, + max_count: Optional[int] = None, + reverse: bool = True, +) -> List[CommitCandidate]: + """List commits from a Git repository.""" + repo = Repo(str(repo_path)) + repo_name = Path(repo_path).name + commits = [] + iterator = list(repo.iter_commits()) + if reverse: + iterator = reversed(iterator) + for commit in iterator: + if max_count and len(commits) >= max_count: + break + stats = commit.stats.total + files = [] + for item in commit.stats.files.items(): + filename, file_stats = item + files.append( + { + "path": filename, + "insertions": file_stats["insertions"], + "deletions": file_stats["deletions"], + "lines": file_stats["lines"], + } + ) + commits.append( + CommitCandidate( + repo=repo_name, + sha=commit.hexsha, + message=commit.message.strip(), + author=str(commit.author), + date=commit.committed_datetime.isoformat(), + stats=stats, + files=files, + ) + ) + return commits + + +def score_commit(commit: CommitCandidate) -> float: + """Score a commit by size, message quality, and language diversity.""" + total_lines = commit.stats.get("lines", 0) + # Prefer moderate size: ~50-200 lines ideal + size_score = 1.0 - abs(total_lines - 125) / 200.0 + size_score = max(0.0, min(1.0, size_score)) + + # Message quality: length and presence of verb/noun clues + msg = commit.message.lower() + msg_score = min(1.0, len(commit.message) / 40.0) + if any(k in msg for k in ("fix", "bug", "refactor", "feature", "add", "update")): + msg_score = min(1.0, msg_score + 0.2) + + # Language diversity bonus based on file extensions + exts = {Path(f["path"]).suffix.lower() for f in commit.files if Path(f["path"]).suffix} + diversity_score = min(1.0, len(exts) / 3.0) + + return size_score * 0.5 + msg_score * 0.3 + diversity_score * 0.2 + + +def select_candidates( + repo_path: str | Path, + count: int = 12, + languages: Optional[List[str]] = None, + scan_limit: int = 300, +) -> List[CommitCandidate]: + """Select top-scoring commits, optionally balanced by language. + + Only the most recent ``scan_limit`` commits are scanned: computing + per-commit stats spawns a git subprocess each time, so scanning the full + history of a large repository is prohibitively slow. + """ + languages = languages or ["python", "java", "javascript"] + commits = list_commits(repo_path, max_count=scan_limit, reverse=False) + scored = [(c, score_commit(c)) for c in commits] + scored.sort(key=lambda x: x[1], reverse=True) + + # Simple balancing: prefer at least one commit per target language when detectable + by_lang = {lang: [] for lang in languages} + others = [] + for commit, score in scored: + ext_set = {Path(f["path"]).suffix.lower() for f in commit.files} + placed = False + for lang in languages: + hint = ".py" if lang == "python" else ".java" if lang == "java" else ".js" + if hint in ext_set: + by_lang[lang].append((commit, score)) + placed = True + break + if not placed: + others.append((commit, score)) + + result = [] + per_lang = max(1, count // len(languages)) + for lang in languages: + result.extend(by_lang[lang][:per_lang]) + result.extend(others) + result = result[:count] + result.sort(key=lambda x: x[1], reverse=True) + return [c for c, _ in result] diff --git a/backend/app/dataset/rules/base.py b/backend/app/dataset/rules/base.py new file mode 100644 index 0000000..b23bf57 --- /dev/null +++ b/backend/app/dataset/rules/base.py @@ -0,0 +1,30 @@ +"""Base class for pluggable defect mutation rules.""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import List, Optional + + +@dataclass +class Mutation: + defect_type: str + language: str + line_start: int + line_end: int + mutated_source: str + reference_fix: str + description: str + + +class MutationRule(ABC): + name: str = "" + language: str = "" + defect_type: str = "" + + @abstractmethod + def detect_and_mutate(self, source: str, filename: str = "") -> Optional[Mutation]: + """Return a mutation if the rule can introduce a defect, else None.""" + ... + + def _line_for_position(self, source: str, position: int) -> int: + return source[:position].count("\n") + 1 diff --git a/backend/app/dataset/rules/java/__init__.py b/backend/app/dataset/rules/java/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/dataset/rules/java/boundary_error.py b/backend/app/dataset/rules/java/boundary_error.py new file mode 100644 index 0000000..b95178b --- /dev/null +++ b/backend/app/dataset/rules/java/boundary_error.py @@ -0,0 +1,57 @@ +"""AST-level boundary error injection for Java using javalang.""" + +from typing import Optional + +import javalang + +from app.dataset.rules.base import Mutation, MutationRule + + +class JavaBoundaryErrorRule(MutationRule): + """Mutate array length boundary from `<` to `<=`. + + Uses `javalang` to locate a binary comparison against `.length` and flips + the operator to introduce an off-by-one access. + """ + + name = "java_boundary_error" + language = "java" + defect_type = "boundary_condition_error" + + def detect_and_mutate(self, source: str, filename: str = "") -> Optional[Mutation]: + try: + tree = javalang.parse.parse(source) + except Exception: + return None + + for path, node in tree: + if not isinstance(node, javalang.tree.BinaryOperation): + continue + if node.operator != "<": + continue + right = node.operandr + if isinstance(right, javalang.tree.MemberReference) and right.member == "length": + # BinaryOperation itself may lack position; use enclosing statement + if_statement = next((n for n in path if isinstance(n, javalang.tree.IfStatement)), None) + pos = if_statement.position if if_statement else node.position + if not pos: + continue + lines = source.splitlines(keepends=True) + line_no = pos.line + line = lines[line_no - 1] + mutated_line = line.replace("< ", "<= ", 1) + if mutated_line == line: + mutated_line = line.replace("<", "<=", 1) + if mutated_line == line: + continue + mutated = "".join(lines[:line_no - 1] + [mutated_line] + lines[line_no:]) + return Mutation( + defect_type=self.defect_type, + language=self.language, + line_start=line_no, + line_end=line_no, + mutated_source=mutated, + reference_fix="Use strict `< length` to avoid ArrayIndexOutOfBoundsException.", + description="Changed array boundary check to off-by-one (<= length).", + ) + return None diff --git a/backend/app/dataset/rules/java/concurrency.py b/backend/app/dataset/rules/java/concurrency.py new file mode 100644 index 0000000..0b4de5a --- /dev/null +++ b/backend/app/dataset/rules/java/concurrency.py @@ -0,0 +1,69 @@ +"""AST-level concurrency issue injection for Java using javalang.""" + +from typing import Optional + +import javalang + +from app.dataset.rules.base import Mutation, MutationRule + + +class JavaConcurrencyRule(MutationRule): + """Remove a synchronized block to expose a race condition. + + Uses `javalang` to locate `synchronized (lock) { ... }` and replaces it + with the bare block body. + """ + + name = "java_concurrency" + language = "java" + defect_type = "concurrency_issue" + + def detect_and_mutate(self, source: str, filename: str = "") -> Optional[Mutation]: + try: + tree = javalang.parse.parse(source) + except Exception: + return None + + for path, node in tree: + if not isinstance(node, javalang.tree.SynchronizedStatement): + continue + pos = node.position + if not pos: + continue + lines = source.splitlines(keepends=True) + start = pos.line + # Estimate end by braces of the synchronized block + end = start + depth = 0 + for idx in range(start - 1, len(lines)): + depth += lines[idx].count("{") - lines[idx].count("}") + if depth > 0: + end = idx + 1 + if depth <= 0 and idx > start - 1: + end = idx + 1 + break + body_lines = lines[start - 1:end] + # drop header line and closing brace line, keep body; body is at same + # indentation as the synchronized header minus one level + inner = body_lines[1:-1] if len(body_lines) > 2 else [] + dedented = [] + for line in inner: + if line.startswith(" "): + dedented.append(" " + line[12:]) + elif line.startswith(" "): + dedented.append(" " + line[8:]) + elif line.startswith(" "): + dedented.append(line[4:]) + else: + dedented.append(line) + mutated = "".join(lines[: start - 1] + dedented + lines[end:]) + return Mutation( + defect_type=self.defect_type, + language=self.language, + line_start=start, + line_end=end, + mutated_source=mutated, + reference_fix="Restore `synchronized (lock)` to protect the critical section.", + description="Removed synchronized block, exposing a race condition.", + ) + return None diff --git a/backend/app/dataset/rules/java/logic_operator.py b/backend/app/dataset/rules/java/logic_operator.py new file mode 100644 index 0000000..d2f459d --- /dev/null +++ b/backend/app/dataset/rules/java/logic_operator.py @@ -0,0 +1,52 @@ +"""AST-level logical operator misuse injection for Java using javalang.""" + +from typing import Optional + +import javalang + +from app.dataset.rules.base import Mutation, MutationRule + + +class JavaLogicOperatorRule(MutationRule): + """Swap `&&` with `||` in a boolean expression. + + Uses `javalang` to locate a binary operation with `&&` and replaces the + operator with `||`. + """ + + name = "java_logic_operator" + language = "java" + defect_type = "logic_operator_misuse" + + def detect_and_mutate(self, source: str, filename: str = "") -> Optional[Mutation]: + try: + tree = javalang.parse.parse(source) + except Exception: + return None + + for path, node in tree: + if not isinstance(node, javalang.tree.BinaryOperation): + continue + if node.operator != "&&": + continue + return_statement = next((n for n in path if isinstance(n, javalang.tree.ReturnStatement)), None) + pos = return_statement.position if return_statement else node.position + if not pos: + continue + lines = source.splitlines(keepends=True) + line_no = pos.line + line = lines[line_no - 1] + mutated_line = line.replace("&&", "||", 1) + if mutated_line == line: + continue + mutated = "".join(lines[:line_no - 1] + [mutated_line] + lines[line_no:]) + return Mutation( + defect_type=self.defect_type, + language=self.language, + line_start=line_no, + line_end=line_no, + mutated_source=mutated, + reference_fix="Restore `&&` for correct conjunction semantics.", + description="Replaced boolean `&&` with `||`.", + ) + return None diff --git a/backend/app/dataset/rules/java/null_pointer.py b/backend/app/dataset/rules/java/null_pointer.py new file mode 100644 index 0000000..d29d237 --- /dev/null +++ b/backend/app/dataset/rules/java/null_pointer.py @@ -0,0 +1,75 @@ +"""AST-level null-pointer injection for Java using javalang.""" + +from typing import Optional + +import javalang + +from app.dataset.rules.base import Mutation, MutationRule + + +class JavaNoneReferenceRule(MutationRule): + """Remove a null-check guard in Java source. + + Uses the pure-Python `javalang` parser to locate an `if (x != null)` guard + and remove it, leaving the dereference unprotected. This keeps mutation + semantics precise without regex/text replacement. + """ + + name = "java_none_reference" + language = "java" + defect_type = "null_pointer" + + def detect_and_mutate(self, source: str, filename: str = "") -> Optional[Mutation]: + try: + tree = javalang.parse.parse(source) + except Exception: + return None + + for path, node in tree: + if not isinstance(node, javalang.tree.IfStatement): + continue + cond = node.condition + # Match: x != null + if ( + isinstance(cond, javalang.tree.BinaryOperation) + and cond.operator == "!=" + and isinstance(cond.operandr, javalang.tree.Literal) + and cond.operandr.value == "null" + ): + var_name = getattr(cond.operandl, "member", str(cond.operandl)) + lines = source.splitlines(keepends=True) + start = node.position.line if node.position else 1 + # Estimate end line by finding matching brace (simplistic) + end = start + depth = 0 + for idx in range(start - 1, len(lines)): + depth += lines[idx].count("{") - lines[idx].count("}") + if depth > 0: + end = idx + 1 + if depth <= 0 and idx > start - 1: + end = idx + 1 + break + body_lines = lines[start - 1:end] + # keep body lines between header and closing brace, dedent one level + inner = body_lines[1:-1] if len(body_lines) > 2 else [] + dedented = [] + for line in inner: + if line.startswith(" "): + dedented.append(" " + line[12:]) + elif line.startswith(" "): + dedented.append(" " + line[8:]) + elif line.startswith(" "): + dedented.append(line[4:]) + else: + dedented.append(line) + mutated = "".join(lines[: start - 1] + dedented + lines[end:]) + return Mutation( + defect_type=self.defect_type, + language=self.language, + line_start=start, + line_end=end, + mutated_source=mutated, + reference_fix=f"Add `if ({var_name} != null)` guard before dereferencing.", + description=f"Removed null-check guard for '{var_name}'.", + ) + return None diff --git a/backend/app/dataset/rules/java/resource_leak.py b/backend/app/dataset/rules/java/resource_leak.py new file mode 100644 index 0000000..9b9e40b --- /dev/null +++ b/backend/app/dataset/rules/java/resource_leak.py @@ -0,0 +1,50 @@ +"""AST-level resource leak injection for Java using javalang.""" + +from typing import Optional + +import javalang + +from app.dataset.rules.base import Mutation, MutationRule + + +class JavaResourceLeakRule(MutationRule): + """Replace try-with-resources with a plain try block, leaking the resource. + + Uses `javalang` to locate a try-with-resources statement and removes the + resource specification, leaving the stream unclosed. + """ + + name = "java_resource_leak" + language = "java" + defect_type = "resource_not_closed" + + def detect_and_mutate(self, source: str, filename: str = "") -> Optional[Mutation]: + try: + tree = javalang.parse.parse(source) + except Exception: + return None + + for path, node in tree: + if not isinstance(node, javalang.tree.TryStatement): + continue + if not node.resources: + continue + pos = node.position + if not pos: + continue + lines = source.splitlines(keepends=True) + start = pos.line + # Find the resource clause line e.g. try (BufferedReader br = ...) + resource_line = lines[start - 1] + new_header = resource_line.split("(", 1)[0].rstrip() + " {\n" + mutated = "".join(lines[: start - 1] + [new_header] + lines[start:]) + return Mutation( + defect_type=self.defect_type, + language=self.language, + line_start=start, + line_end=start, + mutated_source=mutated, + reference_fix="Use try-with-resources or explicitly close the stream in finally.", + description="Removed try-with-resources, leaking the acquired resource.", + ) + return None diff --git a/backend/app/dataset/rules/javascript/__init__.py b/backend/app/dataset/rules/javascript/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/dataset/rules/javascript/boundary_error.py b/backend/app/dataset/rules/javascript/boundary_error.py new file mode 100644 index 0000000..381f884 --- /dev/null +++ b/backend/app/dataset/rules/javascript/boundary_error.py @@ -0,0 +1,63 @@ +"""AST-level boundary error injection for JavaScript using esprima.""" + +from typing import Optional + +import esprima + +from app.dataset.rules.base import Mutation, MutationRule + + +class JSBoundaryErrorRule(MutationRule): + """Mutate array length boundary from `<` to `<=`. + + Uses `esprima` to locate a binary expression comparing against `.length` + and flips the operator. + """ + + name = "js_boundary_error" + language = "javascript" + defect_type = "boundary_condition_error" + + def detect_and_mutate(self, source: str, filename: str = "") -> Optional[Mutation]: + # Modern JS is usually ESM: try module grammar first, then script. + try: + tree = esprima.parseModule(source, loc=True) + except Exception: + try: + tree = esprima.parseScript(source, loc=True) + except Exception: + return None + + for node in self._walk(tree): + if node.type != "BinaryExpression" or node.operator != "<": + continue + right = node.right + if right.type == "MemberExpression" and getattr(right.property, "name", None) == "length": + line_no = node.loc.start.line + lines = source.splitlines(keepends=True) + line = lines[line_no - 1] + mutated_line = line.replace("<", "<=", 1) + if mutated_line == line: + continue + mutated = "".join(lines[:line_no - 1] + [mutated_line] + lines[line_no:]) + return Mutation( + defect_type=self.defect_type, + language=self.language, + line_start=line_no, + line_end=line_no, + mutated_source=mutated, + reference_fix="Use strict `< length` to avoid out-of-bounds access.", + description="Changed array boundary check to off-by-one (<= length).", + ) + return None + + def _walk(self, node): + yield node + for key in getattr(node, "__dict__", {}): + child = getattr(node, key) + if isinstance(child, list): + for item in child: + if hasattr(item, "type"): + yield from self._walk(item) + elif hasattr(child, "type"): + yield from self._walk(child) diff --git a/backend/app/dataset/rules/javascript/concurrency.py b/backend/app/dataset/rules/javascript/concurrency.py new file mode 100644 index 0000000..bcf1734 --- /dev/null +++ b/backend/app/dataset/rules/javascript/concurrency.py @@ -0,0 +1,60 @@ +"""AST-level concurrency issue injection for JavaScript using esprima.""" + +from typing import Optional + +import esprima + +from app.dataset.rules.base import Mutation, MutationRule + + +class JSConcurrencyRule(MutationRule): + """Remove an `await mutex.acquire()` / `mutex.release()` pair. + + Uses `esprima` to locate a try block followed by a finally that releases a + mutex and removes the finally/release, exposing a race. + """ + + name = "js_concurrency" + language = "javascript" + defect_type = "concurrency_issue" + + def detect_and_mutate(self, source: str, filename: str = "") -> Optional[Mutation]: + # Modern JS is usually ESM: try module grammar first, then script. + try: + tree = esprima.parseModule(source, loc=True) + except Exception: + try: + tree = esprima.parseScript(source, loc=True) + except Exception: + return None + + def walk(node): + yield node + for key in node.__dict__: + child = getattr(node, key) + if isinstance(child, list): + for item in child: + if hasattr(item, "type"): + yield from walk(item) + elif hasattr(child, "type"): + yield from walk(child) + + for node in walk(tree): + if node.type != "TryStatement" or not node.finalizer: + continue + start = node.loc.start.line + end = node.finalizer.loc.end.line + lines = source.splitlines(keepends=True) + # Drop the entire finally block; the try block ends on the same line as finally starts + finally_start = node.finalizer.loc.start.line - 1 + mutated = "".join(lines[:finally_start] + [" }\n"] + lines[end:]) + return Mutation( + defect_type=self.defect_type, + language=self.language, + line_start=start, + line_end=end, + mutated_source=mutated, + reference_fix="Restore mutex release in finally to protect the critical section.", + description="Removed mutex release, exposing a race condition.", + ) + return None diff --git a/backend/app/dataset/rules/javascript/logic_operator.py b/backend/app/dataset/rules/javascript/logic_operator.py new file mode 100644 index 0000000..82250ab --- /dev/null +++ b/backend/app/dataset/rules/javascript/logic_operator.py @@ -0,0 +1,61 @@ +"""AST-level logical operator misuse injection for JavaScript using esprima.""" + +from typing import Optional + +import esprima + +from app.dataset.rules.base import Mutation, MutationRule + + +class JSLogicOperatorRule(MutationRule): + """Swap `&&` with `||` in a boolean expression. + + Uses `esprima` to locate a LogicalExpression using `&&` and replaces the + operator with `||`. + """ + + name = "js_logic_operator" + language = "javascript" + defect_type = "logic_operator_misuse" + + def detect_and_mutate(self, source: str, filename: str = "") -> Optional[Mutation]: + # Modern JS is usually ESM: try module grammar first, then script. + try: + tree = esprima.parseModule(source, loc=True) + except Exception: + try: + tree = esprima.parseScript(source, loc=True) + except Exception: + return None + + for node in self._walk(tree): + if node.type != "LogicalExpression" or node.operator != "&&": + continue + line_no = node.loc.start.line + lines = source.splitlines(keepends=True) + line = lines[line_no - 1] + mutated_line = line.replace("&&", "||", 1) + if mutated_line == line: + continue + mutated = "".join(lines[:line_no - 1] + [mutated_line] + lines[line_no:]) + return Mutation( + defect_type=self.defect_type, + language=self.language, + line_start=line_no, + line_end=line_no, + mutated_source=mutated, + reference_fix="Restore `&&` for correct short-circuit logic.", + description="Replaced boolean `&&` with `||`.", + ) + return None + + def _walk(self, node): + yield node + for key in getattr(node, "__dict__", {}): + child = getattr(node, key) + if isinstance(child, list): + for item in child: + if hasattr(item, "type"): + yield from self._walk(item) + elif hasattr(child, "type"): + yield from self._walk(child) diff --git a/backend/app/dataset/rules/javascript/null_pointer.py b/backend/app/dataset/rules/javascript/null_pointer.py new file mode 100644 index 0000000..31eaa60 --- /dev/null +++ b/backend/app/dataset/rules/javascript/null_pointer.py @@ -0,0 +1,76 @@ +"""AST-level null-pointer injection for JavaScript using esprima.""" + +from typing import Optional + +import esprima + +from app.dataset.rules.base import Mutation, MutationRule + + +class JSNoneReferenceRule(MutationRule): + """Remove a `if (x !== null)` guard in JavaScript. + + Uses the Python port of `esprima` to locate the guard statement and + replaces it with the body, leaving a potential null dereference. + """ + + name = "js_none_reference" + language = "javascript" + defect_type = "null_pointer" + + def detect_and_mutate(self, source: str, filename: str = "") -> Optional[Mutation]: + # Modern JS is usually ESM: try module grammar first, then script. + try: + tree = esprima.parseModule(source, loc=True) + except Exception: + try: + tree = esprima.parseScript(source, loc=True) + except Exception: + return None + + def walk(node): + yield node + for key in node.__dict__: + child = getattr(node, key) + if isinstance(child, list): + for item in child: + if hasattr(item, "type"): + yield from walk(item) + elif hasattr(child, "type"): + yield from walk(child) + + for node in walk(tree): + if node.type != "IfStatement": + continue + cond = node.test + if ( + cond.type == "BinaryExpression" + and cond.operator == "!==" + and cond.right.type == "Literal" + and cond.right.value is None + ): + var_name = getattr(cond.left, "name", str(cond.left)) + start = node.loc.start.line + end = node.consequent.loc.end.line + lines = source.splitlines(keepends=True) + # Drop guard header and closing brace, keep body (1-based -> 0-based) + body_start = node.consequent.loc.start.line + body_end = node.consequent.loc.end.line - 1 + body_lines = lines[body_start:body_end] + dedented = [] + for line in body_lines: + if line.startswith(" "): + dedented.append(line[4:]) + else: + dedented.append(line) + mutated = "".join(lines[: start - 1] + dedented + lines[end:]) + return Mutation( + defect_type=self.defect_type, + language=self.language, + line_start=start, + line_end=end, + mutated_source=mutated, + reference_fix=f"Add `if ({var_name} !== null)` guard before dereferencing.", + description=f"Removed null-check guard for '{var_name}'.", + ) + return None diff --git a/backend/app/dataset/rules/javascript/resource_leak.py b/backend/app/dataset/rules/javascript/resource_leak.py new file mode 100644 index 0000000..9d960f3 --- /dev/null +++ b/backend/app/dataset/rules/javascript/resource_leak.py @@ -0,0 +1,60 @@ +"""AST-level resource leak injection for JavaScript using esprima.""" + +from typing import Optional + +import esprima + +from app.dataset.rules.base import Mutation, MutationRule + + +class JSResourceLeakRule(MutationRule): + """Remove a fetch Response body close/usage, leaking the reader. + + Uses `esprima` to locate a `try/finally` that closes a reader and removes + the finally block. + """ + + name = "js_resource_leak" + language = "javascript" + defect_type = "resource_not_closed" + + def detect_and_mutate(self, source: str, filename: str = "") -> Optional[Mutation]: + # Modern JS is usually ESM: try module grammar first, then script. + try: + tree = esprima.parseModule(source, loc=True) + except Exception: + try: + tree = esprima.parseScript(source, loc=True) + except Exception: + return None + + def walk(node): + yield node + for key in node.__dict__: + child = getattr(node, key) + if isinstance(child, list): + for item in child: + if hasattr(item, "type"): + yield from walk(item) + elif hasattr(child, "type"): + yield from walk(child) + + for node in walk(tree): + if node.type != "TryStatement" or not node.finalizer: + continue + start = node.loc.start.line + end = node.finalizer.loc.end.line + lines = source.splitlines(keepends=True) + # Drop the finally block entirely, close the try block + finally_start = node.finalizer.loc.start.line - 1 + mutated = "".join(lines[:finally_start] + [" }\n"] + lines[end:]) + return Mutation( + defect_type=self.defect_type, + language=self.language, + line_start=start, + line_end=end, + mutated_source=mutated, + reference_fix="Restore finally block to close/release resources.", + description="Removed finally block, leaving resource unreleased.", + ) + return None diff --git a/backend/app/dataset/rules/python/__init__.py b/backend/app/dataset/rules/python/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/dataset/rules/python/_custom_demo.py b/backend/app/dataset/rules/python/_custom_demo.py new file mode 100644 index 0000000..1a0b586 --- /dev/null +++ b/backend/app/dataset/rules/python/_custom_demo.py @@ -0,0 +1,47 @@ +"""Custom rule demonstrating pluggable extensibility (A1b).""" + +import ast +from typing import Optional + +from app.dataset.rules.base import Mutation, MutationRule + + +class UnusedVariableRule(MutationRule): + """A custom rule: replace a variable read with an undefined name. + + This is intentionally simple and demonstrates that adding a new file under + app/dataset/rules// is enough to register a rule. + """ + + name = "unused_variable_demo" + language = "python" + defect_type = "custom_demo" + + def detect_and_mutate(self, source: str, filename: str = "") -> Optional[Mutation]: + try: + tree = ast.parse(source) + except SyntaxError: + return None + + for node in ast.walk(tree): + if isinstance(node, ast.FunctionDef) and node.body: + first = node.body[0] + if isinstance(first, ast.Assign) and isinstance(first.targets[0], ast.Name): + var_name = first.targets[0].id + line_no = first.lineno + lines = source.splitlines(keepends=True) + line = lines[line_no - 1] + mutated_line = line.replace(var_name, "__undefined_" + var_name, 1) + if mutated_line == line: + continue + mutated = "".join(lines[:line_no - 1] + [mutated_line] + lines[line_no:]) + return Mutation( + defect_type=self.defect_type, + language=self.language, + line_start=line_no, + line_end=line_no, + mutated_source=mutated, + reference_fix=f"Use the original variable name '{var_name}'.", + description=f"Custom rule: replaced '{var_name}' with an undefined placeholder.", + ) + return None diff --git a/backend/app/dataset/rules/python/boundary_error.py b/backend/app/dataset/rules/python/boundary_error.py new file mode 100644 index 0000000..3cd1b6c --- /dev/null +++ b/backend/app/dataset/rules/python/boundary_error.py @@ -0,0 +1,50 @@ +"""AST-level boundary condition injection for Python.""" + +import ast +from typing import Optional + +from app.dataset.rules.base import Mutation, MutationRule + + +class BoundaryErrorRule(MutationRule): + """Mutate a list-index boundary check from `< len(seq)` to `<= len(seq)`. + + Uses `ast` to find comparisons guarding index access and flips the operator + so the boundary becomes off-by-one. + """ + + name = "boundary_error" + language = "python" + defect_type = "boundary_condition_error" + + def detect_and_mutate(self, source: str, filename: str = "") -> Optional[Mutation]: + try: + tree = ast.parse(source) + except SyntaxError: + return None + + for node in ast.walk(tree): + if not isinstance(node, ast.If): + continue + test = node.test + if isinstance(test, ast.Compare) and isinstance(test.left, ast.Name): + if len(test.ops) == 1 and isinstance(test.ops[0], ast.Lt): + # i < len(x) -> i <= len(x) + comparator = test.comparators[0] + if isinstance(comparator, ast.Call) and isinstance(comparator.func, ast.Name) and comparator.func.id == "len": + lines = source.splitlines(keepends=True) + line = lines[node.lineno - 1] + mutated_line = line.replace("< len(", "<= len(", 1) + if mutated_line == line: + continue + mutated = "".join(lines[: node.lineno - 1] + [mutated_line] + lines[node.lineno:]) + return Mutation( + defect_type=self.defect_type, + language=self.language, + line_start=node.lineno, + line_end=getattr(node, "end_lineno", node.lineno) or node.lineno, + mutated_source=mutated, + reference_fix="Use strict `< len(seq)` to avoid index-out-of-range.", + description="Changed index boundary check to off-by-one (<= len).", + ) + return None diff --git a/backend/app/dataset/rules/python/concurrency.py b/backend/app/dataset/rules/python/concurrency.py new file mode 100644 index 0000000..6999991 --- /dev/null +++ b/backend/app/dataset/rules/python/concurrency.py @@ -0,0 +1,52 @@ +"""AST-level concurrency safety injection for Python.""" + +import ast +from typing import Optional + +from app.dataset.rules.base import Mutation, MutationRule + + +class ConcurrencyRule(MutationRule): + """Remove a threading.Lock.acquire/release pair to introduce race condition. + + Uses `ast` to find a with-statement using a lock and replaces it with the + bare body, removing synchronization. + """ + + name = "concurrency" + language = "python" + defect_type = "concurrency_issue" + + def detect_and_mutate(self, source: str, filename: str = "") -> Optional[Mutation]: + try: + tree = ast.parse(source) + except SyntaxError: + return None + + for node in ast.walk(tree): + if not isinstance(node, ast.With): + continue + first_item = node.items[0] + ctx = first_item.context_expr + if isinstance(ctx, ast.Call) and isinstance(ctx.func, ast.Attribute) and ctx.func.attr == "acquire": + start = node.lineno + end = getattr(node, "end_lineno", node.lineno) or node.lineno + lines = source.splitlines(keepends=True) + body = lines[start:end] + dedented = [] + for line in body[1:]: + if line.startswith(" "): + dedented.append(line[4:]) + else: + dedented.append(line) + mutated = "".join(lines[: start - 1] + dedented + lines[end:]) + return Mutation( + defect_type=self.defect_type, + language=self.language, + line_start=start, + line_end=end, + mutated_source=mutated, + reference_fix="Restore `with lock:` to protect the critical section.", + description="Removed lock acquisition, exposing a race condition.", + ) + return None diff --git a/backend/app/dataset/rules/python/logic_operator.py b/backend/app/dataset/rules/python/logic_operator.py new file mode 100644 index 0000000..be4ac8d --- /dev/null +++ b/backend/app/dataset/rules/python/logic_operator.py @@ -0,0 +1,44 @@ +"""AST-level logical operator misuse injection for Python.""" + +import ast +from typing import Optional + +from app.dataset.rules.base import Mutation, MutationRule + + +class LogicOperatorRule(MutationRule): + """Swap `and` with `or` in a boolean expression. + + Uses `ast` to locate a BoolOp using `And` and replaces it with `Or`, + preserving exact source position via line replacement. + """ + + name = "logic_operator" + language = "python" + defect_type = "logic_operator_misuse" + + def detect_and_mutate(self, source: str, filename: str = "") -> Optional[Mutation]: + try: + tree = ast.parse(source) + except SyntaxError: + return None + + for node in ast.walk(tree): + if isinstance(node, ast.BoolOp) and isinstance(node.op, ast.And): + line_no = node.lineno + lines = source.splitlines(keepends=True) + line = lines[line_no - 1] + mutated_line = line.replace(" and ", " or ", 1) + if mutated_line == line: + continue + mutated = "".join(lines[:line_no - 1] + [mutated_line] + lines[line_no:]) + return Mutation( + defect_type=self.defect_type, + language=self.language, + line_start=line_no, + line_end=getattr(node, "end_lineno", line_no) or line_no, + mutated_source=mutated, + reference_fix="Restore the original `and` operator for correct short-circuit logic.", + description="Replaced boolean `and` with `or`.", + ) + return None diff --git a/backend/app/dataset/rules/python/null_pointer.py b/backend/app/dataset/rules/python/null_pointer.py new file mode 100644 index 0000000..37befce --- /dev/null +++ b/backend/app/dataset/rules/python/null_pointer.py @@ -0,0 +1,65 @@ +"""AST-level null-pointer / None-reference injection for Python.""" + +import ast +from typing import Optional + +from app.dataset.rules.base import Mutation, MutationRule + + +class NoneReferenceRule(MutationRule): + """Replace a checked variable access with an unchecked None dereference. + + This rule uses the standard library `ast` module to precisely locate a + variable that is used after an `if x is not None:` guard, then removes the + guard. The mutation position is derived from AST line numbers so it is + exact and reproducible. + """ + + name = "none_reference" + language = "python" + defect_type = "null_pointer" + + def detect_and_mutate(self, source: str, filename: str = "") -> Optional[Mutation]: + try: + tree = ast.parse(source) + except SyntaxError: + return None + + for node in ast.walk(tree): + if not isinstance(node, ast.If): + continue + test = node.test + # Match: if x is not None: + if ( + isinstance(test, ast.Compare) + and isinstance(test.left, ast.Name) + and len(test.ops) == 1 + and isinstance(test.ops[0], ast.IsNot) + and len(test.comparators) == 1 + and isinstance(test.comparators[0], ast.Constant) + and test.comparators[0].value is None + ): + var_name = test.left.id + lines = source.splitlines(keepends=True) + start = node.lineno + end = getattr(node, "end_lineno", node.lineno) or node.lineno + body_lines = lines[start:end] + dedented = [] + for line in body_lines[1:]: + if line.startswith(" "): + dedented.append(line[4:]) + elif line.startswith("\t"): + dedented.append(line[1:]) + else: + dedented.append(line) + mutated = "".join(lines[: start - 1] + dedented + lines[end:]) + return Mutation( + defect_type=self.defect_type, + language=self.language, + line_start=start, + line_end=end, + mutated_source=mutated, + reference_fix=f"Add `if {var_name} is not None:` guard before use.", + description=f"Removed None-check guard for variable '{var_name}'.", + ) + return None diff --git a/backend/app/dataset/rules/python/resource_leak.py b/backend/app/dataset/rules/python/resource_leak.py new file mode 100644 index 0000000..fb20140 --- /dev/null +++ b/backend/app/dataset/rules/python/resource_leak.py @@ -0,0 +1,57 @@ +"""AST-level resource leak injection for Python.""" + +import ast +from typing import Optional + +from app.dataset.rules.base import Mutation, MutationRule + + +class ResourceLeakRule(MutationRule): + """Convert a `with open(...)` block into an unclosed `open(...).read()`. + + Uses `ast` to locate a with-statement managing a file resource and replaces + it with a direct call chain that leaks the file handle. + """ + + name = "resource_leak" + language = "python" + defect_type = "resource_not_closed" + + def detect_and_mutate(self, source: str, filename: str = "") -> Optional[Mutation]: + try: + tree = ast.parse(source) + except SyntaxError: + return None + + for node in ast.walk(tree): + if not isinstance(node, ast.With): + continue + first_item = node.items[0] + ctx = first_item.context_expr + if isinstance(ctx, ast.Call) and isinstance(ctx.func, ast.Name) and ctx.func.id == "open": + start = node.lineno + end = getattr(node, "end_lineno", node.lineno) or node.lineno + lines = source.splitlines(keepends=True) + # Keep the with header expression but replace 'with open(...)' by 'f = open(...)' + header = lines[start - 1] + header_expr = header.split("with ", 1)[1].split(" as ", 1)[0].strip().rstrip(":\n") + var = header.split(" as ", 1)[1].strip().rstrip(":\n") if " as " in header else "f" + body = lines[start:end] + dedented = [] + for line in body[1:]: + if line.startswith(" "): + dedented.append(line[4:]) + else: + dedented.append(line) + replacement = [f"{var} = {header_expr}\n"] + dedented + mutated = "".join(lines[: start - 1] + replacement + lines[end:]) + return Mutation( + defect_type=self.defect_type, + language=self.language, + line_start=start, + line_end=end, + mutated_source=mutated, + reference_fix="Use `with open(...) as f:` to ensure the file is closed.", + description="Replaced context-managed open() with an unclosed file handle.", + ) + return None diff --git a/backend/app/dataset/rules/registry.py b/backend/app/dataset/rules/registry.py new file mode 100644 index 0000000..c023801 --- /dev/null +++ b/backend/app/dataset/rules/registry.py @@ -0,0 +1,49 @@ +"""Registry that auto-discovers mutation rules from the rules directory.""" + +import importlib +import inspect +from pathlib import Path +from typing import Dict, List, Type + +from app.dataset.rules.base import MutationRule + + +class RuleRegistry: + def __init__(self, rules_dir: Path): + self.rules_dir = rules_dir + self._rules: Dict[str, List[MutationRule]] = {} + + def discover(self) -> None: + """Scan rules directory and register all MutationRule subclasses.""" + self._rules.clear() + for lang_dir in self.rules_dir.iterdir(): + if not lang_dir.is_dir(): + continue + for py_file in lang_dir.glob("*.py"): + if py_file.name.startswith("_"): + continue + module_name = f"app.dataset.rules.{lang_dir.name}.{py_file.stem}" + try: + module = importlib.import_module(module_name) + except Exception: + continue + for _, obj in inspect.getmembers(module, inspect.isclass): + if ( + issubclass(obj, MutationRule) + and obj is not MutationRule + and not getattr(obj, "__abstractmethods__", False) + ): + rule = obj() + self._rules.setdefault(rule.language, []).append(rule) + + def rules_for(self, language: str) -> List[MutationRule]: + return self._rules.get(language, []) + + def all_rules(self) -> Dict[str, List[MutationRule]]: + return self._rules.copy() + + +def get_registry() -> RuleRegistry: + registry = RuleRegistry(Path(__file__).parent) + registry.discover() + return registry diff --git a/backend/app/db.py b/backend/app/db.py new file mode 100644 index 0000000..5c6faa8 --- /dev/null +++ b/backend/app/db.py @@ -0,0 +1,24 @@ +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker, declarative_base + +from app.config import get_settings + +settings = get_settings() + +engine = create_engine(settings.database_url, pool_pre_ping=True) +SessionLocal = sessionmaker(autocommit=False, autoflush=False, bind=engine) +Base = declarative_base() + + +def get_db(): + db = SessionLocal() + try: + yield db + finally: + db.close() + + +def init_db(): + # Import models so Base.metadata is populated + from app.models import database # noqa: F401 + Base.metadata.create_all(bind=engine) diff --git a/backend/app/experiments/__init__.py b/backend/app/experiments/__init__.py new file mode 100644 index 0000000..d0d5771 --- /dev/null +++ b/backend/app/experiments/__init__.py @@ -0,0 +1,4 @@ +from app.experiments.matrix import generate_full_factorial_matrix +from app.experiments.runner import ExperimentRunner + +__all__ = ["generate_full_factorial_matrix", "ExperimentRunner"] diff --git a/backend/app/experiments/matrix.py b/backend/app/experiments/matrix.py new file mode 100644 index 0000000..7ed28cc --- /dev/null +++ b/backend/app/experiments/matrix.py @@ -0,0 +1,81 @@ +"""Full-factorial experiment matrix generation.""" + +from itertools import product +from typing import Any, Dict, List +from uuid import uuid4 + +from sqlalchemy.orm import Session + +from app.models import Experiment, ExperimentRun, PromptTemplateVersion, Sample +from app.prompts.service import PromptService + + +def generate_full_factorial_matrix( + db: Session, + name: str, + models: List[str], + levels: List[str], + sample_ids: List[str], + repeats: int = 3, + sampling_params: Dict[str, Any] = None, + strategy_id: str = "code_review", +) -> Experiment: + """Generate experiment runs for a full-factorial matrix. + + Models × Levels × Samples × Repeats. + """ + sampling_params = sampling_params or {"temperature": 0.7, "max_tokens": 2048} + experiment = Experiment( + name=name, + models=models, + levels=levels, + sample_ids=sample_ids, + repeats=repeats, + sampling_params=sampling_params, + status="pending", + ) + db.add(experiment) + db.flush() + + prompt_service = PromptService(db) + runs = [] + for model_id, level in product(models, levels): + # Resolve latest template version for this level + versions = prompt_service.list_versions(strategy_id, level) + if not versions: + raise ValueError(f"No prompt template found for {strategy_id}/{level}") + template_version = versions[-1] + + for sample_id in sample_ids: + from uuid import UUID + + sample_uuid = UUID(sample_id) if isinstance(sample_id, str) else sample_id + sample = db.query(Sample).filter_by(id=sample_uuid).first() + if not sample: + raise ValueError(f"Sample not found: {sample_id}") + + for repeat_index in range(1, repeats + 1): + run = ExperimentRun( + run_id=str(uuid4()), + experiment_id=experiment.id, + sample_id=sample.id, + model_id=model_id, + template_version_id=template_version.id, + repeat_index=repeat_index, + status="pending", + sampling_params=sampling_params, + ) + runs.append(run) + + db.add_all(runs) + db.commit() + db.refresh(experiment) + return experiment + + +def count_pending_runs(db: Session, experiment_id: str) -> int: + return ( + db.query(ExperimentRun) + .filter_by(experiment_id=experiment_id, status="pending") + .count() + ) diff --git a/backend/app/experiments/runner.py b/backend/app/experiments/runner.py new file mode 100644 index 0000000..194c117 --- /dev/null +++ b/backend/app/experiments/runner.py @@ -0,0 +1,112 @@ +"""Async experiment runner with retry, isolation, and resume support.""" + +import asyncio +from datetime import datetime, timezone +from typing import Any, Dict, List, Optional + +from sqlalchemy.orm import Session + +from app.dataset.diff_extractor import extract_diff_bundle +from app.model_adapters.factory import create_adapter +from app.models import ExperimentRun, Result +from app.prompts.service import PromptService + + +class ExperimentRunner: + def __init__( + self, + db: Session, + concurrency: int = 5, + max_retries: int = 3, + strategy_id: str = "code_review", + ): + self.db = db + self.concurrency = concurrency + self.max_retries = max_retries + self.strategy_id = strategy_id + self.prompt_service = PromptService(db) + + def _get_pending_runs(self, experiment_id: Optional[str] = None) -> List[ExperimentRun]: + query = self.db.query(ExperimentRun) + if experiment_id: + query = query.filter_by(experiment_id=experiment_id) + return query.filter(ExperimentRun.status.in_(["pending", "failed"])).all() + + def _reset_stale_running(self) -> None: + """Mark runs stuck in running without result back to pending.""" + stale = ( + self.db.query(ExperimentRun) + .filter_by(status="running") + .filter(~ExperimentRun.result.has()) + .all() + ) + for run in stale: + run.status = "pending" + self.db.commit() + + async def run_experiment( + self, + experiment_id: Optional[str] = None, + progress_callback=None, + ) -> Dict[str, Any]: + self._reset_stale_running() + pending = self._get_pending_runs(experiment_id) + semaphore = asyncio.Semaphore(self.concurrency) + + async def execute(run: ExperimentRun): + async with semaphore: + return await self._execute_run(run) + + tasks = [asyncio.create_task(execute(run)) for run in pending] + results = [] + for coro in asyncio.as_completed(tasks): + result = await coro + results.append(result) + if progress_callback: + progress_callback(result) + + return {"total": len(pending), "completed": len(results)} + + async def _execute_run(self, run: ExperimentRun) -> Dict[str, Any]: + run.status = "running" + run.started_at = datetime.now(timezone.utc) + self.db.commit() + + try: + sample = run.sample + prompt = self.prompt_service.render( + self.strategy_id, + run.template_version.version_number, + {"language": sample.language, "diff": sample.diff}, + ) + adapter = create_adapter(run.model_id) + response = await adapter.chat(prompt, run.sampling_params) + + run.status = "done" + run.completed_at = datetime.now(timezone.utc) + result = Result( + run_id=run.id, + raw_output=response.text, + token_usage=response.token_usage, + latency_ms=response.latency_ms, + ) + self.db.add(result) + self.db.commit() + return { + "run_id": str(run.id), + "status": "done", + "model_id": run.model_id, + } + except Exception as e: + run.retry_count += 1 + if run.retry_count > self.max_retries: + run.status = "failed" + else: + run.status = "pending" # will be retried on next resume + run.completed_at = datetime.now(timezone.utc) + self.db.commit() + return { + "run_id": str(run.id), + "status": run.status, + "error": str(e), + } diff --git a/backend/app/model_adapters/__init__.py b/backend/app/model_adapters/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/model_adapters/base.py b/backend/app/model_adapters/base.py new file mode 100644 index 0000000..2b3261b --- /dev/null +++ b/backend/app/model_adapters/base.py @@ -0,0 +1,93 @@ +"""Abstract base class for model adapters.""" + +from abc import ABC, abstractmethod +import asyncio +import time +from dataclasses import dataclass +from typing import Any, Dict, Optional + +import httpx + + +@dataclass +class ChatResponse: + text: str + token_usage: Dict[str, int] + latency_ms: float + + +class ModelAdapter(ABC): + """Unified interface for LLM vendors. + + Subclasses only need to provide base_url, api_key, model name and any + vendor-specific headers. Concurrency and retry logic are inherited. + """ + + def __init__( + self, + api_key: str, + model: str, + base_url: str, + concurrency: int = 5, + max_retries: int = 3, + timeout: float = 120.0, + ): + self.api_key = api_key + self.model = model + self.base_url = base_url.rstrip("/") + self.semaphore = asyncio.Semaphore(concurrency) + self.max_retries = max_retries + self.timeout = timeout + + @abstractmethod + def _build_payload(self, prompt: str, params: Dict[str, Any]) -> Dict[str, Any]: + ... + + @abstractmethod + def _extract_text(self, data: Dict[str, Any]) -> str: + ... + + def _extract_token_usage(self, data: Dict[str, Any]) -> Dict[str, int]: + usage = data.get("usage", {}) + return { + "prompt_tokens": usage.get("prompt_tokens", 0), + "completion_tokens": usage.get("completion_tokens", 0), + "total_tokens": usage.get("total_tokens", 0), + } + + def _headers(self) -> Dict[str, str]: + return { + "Authorization": f"Bearer {self.api_key}", + "Content-Type": "application/json", + } + + async def chat(self, prompt: str, params: Optional[Dict[str, Any]] = None) -> ChatResponse: + params = params or {} + payload = self._build_payload(prompt, params) + async with self.semaphore: + last_exception: Optional[Exception] = None + for attempt in range(self.max_retries + 1): + start = time.perf_counter() + try: + async with httpx.AsyncClient(timeout=self.timeout) as client: + response = await client.post( + f"{self.base_url}/chat/completions", + headers=self._headers(), + json=payload, + ) + response.raise_for_status() + data = response.json() + latency_ms = (time.perf_counter() - start) * 1000 + return ChatResponse( + text=self._extract_text(data), + token_usage=self._extract_token_usage(data), + latency_ms=latency_ms, + ) + except Exception as e: + last_exception = e + if attempt < self.max_retries: + wait = 2**attempt + await asyncio.sleep(wait) + raise RuntimeError( + f"Model {self.model} failed after {self.max_retries} retries: {last_exception}" + ) diff --git a/backend/app/model_adapters/factory.py b/backend/app/model_adapters/factory.py new file mode 100644 index 0000000..3730c02 --- /dev/null +++ b/backend/app/model_adapters/factory.py @@ -0,0 +1,48 @@ +"""Factory for creating model adapters from configuration.""" + +from app.config import get_settings +from app.model_adapters.base import ModelAdapter +from app.model_adapters.providers import DeepSeekAdapter, KimiAdapter, QwenAdapter + + +_ADAPTER_MAP = { + "deepseek": DeepSeekAdapter, + "kimi": KimiAdapter, + "qwen": QwenAdapter, +} + + +def create_adapter(model_id: str, concurrency: int = 5, max_retries: int = 3) -> ModelAdapter: + settings = get_settings() + model_id = model_id.lower() + adapter_cls = _ADAPTER_MAP.get(model_id) + if not adapter_cls: + raise ValueError(f"Unknown model_id: {model_id}. Available: {list(_ADAPTER_MAP.keys())}") + + if model_id == "deepseek": + return adapter_cls( + api_key=settings.deepseek_api_key or "", + model=settings.deepseek_model, + base_url=settings.deepseek_base_url, + concurrency=concurrency, + max_retries=max_retries, + ) + if model_id == "kimi": + return adapter_cls( + api_key=settings.kimi_api_key or "", + model=settings.kimi_model, + base_url=settings.kimi_base_url, + concurrency=concurrency, + max_retries=max_retries, + ) + return adapter_cls( + api_key=settings.qwen_api_key or "", + model=settings.qwen_model, + base_url=settings.qwen_base_url, + concurrency=concurrency, + max_retries=max_retries, + ) + + +def list_models() -> list[str]: + return list(_ADAPTER_MAP.keys()) diff --git a/backend/app/model_adapters/providers.py b/backend/app/model_adapters/providers.py new file mode 100644 index 0000000..e9a2dd9 --- /dev/null +++ b/backend/app/model_adapters/providers.py @@ -0,0 +1,57 @@ +"""Concrete model adapters for DeepSeek, Kimi, and Qwen.""" + +from typing import Any, Dict + +from app.model_adapters.base import ModelAdapter + + +class DeepSeekAdapter(ModelAdapter): + def _build_payload(self, prompt: str, params: Dict[str, Any]) -> Dict[str, Any]: + return { + "model": self.model, + "messages": [{"role": "user", "content": prompt}], + "temperature": params.get("temperature", 0.7), + "max_tokens": params.get("max_tokens", 8192), + # Disable vendor reasoning mode: thinking tokens would otherwise + # exhaust max_tokens and leave `content` empty, and reasoning + # behavior is an uncontrolled variable in the prompt-strategy + # experiment. + "thinking": {"type": "disabled"}, + } + + def _extract_text(self, data: Dict[str, Any]) -> str: + return data["choices"][0]["message"]["content"] + + +class KimiAdapter(ModelAdapter): + def _build_payload(self, prompt: str, params: Dict[str, Any]) -> Dict[str, Any]: + return { + "model": self.model, + "messages": [{"role": "user", "content": prompt}], + # kimi-k2.x rejects any temperature other than 0.6. + "temperature": params.get("temperature", 0.6), + "max_tokens": params.get("max_tokens", 8192), + # No `thinking` switch: kimi-k2.x rejects it, and its built-in + # reasoning is short enough to leave room for the answer. + } + + def _extract_text(self, data: Dict[str, Any]) -> str: + return data["choices"][0]["message"]["content"] + + +class QwenAdapter(ModelAdapter): + def _build_payload(self, prompt: str, params: Dict[str, Any]) -> Dict[str, Any]: + return { + "model": self.model, + "messages": [{"role": "user", "content": prompt}], + "temperature": params.get("temperature", 0.7), + "max_tokens": params.get("max_tokens", 8192), + # Disable vendor reasoning mode: thinking tokens would otherwise + # exhaust max_tokens and leave `content` empty, and reasoning + # behavior is an uncontrolled variable in the prompt-strategy + # experiment. + "thinking": {"type": "disabled"}, + } + + def _extract_text(self, data: Dict[str, Any]) -> str: + return data["choices"][0]["message"]["content"] diff --git a/backend/app/models/__init__.py b/backend/app/models/__init__.py new file mode 100644 index 0000000..1f9c45e --- /dev/null +++ b/backend/app/models/__init__.py @@ -0,0 +1,21 @@ +from app.models.database import ( + Base, + Defect, + Experiment, + ExperimentRun, + PromptTemplate, + PromptTemplateVersion, + Result, + Sample, +) + +__all__ = [ + "Base", + "Sample", + "Defect", + "PromptTemplate", + "PromptTemplateVersion", + "Experiment", + "ExperimentRun", + "Result", +] diff --git a/backend/app/models/database.py b/backend/app/models/database.py new file mode 100644 index 0000000..08ec0c9 --- /dev/null +++ b/backend/app/models/database.py @@ -0,0 +1,146 @@ +import uuid +from datetime import datetime, timezone +from typing import Optional + +from sqlalchemy import ( + JSON, + Column, + DateTime, + Float, + ForeignKey, + Integer, + String, + Text, + UniqueConstraint, +) +from sqlalchemy.dialects.postgresql import UUID +from sqlalchemy.orm import relationship + +from app.db import Base + + +def now_utc() -> datetime: + return datetime.now(timezone.utc) + + +class Sample(Base): + __tablename__ = "samples" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + repo = Column(String(255), nullable=False) + commit_sha = Column(String(40), nullable=False) + language = Column(String(50), nullable=False) + diff = Column(Text, nullable=False) + before_context = Column(JSON, nullable=True) + after_context = Column(JSON, nullable=True) + created_at = Column(DateTime, default=now_utc, nullable=False) + + defects = relationship("Defect", back_populates="sample", cascade="all, delete-orphan") + runs = relationship("ExperimentRun", back_populates="sample") + + +class Defect(Base): + __tablename__ = "defects" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + sample_id = Column(UUID(as_uuid=True), ForeignKey("samples.id"), nullable=False) + defect_type = Column(String(100), nullable=False) + language = Column(String(50), nullable=False) + line_start = Column(Integer, nullable=True) + line_end = Column(Integer, nullable=True) + description = Column(Text, nullable=True) + reference_fix = Column(Text, nullable=True) + created_at = Column(DateTime, default=now_utc, nullable=False) + + sample = relationship("Sample", back_populates="defects") + + +class PromptTemplate(Base): + __tablename__ = "prompt_templates" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + strategy_id = Column(String(100), nullable=False) + level = Column(String(20), nullable=False) + created_at = Column(DateTime, default=now_utc, nullable=False) + + versions = relationship( + "PromptTemplateVersion", + back_populates="template", + cascade="all, delete-orphan", + order_by="PromptTemplateVersion.version_number", + ) + + __table_args__ = (UniqueConstraint("strategy_id", "level", name="uix_strategy_level"),) + + +class PromptTemplateVersion(Base): + __tablename__ = "prompt_template_versions" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + template_id = Column(UUID(as_uuid=True), ForeignKey("prompt_templates.id"), nullable=False) + version_number = Column(Integer, nullable=False) + body = Column(Text, nullable=False) + variables_schema = Column(JSON, nullable=False, default=dict) + created_at = Column(DateTime, default=now_utc, nullable=False) + + template = relationship("PromptTemplate", back_populates="versions") + runs = relationship("ExperimentRun", back_populates="template_version") + + +class Experiment(Base): + __tablename__ = "experiments" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + name = Column(String(255), nullable=False) + models = Column(JSON, nullable=False) + levels = Column(JSON, nullable=False) + sample_ids = Column(JSON, nullable=False) + repeats = Column(Integer, nullable=False, default=3) + sampling_params = Column(JSON, nullable=False, default=dict) + status = Column(String(20), default="pending", nullable=False) + created_at = Column(DateTime, default=now_utc, nullable=False) + + runs = relationship("ExperimentRun", back_populates="experiment", cascade="all, delete-orphan") + + +class ExperimentRun(Base): + __tablename__ = "experiment_runs" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + run_id = Column(String(64), unique=True, nullable=False, default=lambda: str(uuid.uuid4())) + experiment_id = Column(UUID(as_uuid=True), ForeignKey("experiments.id"), nullable=False) + sample_id = Column(UUID(as_uuid=True), ForeignKey("samples.id"), nullable=False) + model_id = Column(String(100), nullable=False) + template_version_id = Column(UUID(as_uuid=True), ForeignKey("prompt_template_versions.id"), nullable=False) + repeat_index = Column(Integer, nullable=False) + status = Column(String(20), default="pending", nullable=False) + retry_count = Column(Integer, default=0, nullable=False) + sampling_params = Column(JSON, nullable=False, default=dict) + started_at = Column(DateTime, nullable=True) + completed_at = Column(DateTime, nullable=True) + created_at = Column(DateTime, default=now_utc, nullable=False) + + experiment = relationship("Experiment", back_populates="runs") + sample = relationship("Sample", back_populates="runs") + template_version = relationship("PromptTemplateVersion", back_populates="runs") + result = relationship("Result", back_populates="run", uselist=False, cascade="all, delete-orphan") + + +class Result(Base): + __tablename__ = "results" + + id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4) + run_id = Column(UUID(as_uuid=True), ForeignKey("experiment_runs.id"), nullable=False, unique=True) + raw_output = Column(Text, nullable=True) + token_usage = Column(JSON, nullable=True) + latency_ms = Column(Float, nullable=True) + parsed_findings = Column(JSON, nullable=True) + detection_rate = Column(Float, nullable=True) + false_positive_rate = Column(Float, nullable=True) + coverage_rate = Column(Float, nullable=True) + stability_score = Column(Float, nullable=True) + likert_score = Column(Integer, nullable=True) + created_at = Column(DateTime, default=now_utc, nullable=False) + updated_at = Column(DateTime, default=now_utc, onupdate=now_utc, nullable=False) + + run = relationship("ExperimentRun", back_populates="result") diff --git a/backend/app/prompts/__init__.py b/backend/app/prompts/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/prompts/defaults.py b/backend/app/prompts/defaults.py new file mode 100644 index 0000000..02a0bcc --- /dev/null +++ b/backend/app/prompts/defaults.py @@ -0,0 +1,84 @@ +"""Default L1/L2/L3 prompt templates.""" + +from sqlalchemy.orm import Session + +from app.prompts.service import PromptService + +DEFAULT_TEMPLATES = { + ("code_review", "L1"): { + "body": """You are a code reviewer. Review the following code diff and identify any potential bugs or issues. +Only list what is wrong; do not provide locations or fixes. + +Language: {{ language }} +Diff: +``` +{{ diff }} +``` + +Report issues as a plain list.""", + "variables_schema": { + "language": {"type": "string"}, + "diff": {"type": "string"}, + }, + }, + ("code_review", "L2"): { + "body": """You are a code reviewer. Review the following code diff and identify potential bugs or issues. +For each issue, provide: +1. The defect type (one line) +2. The line number range where it occurs +3. A brief explanation + +Language: {{ language }} +Diff: +``` +{{ diff }} +``` + +Format each issue as: +- Type: + Lines: - + Explanation: """, + "variables_schema": { + "language": {"type": "string"}, + "diff": {"type": "string"}, + }, + }, + ("code_review", "L3"): { + "body": """You are a code reviewer. Review the following code diff and identify potential bugs or issues. +For each issue, provide: +1. The defect type (one line) +2. The line number range where it occurs +3. A brief explanation +4. A concrete fix suggestion + +Language: {{ language }} +Diff: +``` +{{ diff }} +``` + +Format each issue as: +- Type: + Lines: - + Explanation: + Fix: """, + "variables_schema": { + "language": {"type": "string"}, + "diff": {"type": "string"}, + }, + }, +} + + +def seed_default_templates(db: Session) -> None: + service = PromptService(db) + for (strategy_id, level), data in DEFAULT_TEMPLATES.items(): + existing = service.list_versions(strategy_id, level) + if existing: + continue + service.create_version( + strategy_id=strategy_id, + level=level, + body=data["body"], + variables_schema=data["variables_schema"], + ) diff --git a/backend/app/prompts/service.py b/backend/app/prompts/service.py new file mode 100644 index 0000000..cbe7a37 --- /dev/null +++ b/backend/app/prompts/service.py @@ -0,0 +1,113 @@ +"""Prompt template storage, versioning, and rendering service.""" + +from typing import Any, Dict, List, Optional + +from jinja2 import BaseLoader, Environment +from sqlalchemy.orm import Session + +from app.models import PromptTemplate, PromptTemplateVersion + + +class PromptService: + def __init__(self, db: Session): + self.db = db + self.jinja = Environment(loader=BaseLoader()) + + def get_or_create_template(self, strategy_id: str, level: str) -> PromptTemplate: + template = ( + self.db.query(PromptTemplate) + .filter_by(strategy_id=strategy_id, level=level) + .first() + ) + if not template: + template = PromptTemplate(strategy_id=strategy_id, level=level) + self.db.add(template) + self.db.commit() + self.db.refresh(template) + return template + + def create_version( + self, + strategy_id: str, + level: str, + body: str, + variables_schema: Optional[Dict[str, Any]] = None, + ) -> PromptTemplateVersion: + template = self.get_or_create_template(strategy_id, level) + next_version = ( + self.db.query(PromptTemplateVersion) + .filter_by(template_id=template.id) + .count() + + 1 + ) + version = PromptTemplateVersion( + template_id=template.id, + version_number=next_version, + body=body, + variables_schema=variables_schema or self._infer_schema(body), + ) + self.db.add(version) + self.db.commit() + self.db.refresh(version) + return version + + def get_version(self, version_id: str) -> Optional[PromptTemplateVersion]: + from uuid import UUID + + try: + return self.db.query(PromptTemplateVersion).filter_by(id=UUID(version_id)).first() + except ValueError: + return None + + def list_versions(self, strategy_id: str, level: str) -> List[PromptTemplateVersion]: + template = ( + self.db.query(PromptTemplate) + .filter_by(strategy_id=strategy_id, level=level) + .first() + ) + if not template: + return [] + return ( + self.db.query(PromptTemplateVersion) + .filter_by(template_id=template.id) + .order_by(PromptTemplateVersion.version_number) + .all() + ) + + def render( + self, + strategy_id: str, + version: Optional[int], + context: Dict[str, Any], + ) -> str: + template = ( + self.db.query(PromptTemplate) + .filter_by(strategy_id=strategy_id) + .first() + ) + if not template: + raise ValueError(f"Prompt template not found: {strategy_id}") + + query = self.db.query(PromptTemplateVersion).filter_by(template_id=template.id) + if version: + version_obj = query.filter_by(version_number=version).first() + else: + version_obj = query.order_by(PromptTemplateVersion.version_number.desc()).first() + + if not version_obj: + raise ValueError(f"Prompt version not found: {strategy_id} v{version}") + + jinja_template = self.jinja.from_string(version_obj.body) + return jinja_template.render(**context) + + def _infer_schema(self, body: str) -> Dict[str, Any]: + """Infer required variables from Jinja2 template.""" + from jinja2.meta import find_undeclared_variables + + ast = self.jinja.parse(body) + variables = find_undeclared_variables(ast) + return {var: {"type": "string"} for var in variables} + + +def get_prompt_service(db: Session) -> PromptService: + return PromptService(db) diff --git a/backend/fill_scores.py b/backend/fill_scores.py new file mode 100644 index 0000000..8755f21 --- /dev/null +++ b/backend/fill_scores.py @@ -0,0 +1,23 @@ +# 将 l3_scores.json 的 AI 预评分写回李克特评分表 Excel +import json +from openpyxl import load_workbook + +XLSX = r"C:\Users\eeymoo\Documents\毕业论文\实验结果\L3修复建议_李克特评分表.xlsx" +scores = json.load(open("l3_scores.json", encoding="utf-8")) + +wb = load_workbook(XLSX) +print("sheets:", wb.sheetnames) +ws = wb["打分表"] if "打分表" in wb.sheetnames else wb.active + +# 表头在第 2 行；C 列 run_id、L 列评分、M 列备注 +filled = 0 +for row in ws.iter_rows(min_row=3): + run_id = row[2].value # C 列 run_id + if run_id in scores: + score, reason = scores[run_id] + row[11].value = score # L 列 评分(1-5) + row[12].value = reason # M 列 备注 + filled += 1 + +wb.save(XLSX) +print(f"filled {filled} rows") diff --git a/backend/judge_all.py b/backend/judge_all.py new file mode 100644 index 0000000..d552756 --- /dev/null +++ b/backend/judge_all.py @@ -0,0 +1,112 @@ +"""Batch-judge all completed runs of an experiment with the LLM judge. + +Usage: python judge_all.py [judge_model] [concurrency] + +For every done run: the judge decides detection + false alarms (semantic, +level-agnostic), while line coverage is computed rule-based from parsed +findings. Idempotent: runs whose result already carries a judge verdict +(judge_verdict key in parsed_findings) are skipped. +""" + +import asyncio +import sys +import uuid + +from app.analysis.judge import ( + build_judge_prompt, + coverage_from_verdict, + parse_verdict, + verdict_to_metrics, +) +from app.analysis.parser import parse_output +from app.db import SessionLocal, init_db +from app.model_adapters.factory import create_adapter +from app.models import ExperimentRun + + +async def main() -> None: + experiment_id = uuid.UUID(sys.argv[1]) + judge_model = sys.argv[2] if len(sys.argv) > 2 else "deepseek" + concurrency = int(sys.argv[3]) if len(sys.argv) > 3 else 5 + + init_db() + db = SessionLocal() + adapter = create_adapter(judge_model) + + runs = ( + db.query(ExperimentRun) + .filter(ExperimentRun.experiment_id == experiment_id, ExperimentRun.status == "done") + .all() + ) + todo = [] + for run in runs: + if not run.result or not run.result.raw_output: + continue + pf = run.result.parsed_findings + if isinstance(pf, dict) and pf.get("judge_verdict"): + continue + todo.append(run) + print(f"{len(todo)} runs to judge (judge={judge_model}, concurrency={concurrency})") + + semaphore = asyncio.Semaphore(concurrency) + + async def judge_one(run): + async with semaphore: + gt_list = [ + { + "defect_type": d.defect_type, + "line_start": d.line_start, + "line_end": d.line_end, + "description": d.description, + "reference_fix": d.reference_fix, + } + for d in run.sample.defects + ] + gt = gt_list[0] if gt_list else {} + prompt = build_judge_prompt(run.result.raw_output, gt) + last_err = None + for _ in range(3): + try: + resp = await adapter.chat(prompt, {"temperature": 0.0, "max_tokens": 2048}) + verdict = parse_verdict(resp.text) + if verdict: + break + except Exception as e: # noqa: BLE001 + last_err = e + await asyncio.sleep(2) + else: + print(f"WARN judge failed for run {run.id}: {last_err or 'unparseable verdict'}") + return None + + metrics = verdict_to_metrics(verdict, n_ground_truth=len(gt_list) or 1) + coverage = coverage_from_verdict(verdict, gt_list) + level = run.template_version.template.level + findings = parse_output(run.result.raw_output, level) + + run.result.detection_rate = metrics["detection_rate"] + run.result.false_positive_rate = metrics["false_positive_rate"] + run.result.coverage_rate = coverage + run.result.parsed_findings = { + "judge_model": judge_model, + "judge_verdict": verdict, + "findings": [ + { + "defect_type": f.defect_type, + "line_start": f.line_start, + "line_end": f.line_end, + "description": f.description, + } + for f in findings + ], + } + db.commit() + return verdict + + results = await asyncio.gather(*(judge_one(run) for run in todo)) + done = sum(1 for r in results if r) + detected = sum(1 for r in results if r and r["detected"]) + print(f"judged {done}/{len(todo)}; detected {detected}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/backend/l3_runs.json b/backend/l3_runs.json new file mode 100644 index 0000000..1b679e2 --- /dev/null +++ b/backend/l3_runs.json @@ -0,0 +1,1298 @@ +[ + { + "run_id": "60a6e958-99e5-4d4f-ad82-88548fe4b039", + "model": "deepseek", + "lang": "java", + "repo": "gson", + "sha": "e28fdb9", + "rep": 1, + "gt": "Removed null-check guard for 'typeCreator'. (file: gson/src/main/java/com/google/gson/internal/ConstructorConstructor.java)", + "fix": "Add `if (typeCreator != null)` guard before dereferencing.", + "detected": true, + "output": "- Null pointer exception risk: `typeCreator` can be null, and the code now passes it directly to the constructor without a null check." + }, + { + "run_id": "8c00b35b-395e-4e42-8118-4589711f6b27", + "model": "deepseek", + "lang": "java", + "repo": "gson", + "sha": "e28fdb9", + "rep": 2, + "gt": "Removed null-check guard for 'typeCreator'. (file: gson/src/main/java/com/google/gson/internal/ConstructorConstructor.java)", + "fix": "Add `if (typeCreator != null)` guard before dereferencing.", + "detected": true, + "output": "- Null pointer risk: `typeCreator` can be `null`, but the code now unconditionally passes it to the constructor, which may fail if it expects a non-null value." + }, + { + "run_id": "25b63c53-9d23-41eb-bef1-82f9dbe2e39d", + "model": "deepseek", + "lang": "java", + "repo": "gson", + "sha": "e28fdb9", + "rep": 3, + "gt": "Removed null-check guard for 'typeCreator'. (file: gson/src/main/java/com/google/gson/internal/ConstructorConstructor.java)", + "fix": "Add `if (typeCreator != null)` guard before dereferencing.", + "detected": true, + "output": "- If `instanceCreators.get(type)` returns `null`, the code will now pass `null` to `InstanceCreatorConstructor`, potentially causing a `NullPointerException` or unexpected behavior, whereas previously it would skip to the next fallback logic." + }, + { + "run_id": "7dd615b3-0f3e-4491-a972-a82ecf74b37c", + "model": "deepseek", + "lang": "java", + "repo": "gson", + "sha": "545b802", + "rep": 1, + "gt": "Changed array boundary check to off-by-one (<= length). (file: gson/src/main/java/com/google/gson/stream/JsonWriter.java)", + "fix": "Use strict `< length` to avoid ArrayIndexOutOfBoundsException.", + "detected": true, + "output": "- The condition `last <= length` will cause `out.write(value, last, length - last)` to be called with `length - last` equal to 0 when `last == length`. While this writes zero bytes, it is unnecessary and could be considered a logic error if `out.write` throws an exception on zero-length writes or if the intent was to skip writing when no remaining characters exist. More critically, if `last` can ever exceed `length` (e.g., due to prior logic), the condition would still pass incorrectly, leading to a negative length in the write call. The original `last < length` correctly guards against both zero and negative remaining lengths." + }, + { + "run_id": "79b3908a-9eb8-48e7-8c42-7784d4cd97d1", + "model": "deepseek", + "lang": "java", + "repo": "gson", + "sha": "545b802", + "rep": 2, + "gt": "Changed array boundary check to off-by-one (<= length). (file: gson/src/main/java/com/google/gson/stream/JsonWriter.java)", + "fix": "Use strict `< length` to avoid ArrayIndexOutOfBoundsException.", + "detected": true, + "output": "- The condition `last <= length` will cause an out-of-bounds exception when `last == length`, because `StringWriter.write(String, int, int)` will attempt to read at index `length` with length `0`, which is invalid in Java (the offset must be strictly less than the string's length).\n- The change from `<` to `<=` also makes the `if` block execute when there is nothing left to write (i.e., `last == length`), which is unnecessary and potentially harmful." + }, + { + "run_id": "241623fc-69fa-4730-bc7b-fdfe75c5c4a6", + "model": "deepseek", + "lang": "java", + "repo": "gson", + "sha": "545b802", + "rep": 3, + "gt": "Changed array boundary check to off-by-one (<= length). (file: gson/src/main/java/com/google/gson/stream/JsonWriter.java)", + "fix": "Use strict `< length` to avoid ArrayIndexOutOfBoundsException.", + "detected": true, + "output": "- The condition `last <= length` will cause an out-of-bounds exception when `last == length`, because `out.write(value, last, length - last)` will be called with a length of 0 and a start index equal to the array length, which is invalid." + }, + { + "run_id": "610e852b-8a21-42eb-b708-7c71a78528a0", + "model": "deepseek", + "lang": "java", + "repo": "gson", + "sha": "e25d4fa", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/test/java/com/google/gson/functional/MapAsArrayTypeAdapterTest.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": false, + "output": "- The `equals` method now returns `true` for any object that is an instance of `Point`, regardless of its coordinates. This violates the contract of `equals` (objects with different `x`/`y` values will be considered equal)." + }, + { + "run_id": "3773ef9e-26df-47f1-ab54-c9e1b6f16e2d", + "model": "deepseek", + "lang": "java", + "repo": "gson", + "sha": "e25d4fa", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/test/java/com/google/gson/functional/MapAsArrayTypeAdapterTest.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": true, + "output": "- The equals method now returns `true` for any `Point` instance, regardless of whether the coordinates match (due to the `||` operator taking precedence over `&&`). \n- If `o` is not a `Point`, the method will throw a `ClassCastException` because the `instanceof` check is bypassed by the `||` short-circuit logic. \n- The equals contract is violated: objects with different `x`/`y` values are now considered equal, leading to incorrect behavior in collections (e.g., `HashSet`, `HashMap`)." + }, + { + "run_id": "bc64e4de-fe6b-4863-bbc5-c330629e9c39", + "model": "deepseek", + "lang": "java", + "repo": "gson", + "sha": "e25d4fa", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/test/java/com/google/gson/functional/MapAsArrayTypeAdapterTest.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": true, + "output": "- The `equals` method now returns `true` for any `Point` object (or any object that is an instance of `Point`), regardless of whether the coordinates match. This violates the contract of `equals` (reflexivity, symmetry, transitivity, consistency) and makes the method incorrect." + }, + { + "run_id": "3a303e33-2510-4316-98f7-2bd318dca4eb", + "model": "deepseek", + "lang": "java", + "repo": "gson", + "sha": "9835b6f", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/main/java/com/google/gson/internal/LinkedTreeMap.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": true, + "output": "- The condition now uses `||` instead of `&&`, which means `contains` will return `true` for any `Entry` instance, even if the entry is not actually present in the collection. This is a logic error.\n- The code will throw a `ClassCastException` if `o` is not an `Entry`, because the `findByEntry` call is still attempted in the right operand of `||` when `o` is not an `Entry`. The short-circuit only prevents evaluation if `o` is an `Entry`, but not if it isn't." + }, + { + "run_id": "04468fe8-047e-4cad-aad5-ebcca6b055f9", + "model": "deepseek", + "lang": "java", + "repo": "gson", + "sha": "9835b6f", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/main/java/com/google/gson/internal/LinkedTreeMap.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": true, + "output": "- The `contains` method now returns `true` for any `Entry` instance, even if the entry is not actually contained in the collection. This is a logical bug because it incorrectly reports that all entries are present." + }, + { + "run_id": "b9099ae9-230a-4e29-9909-937f482e320a", + "model": "deepseek", + "lang": "java", + "repo": "gson", + "sha": "9835b6f", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/main/java/com/google/gson/internal/LinkedTreeMap.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": true, + "output": "- The `||` operator causes a short-circuit evaluation, so if `o` is an `Entry`, the method always returns `true` without checking whether the entry actually exists in the map. This incorrectly reports non-existent entries as contained." + }, + { + "run_id": "fb2a55ae-ef4e-41be-b8e0-7ea3032679da", + "model": "deepseek", + "lang": "javascript", + "repo": "axios", + "sha": "2344aab", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: lib/adapters/fetch.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The condition changed from logical AND (`&&`) to logical OR (`||`), which incorrectly marks ReadableStream as supported even when fetch is not supported, potentially leading to runtime errors when code assumes both are available.\n- The check now returns `true` if either fetch or ReadableStream exists, but the code using this flag likely requires both to be present for correct behavior." + }, + { + "run_id": "56196232-3e5b-4f38-8e74-e60ca6b33e8a", + "model": "deepseek", + "lang": "javascript", + "repo": "axios", + "sha": "2344aab", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: lib/adapters/fetch.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The logical operator changed from `&&` to `||`, which incorrectly makes `isReadableStreamSupported` true if either fetch is supported OR `ReadableStream` exists, even when fetch is not supported. This can lead to using `ReadableStream` in code paths that require fetch, potentially causing runtime errors." + }, + { + "run_id": "f101adbf-3e6a-49db-8bf6-e72f6cbf9899", + "model": "deepseek", + "lang": "javascript", + "repo": "axios", + "sha": "2344aab", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: lib/adapters/fetch.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The logical operator changed from `&&` to `||`, which will incorrectly mark `isReadableStreamSupported` as true if either fetch is supported OR ReadableStream exists, even if the other requirement is missing. This weakens the condition and could lead to runtime errors when the required combination isn’t present." + }, + { + "run_id": "3bd471c9-466e-4692-a2ef-b9be2e3ead56", + "model": "deepseek", + "lang": "javascript", + "repo": "axios", + "sha": "a092bae", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/Axios.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The change from `headers && utils.merge(...)` to `headers || utils.merge(...)` will cause a runtime error when `headers` is falsy (e.g., `null` or `undefined`) because it attempts to access `headers.common` and `headers[config.method]` on a null/undefined value.\n- The logic is inverted: the original code intended to merge only when `headers` exists, but the new code only merges when `headers` is falsy, which is the opposite of the intended behavior.\n- When `headers` is an empty object or a valid object, the `||` operator will return `headers` directly, skipping the merge entirely, so the context headers will not be properly flattened.\n- If `headers` is `null`/`undefined`, the expression will throw a `TypeError` because it tries to access properties of `null`/`undefined`." + }, + { + "run_id": "9bc56114-66e9-46b3-ac1c-37e7f6848295", + "model": "deepseek", + "lang": "javascript", + "repo": "axios", + "sha": "a092bae", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/Axios.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The change from `headers && utils.merge(...)` to `headers || utils.merge(...)` will cause a runtime error if `headers` is `undefined` or `null`, because it will try to access `headers.common` and `headers[config.method]` on an undefined/null value." + }, + { + "run_id": "3666560f-b79f-45cd-a2e1-99f226d32b2d", + "model": "deepseek", + "lang": "javascript", + "repo": "axios", + "sha": "a092bae", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/Axios.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The change from `headers && utils.merge(...)` to `headers || utils.merge(...)` inverts the logic. When `headers` is truthy, the new code assigns the entire `headers` object to `contextHeaders` without merging the method-specific headers, so the method-specific headers (e.g., `headers.post`) will not be included. When `headers` is falsy, it tries to access `headers.common` which will throw a TypeError because `headers` is `undefined` or `null`." + }, + { + "run_id": "13a2d2d7-9483-4381-a7fa-7e2851a7af6d", + "model": "deepseek", + "lang": "javascript", + "repo": "axios", + "sha": "a8e4f13", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- Changing `&&` to `||` will cause `utils.merge` to be called even when only one of the objects is plain, which may produce unexpected behavior or incorrect merging when the other value is not an object (e.g., a primitive or array)." + }, + { + "run_id": "b80f2457-51ef-45fe-aa3f-c7d1985e0468", + "model": "deepseek", + "lang": "javascript", + "repo": "axios", + "sha": "a8e4f13", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The condition change from `&&` to `||` will cause `utils.merge` to be called with non-plain-object arguments, potentially leading to unexpected behavior or errors when only one of the values is a plain object.\n- When `target` is a plain object but `source` is not, the function will now incorrectly merge a non-object source, which could corrupt the merged result.\n- The logic now treats arrays, primitives, or other non-plain objects as mergeable when combined with a plain object, violating the original intent of only merging two plain objects." + }, + { + "run_id": "d1838879-aae6-46e1-a483-56b2f268f796", + "model": "deepseek", + "lang": "javascript", + "repo": "axios", + "sha": "a8e4f13", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The condition change from `&&` to `||` will incorrectly merge when only `target` is a plain object but `source` is not, potentially merging non-object values into the target unexpectedly." + }, + { + "run_id": "27ebde97-9a09-4811-a8d1-b104329c5949", + "model": "deepseek", + "lang": "javascript", + "repo": "axios", + "sha": "2af5c5f", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The condition change from `&&` to `||` means if either `target` or `source` is a plain object, the function will call `utils.merge` with both values. This can lead to unexpected behavior when `target` is a plain object but `source` is not (e.g., a string, number, array), as `utils.merge` may not handle non-object sources correctly or may mutate the target in unintended ways.\n- The original logic ensured both were plain objects before merging; now, a plain object `target` with a non-plain `source` will enter the first branch instead of falling through to the `else if` branch, potentially skipping the intended handling for non-plain sources." + }, + { + "run_id": "efe6536c-f53c-48e9-bbbe-00f056135368", + "model": "deepseek", + "lang": "javascript", + "repo": "axios", + "sha": "2af5c5f", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The condition change from `&&` to `||` will cause `utils.merge` to be called even when `target` is not a plain object (e.g., an array, string, or null), potentially leading to incorrect merging behavior or runtime errors.\n- When `target` is not a plain object but `source` is, the first branch will now execute instead of the second, passing the non-plain-object `target` directly into `merge`, which may corrupt the merged result or fail to handle the target correctly." + }, + { + "run_id": "e66821dd-175b-4f58-bef4-f2a903679094", + "model": "deepseek", + "lang": "javascript", + "repo": "axios", + "sha": "2af5c5f", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The condition changed from `&&` to `||`, which changes the logic flow: if either `target` or `source` is a plain object, the first branch executes. This can incorrectly merge when `target` is a plain object but `source` is not, or vice versa, leading to unexpected merged results (e.g., merging a non-object source into an object target, or merging an object source into a non-object target) instead of falling through to the appropriate fallback branches." + }, + { + "run_id": "f63dee02-7b3f-44e1-9850-b58bd847fe11", + "model": "deepseek", + "lang": "python", + "repo": "requests", + "sha": "a62a2d3", + "rep": 1, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "- The condition change from `and` to `or` will cause incorrect behavior when `client_cert` is not a tuple but has length 2 (e.g., a list). In such cases, it will still try to index `client_cert[0]` and `client_cert[1]`, which may not be valid or intended. \n- If `client_cert` is a tuple with length not equal to 2 (e.g., length 1 or 3), the `or` condition will still evaluate to `True` because `isinstance(client_cert, tuple)` is true, leading to an `IndexError` when accessing `client_cert[1]` (or `client_cert[0]` if empty). \n- If `client_cert` is a non-iterable object (e.g., an integer), the `len(client_cert)` call will raise a `TypeError`, which was previously avoided because the `and` short-circuited on the `isinstance` check." + }, + { + "run_id": "b019bc4e-2588-4126-8287-05fc9a8cccc0", + "model": "deepseek", + "lang": "python", + "repo": "requests", + "sha": "a62a2d3", + "rep": 2, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "- If `client_cert` is a tuple of length not equal to 2, the condition is still true (due to `or`), causing an `IndexError` when accessing `client_cert[1]`.\n- If `client_cert` is a non-tuple (e.g., a string) with length 2, the code will attempt to index it, potentially causing an error or unexpected behavior (e.g., treating a two-character string as a file path pair).\n- The change broadens the condition incorrectly, allowing invalid inputs to pass through and break the intended logic." + }, + { + "run_id": "61113778-bd49-4f4d-8539-3947b6bb9b08", + "model": "deepseek", + "lang": "python", + "repo": "requests", + "sha": "a62a2d3", + "rep": 3, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "- Changing `and` to `or` allows a non-tuple object with a length of 2 to be indexed, which will raise a `TypeError` if it’s not subscriptable (e.g., a string of length 2).\n- A tuple with length other than 2 (e.g., a 3-element tuple) will now pass the condition and cause an `IndexError` when accessing `client_cert[1]`.\n- A non-tuple iterable with exactly 2 elements (e.g., a list) will now pass the condition and may cause unintended behavior if the code expects a tuple specifically." + }, + { + "run_id": "9592df3d-b43d-4b4a-a435-d2807243c7fa", + "model": "deepseek", + "lang": "python", + "repo": "requests", + "sha": "0c030f7", + "rep": 1, + "gt": "Removed None-check guard for variable 'chardet'. (file: src/requests/packages.py)", + "fix": "Add `if chardet is not None:` guard before use.", + "detected": true, + "output": "- `target` is referenced but never defined in the new code, causing a `NameError` at runtime.\n- The condition `if chardet is not None` was removed, so the loop will execute even when `chardet` is `None` (if `target` were defined, it would fail or behave incorrectly).\n- The loop no longer checks whether `chardet` was imported, potentially iterating over unrelated modules and causing incorrect aliasing." + }, + { + "run_id": "7991529a-53e2-4a93-9278-46399e9ad33e", + "model": "deepseek", + "lang": "python", + "repo": "requests", + "sha": "0c030f7", + "rep": 2, + "gt": "Removed None-check guard for variable 'chardet'. (file: src/requests/packages.py)", + "fix": "Add `if chardet is not None:` guard before use.", + "detected": false, + "output": "- `target` is referenced before assignment if `chardet` is `None`, causing a `NameError`." + }, + { + "run_id": "6cf5d834-90f2-4cfa-914f-1e64c3523ef9", + "model": "deepseek", + "lang": "python", + "repo": "requests", + "sha": "0c030f7", + "rep": 3, + "gt": "Removed None-check guard for variable 'chardet'. (file: src/requests/packages.py)", + "fix": "Add `if chardet is not None:` guard before use.", + "detected": true, + "output": "- `target` is referenced but never defined in the new code (previously it was defined inside the `if chardet is not None` block)\n- The `chardet is not None` guard was removed, so the code may fail if `chardet` is None or undefined\n- The loop now runs unconditionally over all `sys.modules`, which could cause unnecessary processing or errors if `target` is undefined" + }, + { + "run_id": "d9af45af-d57c-4867-8153-9dcc94583b90", + "model": "deepseek", + "lang": "python", + "repo": "requests", + "sha": "145b539", + "rep": 1, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "- The change from `and` to `or` alters the logic so that `ssl_context` is set even when `has_poolmanager_ssl_context` is `True` and `verify` is `True`, which may override an existing valid SSL context with the preloaded one, potentially causing incorrect SSL verification behavior." + }, + { + "run_id": "778b3ea1-26c0-482d-bc1b-1e116f1147b2", + "model": "deepseek", + "lang": "python", + "repo": "requests", + "sha": "145b539", + "rep": 2, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "- Changing `and` to `or` changes the logic so that when `verify is True` but `has_poolmanager_ssl_context` is also true, the `ssl_context` is still set, which may override an existing valid context. \n- When `verify` is `True` and `has_poolmanager_ssl_context` is `False`, the condition now also matches the case where `verify` is `False` (since `or` short-circuits), but the original `elif` only executed when `verify` was truthy—this could cause unintended behavior for `verify=False` combined with `has_poolmanager_ssl_context=False`, incorrectly assigning the preloaded context. \n- The new condition may cause the `elif isinstance(verify, str)` branch to be skipped in cases where `verify` is a string but `has_poolmanager_ssl_context` is `False`, because the first branch is now more inclusive and will catch it, preventing proper string handling." + }, + { + "run_id": "5781e3d2-b79f-4ef8-b291-ff69d554c06f", + "model": "deepseek", + "lang": "python", + "repo": "requests", + "sha": "145b539", + "rep": 3, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "- Changing `and` to `or` alters the logic so that `ssl_context` is set even when `verify` is `False` (since `not has_poolmanager_ssl_context` could be true), which contradicts the intent of the preceding `cert_reqs` setting.\n\n- The new condition now also sets `ssl_context` when `verify` is `False` but `has_poolmanager_ssl_context` is `False`, potentially overriding the `CERT_NONE` behavior with a preloaded context that may require certificates, causing SSL verification to still occur despite `verify=False`." + }, + { + "run_id": "33552154-205c-4545-922c-dacc12f4e91c", + "model": "deepseek", + "lang": "python", + "repo": "requests", + "sha": "f8bec2f", + "rep": 1, + "gt": "Removed None-check guard for variable 'client_cert'. (file: src/requests/adapters.py)", + "fix": "Add `if client_cert is not None:` guard before use.", + "detected": true, + "output": "- The `if client_cert is not None:` condition was removed, so `client_cert` may be `None`, and accessing `client_cert[0]` will raise a `TypeError`.\n- The new code assumes `client_cert` is always a tuple, but the `else` branch (which handles a plain string path) is now unreachable because the `if` condition was removed. This breaks the documented behavior of allowing a single cert path string.\n- The indentation of the `pool_kwargs[\"cert_file\"] = client_cert[0]` and `pool_kwargs[\"key_file\"] = client_cert[1]` lines is inconsistent with the surrounding code, suggesting a logic block error (the `else` is now paired with the `if` that checks tuple length, but the `if` is missing)." + }, + { + "run_id": "0e8a0198-721f-4e4f-8a63-0bfa9762bfb3", + "model": "deepseek", + "lang": "python", + "repo": "requests", + "sha": "f8bec2f", + "rep": 2, + "gt": "Removed None-check guard for variable 'client_cert'. (file: src/requests/adapters.py)", + "fix": "Add `if client_cert is not None:` guard before use.", + "detected": true, + "output": "- The `if client_cert is not None:` check was removed, so the code now assumes `client_cert` is always a tuple with two elements. If `client_cert` is `None`, it will raise an error when trying to index it.\n- The `else` branch now incorrectly handles the case where `client_cert` is not a tuple (e.g., a string), but it will never be reached because the `if` condition is gone.\n- The indentation of the `pool_kwargs[\"cert_file\"] = client_cert[0]` and `pool_kwargs[\"key_file\"] = client_cert[1]` lines is wrong—they are inside a block that no longer exists, causing a syntax error or logic error." + }, + { + "run_id": "94b18699-5c10-4f3f-9b5c-33d70b4eb35f", + "model": "deepseek", + "lang": "python", + "repo": "requests", + "sha": "f8bec2f", + "rep": 3, + "gt": "Removed None-check guard for variable 'client_cert'. (file: src/requests/adapters.py)", + "fix": "Add `if client_cert is not None:` guard before use.", + "detected": true, + "output": "- The original `if client_cert is not None:` check was removed, so `client_cert` could be `None` and cause an error when accessing `client_cert[0]` and `client_cert[1]`.\n- The indentation of the new `pool_kwargs[\"cert_file\"] = client_cert[0]` and `pool_kwargs[\"key_file\"] = client_cert[1]` lines is incorrect (they appear to be inside the `else` block of a different conditional, not the `if` block they belong to).\n- The `else` clause now incorrectly handles the case where `client_cert` is not a tuple of length 2, but it also runs when `client_cert` is `None`, which is wrong.\n- The logic no longer checks `isinstance(client_cert, tuple)` or `len(client_cert) == 2`, so a non-tuple or tuple of wrong length will cause an indexing error." + }, + { + "run_id": "8b0d0aaa-081f-4589-8c5b-cc5e297bd839", + "model": "kimi", + "lang": "java", + "repo": "gson", + "sha": "e28fdb9", + "rep": 1, + "gt": "Removed null-check guard for 'typeCreator'. (file: gson/src/main/java/com/google/gson/internal/ConstructorConstructor.java)", + "fix": "Add `if (typeCreator != null)` guard before dereferencing.", + "detected": true, + "output": "- The unconditional return prevents the raw-type fallback logic from ever running.\n- `typeCreator` may be null when the type is absent from `instanceCreators`, and passing null to `InstanceCreatorConstructor` may cause a NullPointerException.\n- Code after the unconditional return is now unreachable (dead code)." + }, + { + "run_id": "6aa57faa-2326-49a5-82fd-6a111468c328", + "model": "kimi", + "lang": "java", + "repo": "gson", + "sha": "e28fdb9", + "rep": 2, + "gt": "Removed null-check guard for 'typeCreator'. (file: gson/src/main/java/com/google/gson/internal/ConstructorConstructor.java)", + "fix": "Add `if (typeCreator != null)` guard before dereferencing.", + "detected": true, + "output": "- Unconditional `return` bypasses all subsequent constructor-resolution fallbacks.\n- `typeCreator` may be `null`, which the original null check avoided.\n- Passing a potentially `null` `typeCreator` into `InstanceCreatorConstructor` risks a `NullPointerException`.\n- The unconditional return makes the following code unreachable, causing a compile-time error.\n- The surrounding indentation is inconsistent after the change." + }, + { + "run_id": "099b5e77-ae22-42ea-8546-505ec5c8e0aa", + "model": "kimi", + "lang": "java", + "repo": "gson", + "sha": "e28fdb9", + "rep": 3, + "gt": "Removed null-check guard for 'typeCreator'. (file: gson/src/main/java/com/google/gson/internal/ConstructorConstructor.java)", + "fix": "Add `if (typeCreator != null)` guard before dereferencing.", + "detected": true, + "output": "- `instanceCreators.get(type)` can return `null`, which is now passed directly to `InstanceCreatorConstructor` without a null check.\n- The unconditional `return` makes the subsequent raw-type matching code and any following logic unreachable." + }, + { + "run_id": "dd40cf99-2c0a-4275-8c90-a9dfe9b731c0", + "model": "kimi", + "lang": "java", + "repo": "gson", + "sha": "545b802", + "rep": 1, + "gt": "Changed array boundary check to off-by-one (<= length). (file: gson/src/main/java/com/google/gson/stream/JsonWriter.java)", + "fix": "Use strict `< length` to avoid ArrayIndexOutOfBoundsException.", + "detected": true, + "output": "- The modified condition `last <= length` triggers an unnecessary zero-length `out.write` call when `last` equals `length`, which is redundant and may cause unintended side effects depending on the `Writer` implementation." + }, + { + "run_id": "0b9735ad-ca6b-4601-a1a6-f2d32b64ddd3", + "model": "kimi", + "lang": "java", + "repo": "gson", + "sha": "545b802", + "rep": 2, + "gt": "Changed array boundary check to off-by-one (<= length). (file: gson/src/main/java/com/google/gson/stream/JsonWriter.java)", + "fix": "Use strict `< length` to avoid ArrayIndexOutOfBoundsException.", + "detected": true, + "output": "- The condition `last <= length` permits a zero-length write when `last` equals `length`, which is redundant and may have side effects depending on the `Writer` implementation." + }, + { + "run_id": "1e923a86-07b1-4531-9ceb-d05509079edf", + "model": "kimi", + "lang": "java", + "repo": "gson", + "sha": "545b802", + "rep": 3, + "gt": "Changed array boundary check to off-by-one (<= length). (file: gson/src/main/java/com/google/gson/stream/JsonWriter.java)", + "fix": "Use strict `< length` to avoid ArrayIndexOutOfBoundsException.", + "detected": true, + "output": "- The `<=` condition can cause a final write of 0 characters when `last` equals `length`, which is unnecessary and could fail with custom or strict `Writer` implementations that reject zero-length writes." + }, + { + "run_id": "e05c2805-828d-4e39-8ee1-01b32f154570", + "model": "kimi", + "lang": "java", + "repo": "gson", + "sha": "e25d4fa", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/test/java/com/google/gson/functional/MapAsArrayTypeAdapterTest.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": false, + "output": "- The `equals` method now returns `true` for any `Point` instance, ignoring the `x` and `y` values.\n- Passing a non-`Point` object causes a `ClassCastException` because the cast is performed without first verifying the type.\n- This violates the expected contract of `equals`, making distinct points incorrectly compare as equal." + }, + { + "run_id": "d36b6d9d-e9d6-47fe-bae9-f4ff80e76c8b", + "model": "kimi", + "lang": "java", + "repo": "gson", + "sha": "e25d4fa", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/test/java/com/google/gson/functional/MapAsArrayTypeAdapterTest.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": false, + "output": "- The equals method now returns true for any `Point` instance regardless of its coordinates.\n- Casting `o` to `Point` outside of the `instanceof` guard will throw a `ClassCastException` when `o` is not a `Point`." + }, + { + "run_id": "fb551ed2-5bdb-4708-bdce-bf6a64eeaa36", + "model": "kimi", + "lang": "java", + "repo": "gson", + "sha": "e25d4fa", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/test/java/com/google/gson/functional/MapAsArrayTypeAdapterTest.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": true, + "output": "- Changed `&&` to `||` causes any `Point` instance to be considered equal regardless of `x` and `y`.\n- When `o` is not a `Point`, the unchecked cast will throw a `ClassCastException`.\n- The `equals` implementation violates the `equals` contract." + }, + { + "run_id": "4050c09d-b27e-4ffa-a938-f49057627bd6", + "model": "kimi", + "lang": "java", + "repo": "gson", + "sha": "9835b6f", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/main/java/com/google/gson/internal/LinkedTreeMap.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": true, + "output": "- `contains` now returns `true` for any object that is an `Entry`, regardless of whether it is actually present in the set.\n- It may throw `ClassCastException` when `o` is not an `Entry` because the unconditional cast is still evaluated when `instanceof` is `false`.\n- It may pass `null` or an unrelated object to `findByEntry`, risking a `NullPointerException` or incorrect logic.\n- The changed condition violates the `Collection.contains` contract." + }, + { + "run_id": "8760a46d-1bba-4a76-b8d2-44338b90bc46", + "model": "kimi", + "lang": "java", + "repo": "gson", + "sha": "9835b6f", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/main/java/com/google/gson/internal/LinkedTreeMap.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": true, + "output": "- `contains` now returns `true` for any object that is an `Entry`, even if it is not actually present in the collection.\n- For objects that are not `Entry` instances, the unconditional cast to `Entry` can throw a `ClassCastException` instead of returning `false`.\n- The logical condition no longer requires both type-checking and successful lookup, breaking the expected semantics of `contains`." + }, + { + "run_id": "e5a35dbf-b76d-4402-aab8-c7ca17cfe6f5", + "model": "kimi", + "lang": "java", + "repo": "gson", + "sha": "9835b6f", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/main/java/com/google/gson/internal/LinkedTreeMap.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": true, + "output": "- `contains` will now return `true` for any `Entry` instance, regardless of whether it is actually present in the collection.\n- If the argument is not an `Entry`, the short-circuiting `||` fails to prevent the unsafe cast, which can throw a `ClassCastException`." + }, + { + "run_id": "f63431e4-6f3d-4c5a-9aed-c8fca65264d0", + "model": "kimi", + "lang": "javascript", + "repo": "axios", + "sha": "2344aab", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: lib/adapters/fetch.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The logic now considers ReadableStream supported if either `fetch` is supported or `ReadableStream` exists, rather than requiring both.\n- This can report ReadableStream support as true even when `fetch` is unavailable, which may cause runtime errors in code that assumes `fetch` is present.\n- This can also report ReadableStream support as true when `fetch` is available but `ReadableStream` is not actually a function." + }, + { + "run_id": "b38dc15b-94f0-4d5a-b984-839cf485659c", + "model": "kimi", + "lang": "javascript", + "repo": "axios", + "sha": "2344aab", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: lib/adapters/fetch.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- `isReadableStreamSupported` may be `true` when `ReadableStream` is not actually available, as long as `isFetchSupported` is `true`.\n- This creates a false positive that can lead to code attempting to use an undefined `ReadableStream`.\n- Conversely, it changes the intended dependency so the flag no longer requires both fetch and `ReadableStream` support together." + }, + { + "run_id": "b9f4982d-f3bb-4882-b903-b92b241da947", + "model": "kimi", + "lang": "javascript", + "repo": "axios", + "sha": "2344aab", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: lib/adapters/fetch.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- Changing the condition from `&&` to `||` marks ReadableStream as supported even when `fetch` is unavailable.\n- This can enable code paths that depend on `fetch` plus streaming, causing runtime errors in environments with `ReadableStream` but no `fetch`.\n- The logic no longer accurately represents \"ReadableStream support in the context of fetch\"." + }, + { + "run_id": "e5bc12ac-ac39-43fe-a1af-0d9e12c3ca2e", + "model": "kimi", + "lang": "javascript", + "repo": "axios", + "sha": "a092bae", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/Axios.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The `||` operator short-circuits when `headers` is truthy, causing `contextHeaders` to be assigned the entire `headers` object instead of the merged `common` and method-specific headers.\n- Method-specific and common headers are no longer merged when headers are provided.\n- The header flattening logic is effectively bypassed for the common case where `headers` exists." + }, + { + "run_id": "e69d92c0-bdcf-4f0b-9659-38fd7117cd85", + "model": "kimi", + "lang": "javascript", + "repo": "axios", + "sha": "a092bae", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/Axios.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- When `headers` is truthy, `contextHeaders` becomes the entire `headers` object instead of the merged result of `headers.common` and `headers[config.method]`, breaking header flattening.\n- When `headers` is falsy, the right-hand side is still evaluated and accesses properties on `null`/`undefined`, causing a `TypeError`.\n- `contextHeaders` may reference the same object as `headers`, so later deletion of method-specific header keys mutates the same object unexpectedly, producing incorrect merges and side effects." + }, + { + "run_id": "441de8cd-1697-4155-90fc-1c92059980f1", + "model": "kimi", + "lang": "javascript", + "repo": "axios", + "sha": "a092bae", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/Axios.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- `headers || utils.merge(...)` makes `contextHeaders` equal the entire `headers` object when `headers` is truthy, so `headers.common` and `headers[config.method]` are never merged.\n- When `headers` is falsy (`null`/`undefined`), the right-hand side evaluates `utils.merge(headers.common, headers[config.method])` and will throw a TypeError trying to read properties of `headers`.\n- If `headers` is truthy, `contextHeaders` points directly to the original headers object, so later mutations (deleting method/common keys) can unexpectedly affect the source object." + }, + { + "run_id": "b9ce7649-abdd-403e-977b-308470bf1d16", + "model": "kimi", + "lang": "javascript", + "repo": "axios", + "sha": "a8e4f13", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The condition now triggers when only one of `target` or `source` is a plain object, so a non-plain object may be passed into `utils.merge`, potentially causing incorrect merging behavior.\n- If `target` is not a plain object but `source` is, `target` itself may be mutated or returned unexpectedly instead of merging into a new object.\n- If `source` is not a plain object but `target` is, `utils.merge` may receive an unsupported source type, leading to malformed output or runtime errors." + }, + { + "run_id": "e587a125-3029-4e71-be4f-9c450b2d9300", + "model": "kimi", + "lang": "javascript", + "repo": "axios", + "sha": "a8e4f13", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": false, + "output": "- The second `else if (utils.isPlainObject(source))` branch is now unreachable dead code because the first condition already covers all cases where `source` is a plain object.\n- When `target` is a plain object but `source` is not, the function now incorrectly merges a non-plain-object value into a plain object, potentially causing type corruption or unintended property mutation.\n- The `caseless` merge behavior may now be applied to mismatched types (e.g., plain object + array/primitive), which the original logic explicitly avoided.\n- If `utils.merge` mutates its first argument, passing a non-plain-object `source` or a plain-object `target` where the original branch would not have triggered could introduce unexpected side effects." + }, + { + "run_id": "abb8b86a-bc7f-44cc-b7ae-4748736479e7", + "model": "kimi", + "lang": "javascript", + "repo": "axios", + "sha": "a8e4f13", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- Merging is now triggered when only `target` or only `source` is a plain object, which may incorrectly mix object and non-object values.\n- Passing a non-plain-object value into `utils.merge` may cause unexpected coercion or property iteration.\n- The branch that specifically handled a plain `source` has become unreachable.\n- The change may mutate or pollute `target` when `source` is not a plain object." + }, + { + "run_id": "0824d180-dba6-46e5-9965-5ad0afc5fcc2", + "model": "kimi", + "lang": "javascript", + "repo": "axios", + "sha": "2af5c5f", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- Changing `&&` to `||` allows non-plain-object values to be passed into `utils.merge`, which may produce incorrect results or unintended mutations.\n- When `target` is not a plain object but `source` is, `target` is used as the merge destination instead of an empty object.\n- When `source` is not a plain object but `target` is, a non-plain `source` may be merged into `target`.\n- The subsequent `else if (utils.isPlainObject(source))` branch becomes unreachable/dead code." + }, + { + "run_id": "6a9057fb-8f63-4b50-9734-abbd45c6d559", + "model": "kimi", + "lang": "javascript", + "repo": "axios", + "sha": "2af5c5f", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The condition now triggers when only one of `target` or `source` is a plain object, so `utils.merge` may receive a non-plain object or primitive.\n- If `target` is a plain object but `source` is not, `target` can be mutated and non-plain values may be merged in unexpectedly.\n- If `source` is a plain object but `target` is not, the merge may produce an incorrect result for non-plain targets like arrays.\n- The `else if (utils.isPlainObject(source))` branch becomes unreachable because the new `||` condition already handles every case where `source` is a plain object.\n- The semantic intent of merging two plain objects is broken." + }, + { + "run_id": "440f5d47-9cb8-4045-a598-dff736a8ddb7", + "model": "kimi", + "lang": "javascript", + "repo": "axios", + "sha": "2af5c5f", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- Changing the condition from `&&` to `||` means a plain object may now be merged with a non-plain value, leading to incorrect or unexpected merging behavior.\n- The subsequent `else if (utils.isPlainObject(source))` branch is now unreachable.\n- The `caseless` merge may be applied in cases where it was not intended, such as when `target` is a plain object but `source` is not." + }, + { + "run_id": "58075bc7-dffa-4179-a710-f9a464a9477a", + "model": "kimi", + "lang": "python", + "repo": "requests", + "sha": "a62a2d3", + "rep": 1, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "- The condition now uses `or` instead of `and`, so any two-element iterable (including strings, lists, etc.) will be incorrectly unpacked as `cert_file`/`key_file`.\n- Tuple values with a length other than `2` will now enter the branch and cause an `IndexError` or assign the wrong values.\n- The change broadens accepted types beyond the intended `(cert, key)` pair semantics, risking misconfiguration." + }, + { + "run_id": "4d1cdddc-6d50-4cef-9070-f6e190ac8f95", + "model": "kimi", + "lang": "python", + "repo": "requests", + "sha": "a62a2d3", + "rep": 2, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "- The new condition `isinstance(client_cert, tuple) or len(client_cert) == 2` allows non-tuple objects of length 2 to reach the tuple indexing branch, which may not be valid.\n- If `client_cert` is a tuple of length other than 2, the condition is true but the code unconditionally indexes `[0]` and `[1]`, causing an `IndexError` for single-element tuples and ignoring extra elements for longer tuples.\n- The original logic required `client_cert` to be both a tuple and have length 2; the new logic loses the length check for tuples." + }, + { + "run_id": "c623d3cc-77fc-478c-9825-40e47f350111", + "model": "kimi", + "lang": "python", + "repo": "requests", + "sha": "a62a2d3", + "rep": 3, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "- The condition treats a length-2 string as a certificate/key pair.\n- The condition no longer enforces that length-2 values are tuples, so arbitrary two-element sequences may be accepted as cert/key pairs.\n- If `client_cert` is a tuple whose length is not `2`, accessing `client_cert[0]` and `client_cert[1]` raises an `IndexError`.\n- If `client_cert` is not a sized object, calling `len(client_cert)` may raise a `TypeError`." + }, + { + "run_id": "5a764dd0-66ad-4b2c-9d44-2640d94959ca", + "model": "kimi", + "lang": "python", + "repo": "requests", + "sha": "0c030f7", + "rep": 1, + "gt": "Removed None-check guard for variable 'chardet'. (file: src/requests/packages.py)", + "fix": "Add `if chardet is not None:` guard before use.", + "detected": true, + "output": "- `target` is used but is no longer defined in the visible code, which will cause a `NameError`.\n- The `if chardet is not None:` guard is removed, so the code no longer skips chardet aliasing when `chardet` is unavailable.\n- Without assigning `target = chardet.__name__`, the module aliasing may use an incorrect or stale target value." + }, + { + "run_id": "9246543f-13d8-44de-a056-4f96d4cb5dcf", + "model": "kimi", + "lang": "python", + "repo": "requests", + "sha": "0c030f7", + "rep": 2, + "gt": "Removed None-check guard for variable 'chardet'. (file: src/requests/packages.py)", + "fix": "Add `if chardet is not None:` guard before use.", + "detected": true, + "output": "- `target` is referenced but no longer defined, which will raise a `NameError` at runtime.\n- `chardet` is referenced in the removed guard but its absence can cause the loop to run unconditionally, breaking when `chardet` is not available.\n- Mapping logic for the actual package name (e.g., `charset_normalizer` vs `chardet`) is lost, so `requests.packages.chardet` aliases may not be created correctly." + }, + { + "run_id": "8435fe89-1c9d-4752-a387-a1cb7c57da15", + "model": "kimi", + "lang": "python", + "repo": "requests", + "sha": "0c030f7", + "rep": 3, + "gt": "Removed None-check guard for variable 'chardet'. (file: src/requests/packages.py)", + "fix": "Add `if chardet is not None:` guard before use.", + "detected": true, + "output": "- The variable `target` is used but is no longer assigned, so it will be undefined.\n- The `chardet is not None` guard was removed, meaning the loop can run and fail when `chardet` is unavailable." + }, + { + "run_id": "7586c40e-cb83-4a86-983f-f59c5e6668c6", + "model": "kimi", + "lang": "python", + "repo": "requests", + "sha": "145b539", + "rep": 1, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "- The new `or` condition causes string `verify` paths to fall into the preloaded-ssl-context branch whenever `has_poolmanager_ssl_context` is `False`, bypassing the `isinstance(verify, str)` certificate-path handling.\n- It also applies `_preloaded_ssl_context` when `verify is True` even if `has_poolmanager_ssl_context` is `True`, potentially overriding an existing PoolManager SSL context." + }, + { + "run_id": "aeba23c7-8be9-45cf-a578-00fd40bc6da7", + "model": "kimi", + "lang": "python", + "repo": "requests", + "sha": "145b539", + "rep": 2, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "- The changed condition now triggers when `verify` is `True` even if `has_poolmanager_ssl_context` is `True`, which can force an unnecessary or incorrect preloaded SSL context.\n- It also triggers for string `verify` values whenever `has_poolmanager_ssl_context` is `False`, causing custom CA-bundle paths to be skipped.\n- The `isinstance(verify, str)` branch becomes unreachable for string `verify` values when `has_poolmanager_ssl_context` is `False`." + }, + { + "run_id": "d61481bb-e7cf-40fc-a5bb-778e528d1e8a", + "model": "kimi", + "lang": "python", + "repo": "requests", + "sha": "145b539", + "rep": 3, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "- The condition now assigns `_preloaded_ssl_context` when `verify` is `True` even if `has_poolmanager_ssl_context` is `True`, overriding an existing SSL context.\n- When `verify` is a string (path to a CA bundle) and `has_poolmanager_ssl_context` is `False`, the modified branch short-circuits before the `isinstance(verify, str)` handling, preventing proper CA bundle configuration.\n- The change applies a preloaded SSL context to non-boolean, non-`False` values of `verify` whenever `has_poolmanager_ssl_context` is `False`, which is broader than the original intent.\n- Changing `and` to `or` breaks the intended guard: the preloaded context should only be used when `verify` is explicitly `True` and no SSL context already exists." + }, + { + "run_id": "44bc02fc-14cf-4557-999e-304a911d8478", + "model": "kimi", + "lang": "python", + "repo": "requests", + "sha": "f8bec2f", + "rep": 1, + "gt": "Removed None-check guard for variable 'client_cert'. (file: src/requests/adapters.py)", + "fix": "Add `if client_cert is not None:` guard before use.", + "detected": true, + "output": "- The `else` clause has no matching `if`, producing invalid indentation/syntax.\n- `client_cert` is indexed with `[0]` and `[1]` unconditionally, so `None`, a string, or a tuple of the wrong length will raise a `TypeError` or `IndexError`.\n- A string `client_cert` path is no longer treated as a single cert file path; instead, only its first character would be used.\n- The `None` guard around the client cert handling was removed, so the code will always execute even when no client cert is provided." + }, + { + "run_id": "35357d01-5e38-4edd-aef2-d77bd60569c2", + "model": "kimi", + "lang": "python", + "repo": "requests", + "sha": "f8bec2f", + "rep": 2, + "gt": "Removed None-check guard for variable 'client_cert'. (file: src/requests/adapters.py)", + "fix": "Add `if client_cert is not None:` guard before use.", + "detected": true, + "output": "- `client_cert` is no longer checked for `None`, which will cause a `TypeError` if it is `None`.\n- The tuple/`isinstance` validation is missing, so a plain string certificate path will be incorrectly indexed (`client_cert[0]` and `client_cert[1]`) rather than used as-is.\n- The changed code is over-indented and contains an `else:` with no matching `if`, resulting in a syntax/`IndentationError`.\n- The original behavior of accepting just a client cert path in the non-tuple case is broken." + }, + { + "run_id": "28d66204-a406-4739-8622-b5ffdae05c7b", + "model": "kimi", + "lang": "python", + "repo": "requests", + "sha": "f8bec2f", + "rep": 3, + "gt": "Removed None-check guard for variable 'client_cert'. (file: src/requests/adapters.py)", + "fix": "Add `if client_cert is not None:` guard before use.", + "detected": true, + "output": "- `else` branch has no matching `if`, causing a syntax error.\n- The `None` guard for `client_cert` was removed, so `None` may be indexed or assigned as a certificate path.\n- The tuple type and length check was removed, so non-tuple or wrong-length `client_cert` values will raise indexing errors.\n- The single-string client certificate fallback behavior is broken because the tuple branch unconditionally indexes `client_cert`.\n- Indentation of the new block is inconsistent with the surrounding code, likely causing an indentation error." + }, + { + "run_id": "5c699718-681e-4eda-b504-8fa6b6e94530", + "model": "qwen", + "lang": "java", + "repo": "gson", + "sha": "e28fdb9", + "rep": 1, + "gt": "Removed null-check guard for 'typeCreator'. (file: gson/src/main/java/com/google/gson/internal/ConstructorConstructor.java)", + "fix": "Add `if (typeCreator != null)` guard before dereferencing.", + "detected": true, + "output": "Potential NullPointerException when typeCreator is null." + }, + { + "run_id": "77c54e11-d2a7-4dbf-b5f2-164f249b3550", + "model": "qwen", + "lang": "java", + "repo": "gson", + "sha": "e28fdb9", + "rep": 2, + "gt": "Removed null-check guard for 'typeCreator'. (file: gson/src/main/java/com/google/gson/internal/ConstructorConstructor.java)", + "fix": "Add `if (typeCreator != null)` guard before dereferencing.", + "detected": true, + "output": "- Potential NullPointerException: `typeCreator` can be `null` if `instanceCreators.get(type)` returns `null`, leading to a null reference being passed to the `InstanceCreatorConstructor` constructor.\n- Logic error: The check for `typeCreator != null` was removed, meaning the fallback logic (\"Next try raw type match\") will no longer be executed if a match isn't found in the `instanceCreators` map." + }, + { + "run_id": "3ac7d755-afd7-4aa7-a705-6f6cdbb4ad60", + "model": "qwen", + "lang": "java", + "repo": "gson", + "sha": "e28fdb9", + "rep": 3, + "gt": "Removed null-check guard for 'typeCreator'. (file: gson/src/main/java/com/google/gson/internal/ConstructorConstructor.java)", + "fix": "Add `if (typeCreator != null)` guard before dereferencing.", + "detected": true, + "output": "Potential NullPointerException if `typeCreator` is null.\nUnreachable code introduced below the return statement." + }, + { + "run_id": "7ee913bd-7d41-46ee-a154-68d5395052cc", + "model": "qwen", + "lang": "java", + "repo": "gson", + "sha": "545b802", + "rep": 1, + "gt": "Changed array boundary check to off-by-one (<= length). (file: gson/src/main/java/com/google/gson/stream/JsonWriter.java)", + "fix": "Use strict `< length` to avoid ArrayIndexOutOfBoundsException.", + "detected": true, + "output": "- The condition `last <= length` will cause an index out of bounds error when `last` equals `length` because `value[last]` would be accessed, which is invalid for a zero-based index." + }, + { + "run_id": "a4d4a3a6-0f15-4f6f-8ef3-0691cf787974", + "model": "qwen", + "lang": "java", + "repo": "gson", + "sha": "545b802", + "rep": 2, + "gt": "Changed array boundary check to off-by-one (<= length). (file: gson/src/main/java/com/google/gson/stream/JsonWriter.java)", + "fix": "Use strict `< length` to avoid ArrayIndexOutOfBoundsException.", + "detected": true, + "output": "- The condition `last <= length` causes an off-by-one error, potentially writing out-of-bounds data." + }, + { + "run_id": "5b33523e-6441-4ce0-b206-5916650b1206", + "model": "qwen", + "lang": "java", + "repo": "gson", + "sha": "545b802", + "rep": 3, + "gt": "Changed array boundary check to off-by-one (<= length). (file: gson/src/main/java/com/google/gson/stream/JsonWriter.java)", + "fix": "Use strict `< length` to avoid ArrayIndexOutOfBoundsException.", + "detected": true, + "output": "- The condition `last <= length` can cause an out-of-bounds access when `last` equals `length`, as it will attempt to write a substring starting at `length` with a length of `0`, which is invalid in this context." + }, + { + "run_id": "e439022d-791a-4079-8657-5f7268f05ec9", + "model": "qwen", + "lang": "java", + "repo": "gson", + "sha": "e25d4fa", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/test/java/com/google/gson/functional/MapAsArrayTypeAdapterTest.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": true, + "output": "- The logical operator in the `equals` method is incorrect, using `||` instead of `&&`, which would cause incorrect comparison logic and potential ClassCastException if `o` is not a `Point`." + }, + { + "run_id": "53bd06a1-8d20-4222-a89e-12e4e0ceb8aa", + "model": "qwen", + "lang": "java", + "repo": "gson", + "sha": "e25d4fa", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/test/java/com/google/gson/functional/MapAsArrayTypeAdapterTest.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": true, + "output": "The logical operator in the `equals` method is incorrectly changed from `&&` to `||`, which causes the method to always return `true` if the object is an instance of `Point`, regardless of the `x` and `y` values. \nThe logic violates the contract of the `equals` method, as it would incorrectly consider two different `Point` objects as equal if they are both instances of `Point` but have different coordinates. \nThe code will throw a `ClassCastException` if `o` is not an instance of `Point` because the cast `(Point) o` is attempted without proper type checking due to the `||` operator." + }, + { + "run_id": "e528d730-288e-434b-bd44-49801536c816", + "model": "qwen", + "lang": "java", + "repo": "gson", + "sha": "e25d4fa", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/test/java/com/google/gson/functional/MapAsArrayTypeAdapterTest.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": true, + "output": "1. The logical operator in the `equals` method is incorrect (`||` instead of `&&`), causing the method to always return `true` if the object is an instance of `Point`, regardless of the coordinate values.\n2. The `equals` method violates the contract by potentially returning `true` for objects that are not equal in value.\n3. If the object is not an instance of `Point`, the right-hand side of the `||` will still attempt to cast it to `Point`, which could cause a `ClassCastException`." + }, + { + "run_id": "c8cb246f-a361-455d-b9e6-b491ba729bb3", + "model": "qwen", + "lang": "java", + "repo": "gson", + "sha": "9835b6f", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/main/java/com/google/gson/internal/LinkedTreeMap.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": true, + "output": "The logical operator in the `contains` method was changed from `&&` to `||`. This causes the method to always return `true` if the object `o` is an instance of `Entry`, regardless of whether the entry actually exists in the collection. This breaks the contract of the `contains` method, which should only return `true` if the element is present. Additionally, this change introduces a potential `ClassCastException` because if `o` is not an instance of `Entry`, the unconditionally executed `findByEntry((Entry) o)` will attempt an invalid cast." + }, + { + "run_id": "8e5bde72-88c5-4380-b69b-f337858ef808", + "model": "qwen", + "lang": "java", + "repo": "gson", + "sha": "9835b6f", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/main/java/com/google/gson/internal/LinkedTreeMap.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": true, + "output": "- Incorrect boolean logic operator used; changes a logical 'and' to 'or', which will cause the method to return true for any object that is an instance of Entry, regardless of whether it exists in the collection, and will potentially throw a ClassCastException for non-Entry objects when passed to findByEntry." + }, + { + "run_id": "00d028e4-8adb-4ca5-a8d7-a4ede36967d6", + "model": "qwen", + "lang": "java", + "repo": "gson", + "sha": "9835b6f", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: gson/src/main/java/com/google/gson/internal/LinkedTreeMap.java)", + "fix": "Restore `&&` for correct conjunction semantics.", + "detected": true, + "output": "The logic operator in the `contains` method was changed from `&&` to `||`, which breaks the intended functionality. The method now checks if `o` is an instance of `Entry` OR if `findByEntry` returns non-null, instead of requiring both conditions. This can lead to incorrect results, especially when `o` is not an instance of `Entry`, potentially causing a `ClassCastException` when casting `o` to `Entry` in the `findByEntry` call." + }, + { + "run_id": "1d33128b-1898-4c8c-af61-2b62de022659", + "model": "qwen", + "lang": "javascript", + "repo": "axios", + "sha": "2344aab", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: lib/adapters/fetch.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- Incorrect logical operator: changing `&&` to `||` for `isReadableStreamSupported` may incorrectly mark the stream as supported even if `isFetchSupported` is false, leading to potential runtime errors when attempting to use fetch-dependent features." + }, + { + "run_id": "b284228f-9ede-433a-a9db-df43627a2328", + "model": "qwen", + "lang": "javascript", + "repo": "axios", + "sha": "2344aab", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: lib/adapters/fetch.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The logic for checking if `ReadableStream` is supported has been changed from `&&` (AND) to `||` (OR). This means that if either `isFetchSupported` is true OR `isFunction(ReadableStream)` is true, `isReadableStreamSupported` will be true, regardless of the other condition. This is likely incorrect because `ReadableStream` support should depend on both `fetch` being supported and `ReadableStream` actually being a function. The change could lead to incorrect feature detection behavior." + }, + { + "run_id": "478aabc9-8811-4374-ad7d-22082a763a1a", + "model": "qwen", + "lang": "javascript", + "repo": "axios", + "sha": "2344aab", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: lib/adapters/fetch.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- Incorrect logic operator used: `||` (OR) instead of `&&` (AND) for `isReadableStreamSupported` assignment, which may result in `true` even when `isFetchSupported` is `false`." + }, + { + "run_id": "3314cf51-90c3-4ed0-93e0-b0ae95ef5286", + "model": "qwen", + "lang": "javascript", + "repo": "axios", + "sha": "a092bae", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/Axios.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "The code incorrectly merges headers even when `headers` is null or undefined, leading to a TypeError.\nThe logic relies on short-circuit evaluation which returns the non-boolean value of `headers` rather than ensuring `contextHeaders` is always the result of a merge operation.\n`headers.common` and `headers[config.method]` will be accessed regardless of whether `headers` is an object, causing runtime errors." + }, + { + "run_id": "d55bd2c3-0394-494c-b4fe-18a2191384ca", + "model": "qwen", + "lang": "javascript", + "repo": "axios", + "sha": "a092bae", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/Axios.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": false, + "output": "- Potential TypeError: `headers.common` and `headers[config.method]` will be accessed even when `headers` is falsy, causing a runtime error if `headers` is undefined or null.\n- Incorrect logic: The fallback to `headers` when `headers` is falsy is likely unintended, as it assigns `headers` (which could be undefined or null) to `contextHeaders` instead of merging the common and method-specific headers." + }, + { + "run_id": "71cb0124-fc11-4b9e-b99b-b632b3bdfc79", + "model": "qwen", + "lang": "javascript", + "repo": "axios", + "sha": "a092bae", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/Axios.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The expression `headers || utils.merge(...)` will bypass the merge operation entirely if `headers` is truthy, preventing the intended flattening of headers.\n- If `headers` is falsy, `utils.merge` will be called with `undefined.common` and `undefined[config.method]`, which will likely cause a runtime error.\n- The logic change incorrectly assumes that `headers` being truthy means it is already the final flattened result, ignoring the need to merge common and method-specific headers." + }, + { + "run_id": "904cd26b-f68f-4792-95be-75928e5df864", + "model": "qwen", + "lang": "javascript", + "repo": "axios", + "sha": "a8e4f13", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "- The logical operator in the first condition was changed from `&&` (AND) to `||` (OR), which likely introduces a logic error. This change causes the merge operation to execute if either `target` or `source` is a plain object, rather than only when both are plain objects. This can lead to unexpected behavior or runtime errors when attempting to merge incompatible types (e.g., an object and a non-object)." + }, + { + "run_id": "5d03a902-90df-4bc4-942d-4b3e7daa07af", + "model": "qwen", + "lang": "javascript", + "repo": "axios", + "sha": "a8e4f13", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "The logical operator in the first condition was changed from `&&` to `||`, which causes the function to merge objects even when one of the arguments is not a plain object, potentially leading to incorrect behavior or runtime errors." + }, + { + "run_id": "920a0a2e-adbf-4fec-b9a1-843eb3c88448", + "model": "qwen", + "lang": "javascript", + "repo": "axios", + "sha": "a8e4f13", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "1. The logical operator in the first `if` condition was changed from `&&` (AND) to `||` (OR), which changes the logic to merge objects if either `target` or `source` is a plain object, rather than only when both are plain objects. This could lead to incorrect merging behavior when only one of them is a plain object. \n2. The change may cause the function to skip the `else if` block unexpectedly, potentially bypassing intended fallback logic. \n3. If `target` is not a plain object but `source` is, the function may still attempt to merge, which could produce unintended results." + }, + { + "run_id": "b6ed21e1-581e-474d-9c84-13b3eea57c95", + "model": "qwen", + "lang": "javascript", + "repo": "axios", + "sha": "2af5c5f", + "rep": 1, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "* Incorrect logical operator in condition: changing `&&` to `||` will cause the merge logic to execute even when only one of the arguments is a plain object, potentially leading to unexpected behavior or errors. \n* The second condition (`else if (utils.isPlainObject(source))`) may become unreachable or redundant if the first condition with `||` is always true when `source` is a plain object. \n* Potential for incorrect merging behavior when `target` is not a plain object but `source` is, as the merge operation may not handle this case correctly." + }, + { + "run_id": "0635f620-0e27-4803-8411-580610ebdc10", + "model": "qwen", + "lang": "javascript", + "repo": "axios", + "sha": "2af5c5f", + "rep": 2, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "* Incorrect logical operator usage (OR instead of AND) in the condition for merging plain objects, which can lead to unexpected merging behavior when only one of the arguments is a plain object." + }, + { + "run_id": "bc8324f6-65ba-41e2-a2e9-85f22fd35b31", + "model": "qwen", + "lang": "javascript", + "repo": "axios", + "sha": "2af5c5f", + "rep": 3, + "gt": "Replaced boolean `&&` with `||`. (file: lib/core/mergeConfig.js)", + "fix": "Restore `&&` for correct short-circuit logic.", + "detected": true, + "output": "1. Incorrect logical operator: The condition changed from `&&` to `||`, which changes the logic entirely and likely leads to unintended behavior (e.g., merging when `target` is not an object or when `source` is not an object).\n2. Redundant condition: The second `else if` check for `utils.isPlainObject(source)` is now redundant because the first condition (with `||`) will always capture cases where `source` is a plain object.\n3. Potential runtime error: The `utils.merge` call may fail if `target` or `source` are not plain objects when the `||` condition evaluates to `true`.\n4. Unreachable code: The second `else if` branch may become unreachable depending on the logic flow after the change." + }, + { + "run_id": "ea252238-71e2-41ae-896f-02df20cc20d5", + "model": "qwen", + "lang": "python", + "repo": "requests", + "sha": "a62a2d3", + "rep": 1, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "* The condition `isinstance(client_cert, tuple) or len(client_cert) == 2` is logically incorrect and will likely cause a runtime error.\n* If `client_cert` is not a tuple, calling `len(client_cert)` will raise a `TypeError` for non-iterable types.\n* The condition allows non-tuple iterables (like lists or strings) to pass, which may lead to incorrect indexing or type errors if the structure is not exactly as expected." + }, + { + "run_id": "35f3b71f-2151-4dd5-a0c8-0529f1ebe0ea", + "model": "qwen", + "lang": "python", + "repo": "requests", + "sha": "a62a2d3", + "rep": 2, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "The condition uses a logical OR instead of a logical AND, causing `client_cert[0]` and `client_cert[1]` to be accessed on objects that may not be tuples or have a length of 2, potentially raising exceptions." + }, + { + "run_id": "d1b85c7a-8d30-4140-be90-9c658bcba907", + "model": "qwen", + "lang": "python", + "repo": "requests", + "sha": "a62a2d3", + "rep": 3, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "- The condition `isinstance(client_cert, tuple) or len(client_cert) == 2` is logically incorrect and will raise a `TypeError` if `client_cert` is not a tuple or does not support `len()`. This can happen if `client_cert` is a string or another non-iterable type." + }, + { + "run_id": "aae79429-3ce1-4256-b407-aae16cbbe011", + "model": "qwen", + "lang": "python", + "repo": "requests", + "sha": "0c030f7", + "rep": 1, + "gt": "Removed None-check guard for variable 'chardet'. (file: src/requests/packages.py)", + "fix": "Add `if chardet is not None:` guard before use.", + "detected": true, + "output": "- The variable `target` is used but undefined after the removal of the conditional block.\n- The variable `chardet` is referenced in the removed condition but not defined in the remaining code, potentially causing an `UnboundLocalError` if it was not defined earlier.\n- The unconditional execution of the loop assumes `target` is always defined, which may not be true if `chardet` is `None`.\n- The code attempts to replace `target` with the string `\"chardet\"` in the module name, which could cause incorrect mapping if `target` is not dynamically set." + }, + { + "run_id": "b56f28e4-aeda-4ce1-b675-d59fe2ea442d", + "model": "qwen", + "lang": "python", + "repo": "requests", + "sha": "0c030f7", + "rep": 2, + "gt": "Removed None-check guard for variable 'chardet'. (file: src/requests/packages.py)", + "fix": "Add `if chardet is not None:` guard before use.", + "detected": true, + "output": "- The variable `target` is referenced but not defined within the scope of the new code.\n- The variable `chardet` is checked in the removed code but is referenced indirectly via `target` in the new code without a null check, potentially causing an UnboundLocalError or NameError if `chardet` was originally None.\n- The variable `chardet` is used in the string replacement logic, but it is not defined in the new code, causing potential NameError or incorrect string replacement." + }, + { + "run_id": "04e141ba-d32f-4021-ad22-32d44d368207", + "model": "qwen", + "lang": "python", + "repo": "requests", + "sha": "0c030f7", + "rep": 3, + "gt": "Removed None-check guard for variable 'chardet'. (file: src/requests/packages.py)", + "fix": "Add `if chardet is not None:` guard before use.", + "detected": true, + "output": "- The variable `target` is used without being defined.\n- The variable `chardet` is used without being defined.\n- The logic assumes `target` is defined, which can cause a NameError." + }, + { + "run_id": "a01d49c7-a15f-492a-af64-218316df945c", + "model": "qwen", + "lang": "python", + "repo": "requests", + "sha": "145b539", + "rep": 1, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "- The logic change from `and` to `or` in the condition causes `pool_kwargs[\"ssl_context\"]` to be set incorrectly when `verify` is False and `has_poolmanager_ssl_context` is False, potentially overwriting the intended `cert_reqs` setting.\n- The condition now also sets `pool_kwargs[\"ssl_context\"]` when `verify` is False and `has_poolmanager_ssl_context` is False, which contradicts the behavior intended for `verify` being False (where `cert_reqs` is set to \"CERT_NONE\").\n- The change may lead to unintended side effects when `verify` is False, as the `ssl_context` is being set regardless of the verification flag." + }, + { + "run_id": "df1d7bc4-2d92-46c7-ab2b-4a59789d828e", + "model": "qwen", + "lang": "python", + "repo": "requests", + "sha": "145b539", + "rep": 2, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "- The logical condition `verify is True or not has_poolmanager_ssl_context` incorrectly applies the `ssl_context` assignment when `verify` is not True but `has_poolmanager_ssl_context` is False, which may not be the intended behavior.\n- The use of `is True` for boolean comparison is generally discouraged in Python; `if verify` is more idiomatic." + }, + { + "run_id": "5573b2cb-1ee2-40ff-a2da-579c1bdb01a5", + "model": "qwen", + "lang": "python", + "repo": "requests", + "sha": "145b539", + "rep": 3, + "gt": "Replaced boolean `and` with `or`. (file: src/requests/adapters.py)", + "fix": "Restore the original `and` operator for correct short-circuit logic.", + "detected": true, + "output": "- The logic change from `and` to `or` incorrectly expands the condition for applying the preloaded SSL context, potentially applying it when `verify` is False or in unintended scenarios.\n- The modified condition may bypass the intended verification checks when `verify` is not explicitly True.\n- The change could lead to unexpected behavior when `has_poolmanager_ssl_context` is False, regardless of the `verify` setting." + }, + { + "run_id": "510180b2-fbed-4fcf-a49b-773948fef10b", + "model": "qwen", + "lang": "python", + "repo": "requests", + "sha": "f8bec2f", + "rep": 1, + "gt": "Removed None-check guard for variable 'client_cert'. (file: src/requests/adapters.py)", + "fix": "Add `if client_cert is not None:` guard before use.", + "detected": true, + "output": "- The code removes the check for `client_cert is not None`, which could lead to an error when `client_cert` is `None`.\n- The code removes the check for whether `client_cert` is a tuple with length 2, assuming it always is, which could cause errors or unexpected behavior if it is not.\n- In the `else` block, `client_cert` is assigned directly to `pool_kwargs[\"cert_file\"]` without checking if it is `None` or a valid path." + }, + { + "run_id": "4c71b118-0259-4c97-9727-2773d1df106c", + "model": "qwen", + "lang": "python", + "repo": "requests", + "sha": "f8bec2f", + "rep": 2, + "gt": "Removed None-check guard for variable 'client_cert'. (file: src/requests/adapters.py)", + "fix": "Add `if client_cert is not None:` guard before use.", + "detected": true, + "output": "- Unconditional access to `client_cert[0]` and `client_cert[1]` will raise an `IndexError` or `TypeError` if `client_cert` is not a sequence of length 2.\n- The `else` block incorrectly assigns `client_cert` (which is `None` due to the `if` condition) to `pool_kwargs[\"cert_file\"]`." + }, + { + "run_id": "8c550cdc-e3bb-447a-9f6d-ec581cee798b", + "model": "qwen", + "lang": "python", + "repo": "requests", + "sha": "f8bec2f", + "rep": 3, + "gt": "Removed None-check guard for variable 'client_cert'. (file: src/requests/adapters.py)", + "fix": "Add `if client_cert is not None:` guard before use.", + "detected": true, + "output": "- Unconditional indexing of `client_cert` will raise an `IndexError` or `TypeError` if it is `None` or not a tuple/list of sufficient length.\n- The logic no longer validates that `client_cert` is a tuple of length 2 before attempting to access elements.\n- The logic no longer checks if `client_cert` is `None`, potentially assigning `None` to `cert_file` or attempting to access `None[0]`.\n- The else block will attempt to set `cert_file` to the value of `client_cert`, which is `None` if the initial `if` condition failed, or an incorrect value if `client_cert` is a tuple of length 1." + } +] \ No newline at end of file diff --git a/backend/l3_scores.json b/backend/l3_scores.json new file mode 100644 index 0000000..bd8b728 --- /dev/null +++ b/backend/l3_scores.json @@ -0,0 +1,434 @@ +{ + "60a6e958-99e5-4d4f-ad82-88548fe4b039": [ + 3, + "准确指出空指针隐患，修复方向隐含（补空检查），未明示" + ], + "8c00b35b-395e-4e42-8118-4589711f6b27": [ + 3, + "指出 null 直接传入风险，修复隐含，表述略笼统" + ], + "25b63c53-9d23-41eb-bef1-82f9dbe2e39d": [ + 3, + "指出 NPE 风险及与原回退逻辑的差异，修复隐含" + ], + "7dd615b3-0f3e-4491-a972-a82ecf74b37c": [ + 4, + "明确指出原始 < 写法才是正确防护，修复方向明确" + ], + "79b3908a-9eb8-48e7-8c42-7784d4cd97d1": [ + 3, + "指出 <= 变更不当、隐含恢复 <，但越界断言不准确" + ], + "241623fc-69fa-4730-bc7b-fdfe75c5c4a6": [ + 3, + "指向明确、隐含恢复 <，但越界理由不准确" + ], + "610e852b-8a21-42eb-b708-7c71a78528a0": [ + 3, + "精确描述 equals 语义被破坏的后果，未点明运算符变更" + ], + "3773ef9e-26df-47f1-ab54-c9e1b6f16e2d": [ + 4, + "明确点出 || 相对 && 的优先级问题及后果，修复方向明确" + ], + "bc64e4de-fe6b-4863-bbc5-c330629e9c39": [ + 3, + "描述 equals 合约被破坏，未点明 &&→|| 变更" + ], + "3a303e33-2510-4316-98f7-2bd318dca4eb": [ + 4, + "明确指出 || 替换 && 及两个具体后果，修复方向明确" + ], + "04468fe8-047e-4cad-aad5-ebcca6b055f9": [ + 3, + "描述 contains 行为异常，未点明运算符变更" + ], + "b9099ae9-230a-4e29-9909-937f482e320a": [ + 4, + "明确点出 || 短路导致误判，修复方向明确" + ], + "fb2a55ae-ef4e-41be-b8e0-7ea3032679da": [ + 4, + "明确指出 &&→|| 变更及后果，修复方向明确" + ], + "56196232-3e5b-4f38-8e74-e60ca6b33e8a": [ + 4, + "明确指出运算符变更及误用场景，修复方向明确" + ], + "f101adbf-3e6a-49db-8bf6-e72f6cbf9899": [ + 4, + "明确指出 &&→|| 及条件被弱化的后果，修复方向明确" + ], + "3bd471c9-466e-4692-a2ef-b9be2e3ead56": [ + 4, + "精确引用前后代码、指出逻辑反转与 TypeError，修复方向明确" + ], + "9bc56114-66e9-46b3-ac1c-37e7f6848295": [ + 4, + "精确引用 &&→|| 变更及空值风险，修复方向明确" + ], + "3666560f-b79f-45cd-a2e1-99f226d32b2d": [ + 4, + "精确引用变更、指出逻辑反转与异常后果，修复方向明确" + ], + "13a2d2d7-9483-4381-a7fa-7e2851a7af6d": [ + 4, + "明确指出 &&→|| 及误合并后果，修复方向明确" + ], + "b80f2457-51ef-45fe-aa3f-c7d1985e0468": [ + 4, + "明确指出变更及两个具体后果，修复方向明确" + ], + "d1838879-aae6-46e1-a483-56b2f268f796": [ + 4, + "明确指出变更及非对象误合并后果，修复方向明确" + ], + "27ebde97-9a09-4811-a8d1-b104329c5949": [ + 4, + "明确指出变更并分析分支走向，修复方向明确" + ], + "efe6536c-f53c-48e9-bbbe-00f056135368": [ + 4, + "明确指出变更及目标被污染风险，修复方向明确" + ], + "e66821dd-175b-4f58-bef4-f2a903679094": [ + 4, + "明确指出变更及分支逻辑变化，修复方向明确" + ], + "f63dee02-7b3f-44e1-9850-b58bd847fe11": [ + 4, + "明确指出 and→or 及三类输入的具体后果，修复方向明确" + ], + "9592df3d-b43d-4b4a-a435-d2807243c7fa": [ + 4, + "明确指出 chardet 空值守卫被移除及后果，修复方向明确" + ], + "7991529a-53e2-4a93-9278-46399e9ad33e": [ + 2, + "仅提及 chardet 为 None 时 NameError 的表象，未识别守卫移除，修复不明确" + ], + "6cf5d834-90f2-4cfa-914f-1e64c3523ef9": [ + 4, + "明确指出守卫被移除及 target 定义位置变化，修复方向明确" + ], + "d9af45af-d57c-4867-8153-9dcc94583b90": [ + 4, + "明确指出 and→or 及 SSL 上下文被覆盖后果，修复方向明确" + ], + "778b3ea1-26c0-482d-bc1b-1e116f1147b2": [ + 4, + "明确指出变更并详析三个分支影响，修复方向明确" + ], + "5781e3d2-b79f-4ef8-b291-ff69d554c06f": [ + 4, + "明确指出变更及 verify=False 时的矛盾行为，修复方向明确" + ], + "33552154-205c-4545-922c-dacc12f4e91c": [ + 4, + "明确指出空值检查被移除、else 不可达及缩进问题，修复方向明确" + ], + "0e8a0198-721f-4e4f-8a63-0bfa9762bfb3": [ + 4, + "明确指出检查被移除及索引异常后果，修复方向明确" + ], + "94b18699-5c10-4f3f-9b5c-33d70b4eb35f": [ + 4, + "明确指出守卫移除及三项具体后果，修复方向明确" + ], + "8b0d0aaa-081f-4589-8c5b-cc5e297bd839": [ + 3, + "准确指出 null 传入风险，未点明守卫被移除，修复隐含" + ], + "6aa57faa-2326-49a5-82fd-6a111468c328": [ + 4, + "明确提及原有 null 检查及其规避作用，修复方向明确" + ], + "099b5e77-ae22-42ea-8546-505ec5c8e0aa": [ + 3, + "指出 null 未经检查直接传入，修复隐含" + ], + "dd40cf99-2c0a-4275-8c90-a9dfe9b731c0": [ + 3, + "指出 <= 变更及零长写入问题，但定性为冗余偏弱，修复方向隐含" + ], + "0b9735ad-ca6b-4601-a1a6-f2d32b64ddd3": [ + 3, + "指出零长写入副作用，未明确恢复 < 的方向" + ], + "1e923a86-07b1-4531-9ceb-d05509079edf": [ + 3, + "指出 <= 导致的零长写入风险，修复方向隐含" + ], + "e05c2805-828d-4e39-8ee1-01b32f154570": [ + 3, + "准确描述 equals 被破坏及 ClassCast 风险，未点明运算符变更" + ], + "d36b6d9d-e9d6-47fe-bae9-f4ff80e76c8b": [ + 3, + "描述 equals 行为异常与强转风险，未点明 &&→||" + ], + "fb551ed2-5bdb-4708-bdce-bf6a64eeaa36": [ + 4, + "明确指出 &&→|| 变更及两个后果，修复方向明确" + ], + "4050c09d-b27e-4ffa-a938-f49057627bd6": [ + 3, + "详述 contains 行为异常与合约违反，未点明运算符" + ], + "8760a46d-1bba-4a76-b8d2-44338b90bc46": [ + 3, + "指出条件不再要求类型与查找同时成立，语义准确但未点明 ||" + ], + "e5a35dbf-b76d-4402-aab8-c7ca17cfe6f5": [ + 4, + "明确点出 || 短路失效导致强转异常，修复方向明确" + ], + "f63431e4-6f3d-4c5a-9aed-c8fca65264d0": [ + 3, + "准确描述「任一满足即真」的语义变化，未点明运算符" + ], + "b38dc15b-94f0-4d5a-b984-839cf485659c": [ + 3, + "指出不再要求两者同时支持，未点明 &&→||" + ], + "b9f4982d-f3bb-4882-b903-b92b241da947": [ + 4, + "明确指出 &&→|| 变更及运行环境风险，修复方向明确" + ], + "e5bc12ac-ac39-43fe-a1af-0d9e12c3ca2e": [ + 4, + "明确点出 || 短路及头合并被绕过，修复方向明确" + ], + "e69d92c0-bdcf-4f0b-9659-38fd7117cd85": [ + 3, + "详述三种行为后果（含 TypeError 与引用别名），未点明运算符变更" + ], + "441de8cd-1697-4155-90fc-1c92059980f1": [ + 4, + "引用 || 表达式并指出合并缺失与异常，修复方向明确" + ], + "b9ce7649-abdd-403e-977b-308470bf1d16": [ + 3, + "指出仅单方满足即触发合并，未点明 &&→||" + ], + "e587a125-3029-4e71-be4f-9c450b2d9300": [ + 3, + "详析分支不可达与类型错配，未点明运算符" + ], + "abb8b86a-bc7f-44cc-b7ae-4748736479e7": [ + 3, + "指出单方满足即触发及分支失效，未点明运算符" + ], + "0824d180-dba6-46e5-9965-5ad0afc5fcc2": [ + 4, + "明确指出 &&→|| 及三种后果，修复方向明确" + ], + "6a9057fb-8f63-4b50-9734-abbd45c6d559": [ + 4, + "明确点出新 || 条件及语义破坏，修复方向明确" + ], + "440f5d47-9cb8-4045-a598-dff736a8ddb7": [ + 4, + "明确指出 &&→|| 及分支不可达，修复方向明确" + ], + "58075bc7-dffa-4179-a710-f9a464a9477a": [ + 4, + "明确指出 or 替换 and 及解包风险，修复方向明确" + ], + "4d1cdddc-6d50-4cef-9070-f6e190ac8f95": [ + 4, + "引用新 or 条件并指出元组长度校验丢失，修复方向明确" + ], + "c623d3cc-77fc-478c-9825-40e47f350111": [ + 3, + "描述 length-2 字符串误配等后果，未点明 and→or" + ], + "b019bc4e-2588-4126-8287-05fc9a8cccc0": [ + 4, + "点出 or 导致条件放宽及 IndexError 机制，修复方向隐含明确" + ], + "61113778-bd49-4f4d-8539-3947b6bb9b08": [ + 4, + "明确写出 and 改为 or 及三类错误输入后果" + ], + "5a764dd0-66ad-4b2c-9d44-2640d94959ca": [ + 4, + "明确指出 if chardet is not None 守卫被移除及 NameError 后果" + ], + "9246543f-13d8-44de-a056-4f96d4cb5dcf": [ + 4, + "指出守卫移除导致循环无条件执行，并说明映射逻辑丢失" + ], + "8435fe89-1c9d-4752-a387-a1cb7c57da15": [ + 4, + "明确指出 chardet is not None 守卫被移除及失败场景" + ], + "7586c40e-cb83-4a86-983f-f59c5e6668c6": [ + 4, + "引用新 or 条件并精确说明两类误入分支的后果" + ], + "aeba23c7-8be9-45cf-a578-00fd40bc6da7": [ + 3, + "准确描述条件触发范围扩大的后果，未点明 and/or 运算符" + ], + "d61481bb-e7cf-40fc-a5bb-778e528d1e8a": [ + 4, + "明确写出 and 改 or 破坏守卫及正确语义" + ], + "44bc02fc-14cf-4557-999e-304a911d8478": [ + 4, + "明确指出 None 守卫被移除及无条件索引后果" + ], + "35357d01-5e38-4edd-aef2-d77bd60569c2": [ + 4, + "指出 client_cert 不再检查 None 及字符串误索引问题" + ], + "28d66204-a406-4739-8622-b5ffdae05c7b": [ + 4, + "明确指出 None 守卫移除及元组校验缺失后果" + ], + "5c699718-681e-4eda-b504-8fa6b6e94530": [ + 2, + "仅一句笼统的 NPE 提示，未说明成因也无修复指向" + ], + "77c54e11-d2a7-4dbf-b5f2-164f249b3550": [ + 4, + "明确指出守卫移除及 fallback 失效机制" + ], + "3ac7d755-afd7-4aa7-a705-6f6cdbb4ad60": [ + 3, + "指出 NPE 与不可达代码后果，但未点明守卫移除机制" + ], + "7ee913bd-7d41-46ee-a154-68d5395052cc": [ + 3, + "指出边界条件越界（off-by-one）后果，修复仅隐含" + ], + "a4d4a3a6-0f15-4f6f-8ef3-0691cf787974": [ + 3, + "指出 <= 越界后果，修复方向隐含" + ], + "5b33523e-6441-4ce0-b206-5916650b1206": [ + 3, + "描述数组越界后果准确，未给出明确修复" + ], + "e439022d-791a-4079-8657-5f7268f05ec9": [ + 4, + "明确指出 || 替换 && 的运算符变化" + ], + "53bd06a1-8d20-4222-a89e-12e4e0ceb8aa": [ + 4, + "明确指出 && 变 || 及短路语义破坏" + ], + "e528d730-288e-434b-bd44-49801536c816": [ + 4, + "明确点出逻辑运算符替换及后果" + ], + "c8cb246f-a361-455d-b9e6-b491ba729bb3": [ + 4, + "明确指出 && 改 || 及错误语义" + ], + "8e5bde72-88c5-4380-b69b-f337858ef808": [ + 4, + "明确指出 &&→|| 变化及潜在 ClassCastException" + ], + "00d028e4-8adb-4ca5-a8d7-a4ede36967d6": [ + 4, + "明确指出 && 改 || 及类型转换异常后果" + ], + "1d33128b-1898-4c8c-af61-2b62de022659": [ + 4, + "明确指出 && 改 || 导致特性检测错误" + ], + "b284228f-9ede-433a-a9db-df43627a2328": [ + 4, + "明确说明 && 变 || 及正确依赖关系" + ], + "478aabc9-8811-4374-ad7d-22082a763a1a": [ + 4, + "明确指出 || 误用代替 &&" + ], + "3314cf51-90c3-4ed0-93e0-b0ae95ef5286": [ + 3, + "准确描述短路返回值变化及运行时错误后果，未点明运算符" + ], + "d55bd2c3-0394-494c-b4fe-18a2191384ca": [ + 3, + "准确描述 headers 为假时的错误访问后果（裁判未判出但内容命中）" + ], + "71cb0124-fc11-4b9e-b99b-b632b3bdfc79": [ + 3, + "准确描述绕过 merge 与 undefined 访问后果，未点明运算符" + ], + "904cd26b-f68f-4792-95be-75928e5df864": [ + 4, + "明确指出 && 改 || 及单方满足即合并的机制" + ], + "5d03a902-90df-4bc4-942d-4b3e7daa07af": [ + 4, + "明确指出 &&→|| 及错误合并行为" + ], + "920a0a2e-adbf-4fec-b9a1-843eb3c88448": [ + 4, + "明确指出运算符变化并分析分支可达性影响" + ], + "b6ed21e1-581e-474d-9c84-13b3eea57c95": [ + 4, + "明确指出 && 改 || 及冗余分支后果" + ], + "0635f620-0e27-4803-8411-580610ebdc10": [ + 4, + "明确指出 OR 误用代替 AND" + ], + "bc8324f6-65ba-41e2-a2e9-85f22fd35b31": [ + 4, + "明确指出 &&→|| 并列出冗余/不可达代码等四层后果" + ], + "ea252238-71e2-41ae-896f-02df20cc20d5": [ + 4, + "引用含 or 的错误条件表达式并说明 TypeError 机制" + ], + "35f3b71f-2151-4dd5-a0c8-0529f1ebe0ea": [ + 4, + "明确指出逻辑 OR 代替 AND 及错误索引后果" + ], + "d1b85c7a-8d30-4140-be90-9c658bcba907": [ + 4, + "引用含 or 的条件并说明 TypeError 触发路径" + ], + "aae79429-3ce1-4256-b407-aae16cbbe011": [ + 3, + "描述条件块移除后 target 未定义及 chardet 为 None 的后果，未点明守卫机制" + ], + "b56f28e4-aeda-4ce1-b675-d59fe2ea442d": [ + 4, + "明确指出缺少 null check 及 UnboundLocalError 成因" + ], + "04e141ba-d32f-4021-ad22-32d44d368207": [ + 3, + "指出变量未定义导致 NameError 后果，较简略未点明守卫" + ], + "a01d49c7-a15f-492a-af64-218316df945c": [ + 4, + "明确写出 and 改 or 及 ssl_context 误设置机制" + ], + "df1d7bc4-2d92-46c7-ab2b-4a59789d828e": [ + 4, + "引用含 or 的新条件并说明误触发场景" + ], + "5573b2cb-1ee2-40ff-a2da-579c1bdb01a5": [ + 4, + "明确指出 and 改 or 扩大条件适用范围" + ], + "510180b2-fbed-4fcf-a49b-773948fef10b": [ + 4, + "明确指出移除了 client_cert is not None 检查" + ], + "4c71b118-0259-4c97-9727-2773d1df106c": [ + 3, + "准确描述无条件索引与 else 赋 None 后果，未点明守卫移除" + ], + "8c550cdc-e3bb-447a-9f6d-ec581cee798b": [ + 4, + "明确指出不再检查 client_cert 是否为 None 及校验缺失" + ] +} \ No newline at end of file diff --git a/backend/make_scoring_sheet.py b/backend/make_scoring_sheet.py new file mode 100644 index 0000000..ee6179a --- /dev/null +++ b/backend/make_scoring_sheet.py @@ -0,0 +1,261 @@ +"""Generate the Likert-5 scoring sheet for L3 fix-suggestion actionability. + +Reads the 108 L3 runs (3 models x 12 samples x 3 repeats) from the +experiment database and produces a workbook with: + Sheet 1 封面: title, key facts, sheet index, notes + Sheet 2 评分说明: the 1-5 rubric and scoring guidelines + Sheet 3 打分表: one row per L3 output, dropdown-validated score column +""" + +import uuid +from pathlib import Path + +from openpyxl import Workbook +from openpyxl.styles import Alignment, Border, Font, PatternFill, Side +from openpyxl.utils import get_column_letter +from openpyxl.worksheet.datavalidation import DataValidation + +from app.db import SessionLocal, init_db +from app.models import ExperimentRun + +EXPERIMENT_ID = "42a452df-6091-47c3-8632-a9ec29815aaa" +OUTPUT = Path(r"C:\Users\eeymoo\Documents\毕业论文\实验结果\L3修复建议_李克特评分表.xlsx") + +GREY_FILL = PatternFill(start_color="F5F5F5", end_color="F5F5F5", fill_type="solid") +HEADER_FILL = PatternFill(start_color="333333", end_color="333333", fill_type="solid") +BLUE_FILL = PatternFill(start_color="E6F0FA", end_color="E6F0FA", fill_type="solid") +THIN = Side(style="thin", color="D0D0D0") +BORDER = Border(left=THIN, right=THIN, top=THIN, bottom=THIN) + + +def collect_l3_runs(): + init_db() + db = SessionLocal() + runs = ( + db.query(ExperimentRun) + .filter(ExperimentRun.experiment_id == uuid.UUID(EXPERIMENT_ID), ExperimentRun.status == "done") + .all() + ) + l3 = [r for r in runs if r.template_version.template.level == "L3"] + l3.sort(key=lambda r: (r.model_id, r.sample.language, str(r.sample_id), r.repeat_index)) + return l3 + + +def style_header(ws, row, cols): + for col in cols: + c = ws.cell(row=row, column=col) + c.font = Font(bold=True, color="FFFFFF", size=11) + c.fill = HEADER_FILL + c.alignment = Alignment(horizontal="center", vertical="center") + + +def build_cover(wb, n_rows): + ws = wb.active + ws.title = "封面" + ws.sheet_view.showGridLines = False + ws.column_dimensions["A"].width = 3 + for col, w in zip("BCDEF", (22, 26, 26, 26, 26)): + ws.column_dimensions[col].width = w + + ws.merge_cells("B2:F2") + ws["B2"] = "L3 修复建议可操作性评分表（李克特五级量表）" + ws["B2"].font = Font(size=18, bold=True) + ws["B2"].alignment = Alignment(horizontal="center", vertical="center") + ws.row_dimensions[2].height = 38 + + ws.merge_cells("B3:F3") + ws["B3"] = "基于大模型的智能代码审查提示词优化研究 · 实验 full-factorial-v1" + ws["B3"].font = Font(size=11, color="666666") + ws["B3"].alignment = Alignment(horizontal="center") + + ws["B5"] = "关键信息" + ws["B5"].font = Font(bold=True, size=12) + facts = [ + ("待评分输出", f"{n_rows} 条（3 模型 × 12 样本 × 3 次重复的 L3 级输出）"), + ("评分维度", "建议可操作性（李克特 1–5 级）"), + ("评分对象", "模型给出的修复建议，对照 Ground Truth 与参考修复"), + ("数据来源", "实验数据库 promptcr.db（experiment: full-factorial-v1）"), + ] + r = 6 + for k, v in facts: + ws.cell(row=r, column=2, value=k).font = Font(bold=True) + ws.cell(row=r, column=2).fill = BLUE_FILL + ws.cell(row=r, column=2).border = BORDER + ws.merge_cells(start_row=r, start_column=3, end_row=r, end_column=6) + ws.cell(row=r, column=3, value=v).border = BORDER + for col in range(3, 7): + ws.cell(row=r, column=col).border = BORDER + r += 1 + + ws["B11"] = "工作表索引" + ws["B11"].font = Font(bold=True, size=12) + index = [ + ("评分说明", "李克特 1–5 级判据与评分注意事项（打钩前必读）"), + ("打分表", f"{n_rows} 条 L3 输出逐条打分，评分列为下拉选择 1–5"), + ] + r = 12 + for name, desc in index: + ws.cell(row=r, column=2, value=name).font = Font(bold=True, color="0066CC") + ws.cell(row=r, column=2).border = BORDER + ws.merge_cells(start_row=r, start_column=3, end_row=r, end_column=6) + ws.cell(row=r, column=3, value=desc) + for col in range(3, 7): + ws.cell(row=r, column=col).border = BORDER + r += 1 + + ws["B15"] = "备注" + ws["B15"].font = Font(bold=True, size=12) + notes = [ + "1. 评分前请先阅读「评分说明」工作表，保持前后判据一致。", + "2. run_id 与实验数据库中的实验单元一一对应，便于回溯原始输出与指标。", + "3. 评分完成后，将按模型 × 级别做描述统计与模糊综合评判（参考亓莱滨方法）。", + "4. 如中途打断，可分多次填写，评分列留空视为未评。", + ] + r = 16 + for note in notes: + ws.merge_cells(start_row=r, start_column=2, end_row=r, end_column=6) + ws.cell(row=r, column=2, value=note).font = Font(size=10, color="666666") + r += 1 + + +def build_rubric(wb): + ws = wb.create_sheet("评分说明") + ws.sheet_view.showGridLines = False + ws.column_dimensions["A"].width = 3 + ws.column_dimensions["B"].width = 8 + ws.column_dimensions["C"].width = 18 + ws.column_dimensions["D"].width = 80 + + ws.merge_cells("B2:D2") + ws["B2"] = "建议可操作性 · 李克特五级量表判据" + ws["B2"].font = Font(size=15, bold=True) + ws["B2"].alignment = Alignment(horizontal="center", vertical="center") + ws.row_dimensions[2].height = 30 + + header_row = 4 + ws.cell(row=header_row, column=2, value="分值") + ws.cell(row=header_row, column=3, value="等级") + ws.cell(row=header_row, column=4, value="判据") + style_header(ws, header_row, (2, 3, 4)) + + rubric = [ + (5, "完全可操作", "建议可直接应用：能针对预埋缺陷给出正确的修复方式与具体代码/步骤，无需修改即可采纳。"), + (4, "基本可操作", "建议方向正确、内容具体，仅需少量人工调整（如修正细节、补全边界）即可应用。"), + (3, "部分可操作", "建议方向基本正确，但存在明显缺漏或不够具体，需较多人工补充、验证后才能实施。"), + (2, "可操作性差", "建议笼统模糊（如仅说「建议检查逻辑」「注意空指针」），或与预埋缺陷仅部分相关，难以直接指导修复。"), + (1, "不可操作", "建议与预埋缺陷无关、明显错误、自相矛盾，或未给出任何修复建议。"), + ] + r = header_row + 1 + for score, label, desc in rubric: + ws.cell(row=r, column=2, value=score).alignment = Alignment(horizontal="center", vertical="center") + ws.cell(row=r, column=2).font = Font(bold=True, size=12, color="0066CC") + ws.cell(row=r, column=3, value=label).font = Font(bold=True) + ws.cell(row=r, column=3).alignment = Alignment(vertical="center") + cell = ws.cell(row=r, column=4, value=desc) + cell.alignment = Alignment(wrap_text=True, vertical="center") + for col in (2, 3, 4): + ws.cell(row=r, column=col).border = BORDER + if score % 2 == 1: + if ws.cell(row=r, column=col).fill.start_color.rgb in (None, "00000000"): + ws.cell(row=r, column=col).fill = GREY_FILL + ws.row_dimensions[r].height = 42 + r += 1 + + r += 1 + ws.cell(row=r, column=2, value="评分注意事项").font = Font(bold=True, size=12) + notes = [ + "1. 仅评价修复建议的可操作性，不评价缺陷检出是否正确（检出正确性由检出率指标衡量）。", + "2. 若一条输出包含多个问题的建议，以针对预埋缺陷（Ground Truth 列所示）的那条建议为评分对象。", + "3. 对照「参考修复」判断建议的正确性，但措辞不要求一致，语义等价即可。", + "4. 若模型未检出预埋缺陷（建议全部指向其他问题），评 1 分。", + "5. 评分时保持标准前后一致；建议先抽 5 条试评，校准后再正式评分。", + ] + r += 1 + for note in notes: + ws.merge_cells(start_row=r, start_column=2, end_row=r, end_column=4) + cell = ws.cell(row=r, column=2, value=note) + cell.font = Font(size=10, color="666666") + cell.alignment = Alignment(wrap_text=True, vertical="center") + ws.row_dimensions[r].height = 28 + r += 1 + + +def build_scoring(wb, runs): + ws = wb.create_sheet("打分表") + ws.sheet_view.showGridLines = False + + headers = [ + ("序号", 6), ("run_id", 34), ("模型", 10), ("语言", 12), ("仓库", 10), + ("commit", 10), ("重复", 6), ("Ground Truth 缺陷", 36), ("参考修复", 36), + ("模型 L3 输出（含修复建议）", 70), ("评分(1-5)", 10), ("备注", 14), + ] + header_row = 2 + for i, (title, width) in enumerate(headers, start=2): + col = get_column_letter(i) + ws.column_dimensions[col].width = width + ws.cell(row=header_row, column=i, value=title) + ws.column_dimensions["A"].width = 2 + style_header(ws, header_row, range(2, 2 + len(headers))) + ws.row_dimensions[header_row].height = 24 + ws.freeze_panes = "B3" + + dv = DataValidation(type="list", formula1='"1,2,3,4,5"', allow_blank=True, showDropDown=False) + dv.error = "请选择 1-5 的整数分值" + dv.errorTitle = "无效评分" + ws.add_data_validation(dv) + + r = header_row + 1 + for idx, run in enumerate(runs, start=1): + gt = run.sample.defects[0] if run.sample.defects else None + values = [ + idx, + str(run.id), + run.model_id, + run.sample.language, + run.sample.repo, + run.sample.commit_sha[:7], + run.repeat_index, + (gt.description or "") if gt else "", + (gt.reference_fix or "") if gt else "", + run.result.raw_output or "", + None, + None, + ] + for i, v in enumerate(values, start=2): + cell = ws.cell(row=r, column=i, value=v) + cell.border = BORDER + if i in (8, 9, 10, 11): # GT / 参考修复 / 模型输出 → wrap + cell.alignment = Alignment(wrap_text=True, vertical="top") + else: + cell.alignment = Alignment(horizontal="center", vertical="top") + ws.cell(row=r, column=12).fill = BLUE_FILL # 评分列高亮 + dv.add(ws.cell(row=r, column=12)) + ws.row_dimensions[r].height = 110 + r += 1 + + ws.auto_filter.ref = f"B{header_row}:M{r - 1}" + + +def main(): + runs = collect_l3_runs() + print(f"L3 runs: {len(runs)}") + wb = Workbook() + build_cover(wb, len(runs)) + build_rubric(wb) + build_scoring(wb, runs) + OUTPUT.parent.mkdir(parents=True, exist_ok=True) + wb.save(OUTPUT) + print("saved:", OUTPUT) + + # 校验：重开文件确认结构 + from openpyxl import load_workbook + wb2 = load_workbook(OUTPUT) + assert wb2.sheetnames == ["封面", "评分说明", "打分表"], wb2.sheetnames + ws = wb2["打分表"] + assert ws.max_row == len(runs) + 2, (ws.max_row, len(runs)) + assert ws["L2"].value == "评分(1-5)" + print("verify ok: sheets =", wb2.sheetnames, "| data rows =", ws.max_row - 2) + + +if __name__ == "__main__": + main() diff --git a/backend/pytest.ini b/backend/pytest.ini new file mode 100644 index 0000000..e318683 --- /dev/null +++ b/backend/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +pythonpath = . +testpaths = tests +addopts = --cov=app --cov-report=term-missing --cov-report=html --cov-fail-under=50 +asyncio_mode = auto diff --git a/backend/run_by_model.py b/backend/run_by_model.py new file mode 100644 index 0000000..d30f58a --- /dev/null +++ b/backend/run_by_model.py @@ -0,0 +1,68 @@ +"""Resume experiment runs filtered by model with custom concurrency. + +Usage: python run_by_model.py +Reuses ExperimentRunner internals; safe to re-run (stale running runs are +reset to pending by the caller beforehand via runner.run_experiment normally — +here we handle pending/failed only, plus stale running without result). +""" + +import asyncio +import sys +import uuid + +from app.db import SessionLocal, init_db +from app.experiments.runner import ExperimentRunner +from app.models import ExperimentRun + + +async def main() -> None: + experiment_id = uuid.UUID(sys.argv[1]) + model_id = sys.argv[2] + concurrency = int(sys.argv[3]) if len(sys.argv) > 3 else 4 + + init_db() + db = SessionLocal() + runner = ExperimentRunner(db) + + # Reclaim runs orphaned in "running" by previously killed processes. + stale = ( + db.query(ExperimentRun) + .filter_by(status="running", model_id=model_id) + .filter(~ExperimentRun.result.has()) + .all() + ) + for run in stale: + run.status = "pending" + db.commit() + + runs = ( + db.query(ExperimentRun) + .filter( + ExperimentRun.experiment_id == experiment_id, + ExperimentRun.model_id == model_id, + ExperimentRun.status.in_(["pending", "failed"]), + ) + .all() + ) + print(f"{model_id}: {len(runs)} runs to execute, concurrency={concurrency}") + + semaphore = asyncio.Semaphore(concurrency) + + async def execute(run): + async with semaphore: + return await runner._execute_run(run) + + tasks = [asyncio.create_task(execute(run)) for run in runs] + done = 0 + for coro in asyncio.as_completed(tasks): + result = await coro + done += 1 + if result["status"] != "done": + print("WARN:", result) + if done % 10 == 0: + print(f"progress: {done}/{len(tasks)}") + print(f"{model_id}: completed {done}/{len(tasks)}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py new file mode 100644 index 0000000..b308226 --- /dev/null +++ b/backend/tests/conftest.py @@ -0,0 +1,19 @@ +"""Shared pytest fixtures.""" + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from app.db import Base + + +@pytest.fixture +def db(): + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(bind=engine) + Session = sessionmaker(bind=engine) + session = Session() + try: + yield session + finally: + session.close() diff --git a/backend/tests/test_analysis.py b/backend/tests/test_analysis.py new file mode 100644 index 0000000..c32db14 --- /dev/null +++ b/backend/tests/test_analysis.py @@ -0,0 +1,28 @@ +"""Tests for analysis parser and metrics.""" + +from app.analysis.parser import compare_findings, parse_output + + +def test_parse_l1_output(): + output = "- null pointer\n- off by one" + findings = parse_output(output, "L1") + assert len(findings) == 2 + assert findings[0].defect_type == "null pointer" + + +def test_parse_l2_output(): + output = """- Type: null_pointer + Lines: 5-6 + Explanation: missing guard""" + findings = parse_output(output, "L2") + assert len(findings) == 1 + assert findings[0].line_start == 5 + assert findings[0].line_end == 6 + + +def test_compare_findings(): + findings = parse_output("- null_pointer", "L1") + gt = [{"id": "1", "defect_type": "null_pointer", "line_start": 5, "line_end": 5}] + metrics = compare_findings(findings, gt) + assert metrics["detection_rate"] == 1.0 + assert metrics["coverage_rate"] == 1.0 diff --git a/backend/tests/test_analysis_service.py b/backend/tests/test_analysis_service.py new file mode 100644 index 0000000..7cbf5d3 --- /dev/null +++ b/backend/tests/test_analysis_service.py @@ -0,0 +1,61 @@ +"""Tests for analysis service.""" + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from app.analysis.service import AnalysisService +from app.db import Base +from app.models import Defect, ExperimentRun, PromptTemplate, PromptTemplateVersion, Result, Sample + + +def get_test_db(): + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(bind=engine) + Session = sessionmaker(bind=engine) + return Session() + + +def test_compute_metrics_for_run(): + db = get_test_db() + sample = Sample(repo="r", commit_sha="abc", language="python", diff="diff") + defect = Defect( + sample=sample, + defect_type="null_pointer", + language="python", + line_start=5, + line_end=5, + ) + db.add(sample) + db.commit() + + from uuid import uuid4 + + template = PromptTemplate(strategy_id="code_review", level="L1") + version = PromptTemplateVersion( + template=template, version_number=1, body="", variables_schema={} + ) + db.add(template) + db.commit() + + run = ExperimentRun( + id=uuid4(), + experiment_id=uuid4(), + sample_id=sample.id, + model_id="deepseek", + template_version_id=version.id, + repeat_index=1, + status="done", + ) + db.add(run) + db.flush() + result = Result( + run_id=run.id, + raw_output="- null_pointer", + ) + db.add(result) + db.commit() + + service = AnalysisService(db) + metrics = service.compute_metrics_for_run(str(run.id)) + assert metrics["detection_rate"] == 1.0 diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py new file mode 100644 index 0000000..8220b2c --- /dev/null +++ b/backend/tests/test_api.py @@ -0,0 +1,42 @@ +"""Tests for FastAPI endpoints.""" + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from app.api.main import app +from app.db import Base, get_db + + +def get_test_db(): + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(bind=engine) + Session = sessionmaker(bind=engine) + db = Session() + try: + yield db + finally: + db.close() + + +app.dependency_overrides[get_db] = get_test_db +client = TestClient(app) + + +def test_health(): + response = client.get("/health") + assert response.status_code == 200 + assert response.json()["status"] == "ok" + + +def test_list_samples_empty(): + response = client.get("/api/datasets/samples") + assert response.status_code == 200 + assert response.json() == [] + + +def test_list_models(): + response = client.get("/api/models") + assert response.status_code == 200 + assert "deepseek" in response.json()["models"] diff --git a/backend/tests/test_charts.py b/backend/tests/test_charts.py new file mode 100644 index 0000000..5b15446 --- /dev/null +++ b/backend/tests/test_charts.py @@ -0,0 +1,21 @@ +"""Tests for chart generation.""" + +from app.analysis.charts import boxplot, grouped_bar, heatmap + + +def test_heatmap_returns_base64(): + data = {"m1": {"L1": 0.5, "L2": 0.8}} + b64 = heatmap(data, title="Test") + assert b64.startswith("iVBOR") or b64.startswith("/9j") + + +def test_boxplot_returns_base64(): + data = {"m1-L1": [0.1, 0.2, 0.3]} + b64 = boxplot(data, title="Test") + assert isinstance(b64, str) and len(b64) > 0 + + +def test_grouped_bar_returns_base64(): + data = {"m1": {"L1": 0.5}} + b64 = grouped_bar(data, title="Test") + assert isinstance(b64, str) and len(b64) > 0 diff --git a/backend/tests/test_experiments.py b/backend/tests/test_experiments.py new file mode 100644 index 0000000..000f2e1 --- /dev/null +++ b/backend/tests/test_experiments.py @@ -0,0 +1,46 @@ +"""Tests for experiment matrix and runner.""" + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from app.db import Base +from app.experiments.matrix import generate_full_factorial_matrix +from app.model_adapters.factory import create_adapter, list_models +from app.prompts.defaults import seed_default_templates + + +def get_test_db(): + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(bind=engine) + Session = sessionmaker(bind=engine) + return Session() + + +def test_list_models(): + assert "deepseek" in list_models() + + +def test_create_adapter_requires_key(): + with pytest.raises(ValueError): + create_adapter("unknown") + + +def test_generate_matrix(): + db = get_test_db() + seed_default_templates(db) + from app.models import Sample + + sample = Sample(repo="test", commit_sha="abc", language="python", diff="diff") + db.add(sample) + db.commit() + + experiment = generate_full_factorial_matrix( + db, + name="test-exp", + models=["deepseek"], + levels=["L1"], + sample_ids=[str(sample.id)], + repeats=2, + ) + assert len(experiment.runs) == 2 diff --git a/backend/tests/test_git_parser.py b/backend/tests/test_git_parser.py new file mode 100644 index 0000000..0319c95 --- /dev/null +++ b/backend/tests/test_git_parser.py @@ -0,0 +1,40 @@ +"""Tests for Git parser and candidate selection.""" + +import os +from pathlib import Path + +import pytest + +from app.dataset.git_parser import score_commit, select_candidates + + +def _make_repo(tmp_path: Path) -> str: + repo = tmp_path / "repo" + repo.mkdir() + os.system( + f'cd {repo} && git init -q && git config user.email "test@example.com" && git config user.name "Test"' + ) + (repo / "main.py").write_text("print('hello')\n") + os.system(f'cd {repo} && git add . && git commit -q -m "init"') + (repo / "main.py").write_text("print('world')\n") + os.system(f'cd {repo} && git add . && git commit -q -m "update main"') + (repo / "app.java").write_text("class App {}\n") + os.system(f'cd {repo} && git add . && git commit -q -m "add java app"') + return str(repo) + + +def test_score_commit_prefers_message_and_size(tmp_path): + repo = _make_repo(tmp_path) + from app.dataset.git_parser import list_commits + + commits = list_commits(repo) + assert len(commits) >= 2 + scores = [score_commit(c) for c in commits] + assert all(0 <= s <= 1 for s in scores) + + +def test_select_candidates(tmp_path): + repo = _make_repo(tmp_path) + candidates = select_candidates(repo, count=2, languages=["python", "java"]) + assert len(candidates) <= 2 + assert all(c.repo == "repo" for c in candidates) diff --git a/backend/tests/test_model_adapters.py b/backend/tests/test_model_adapters.py new file mode 100644 index 0000000..d9de99d --- /dev/null +++ b/backend/tests/test_model_adapters.py @@ -0,0 +1,26 @@ +"""Tests for model adapters.""" + +import pytest + +from app.model_adapters.base import ModelAdapter, ChatResponse +from app.model_adapters.providers import DeepSeekAdapter + + +def test_adapter_payload_and_extraction(): + adapter = DeepSeekAdapter(api_key="test", model="deepseek-chat", base_url="https://x") + payload = adapter._build_payload("hello", {"temperature": 0.5, "max_tokens": 100}) + assert payload["model"] == "deepseek-chat" + assert payload["messages"][0]["content"] == "hello" + + response_text = adapter._extract_text( + {"choices": [{"message": {"content": "hi"}}]} + ) + assert response_text == "hi" + + +def test_adapter_token_usage(): + adapter = DeepSeekAdapter(api_key="test", model="m", base_url="https://x") + usage = adapter._extract_token_usage( + {"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}} + ) + assert usage["total_tokens"] == 15 diff --git a/backend/tests/test_prompts.py b/backend/tests/test_prompts.py new file mode 100644 index 0000000..45f6b52 --- /dev/null +++ b/backend/tests/test_prompts.py @@ -0,0 +1,36 @@ +"""Tests for prompt template service.""" + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from app.db import Base, init_db +from app.prompts.defaults import seed_default_templates +from app.prompts.service import PromptService + + +def get_test_db(): + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(bind=engine) + Session = sessionmaker(bind=engine) + return Session() + + +def test_seed_and_render(): + db = get_test_db() + seed_default_templates(db) + service = PromptService(db) + rendered = service.render("code_review", None, {"language": "python", "diff": "+x"}) + assert "python" in rendered + assert "+x" in rendered + + +def test_create_version_increments(): + db = get_test_db() + service = PromptService(db) + v1 = service.create_version("test", "L1", "hello {{ name }}") + v2 = service.create_version("test", "L1", "hello {{ name }} v2") + assert v1.version_number == 1 + assert v2.version_number == 2 + versions = service.list_versions("test", "L1") + assert len(versions) == 2 diff --git a/backend/tests/test_rules.py b/backend/tests/test_rules.py new file mode 100644 index 0000000..9b535d8 --- /dev/null +++ b/backend/tests/test_rules.py @@ -0,0 +1,44 @@ +"""Tests for mutation rules.""" + +from app.dataset.rules.python.null_pointer import NoneReferenceRule +from app.dataset.rules.python.boundary_error import BoundaryErrorRule +from app.dataset.rules.python.logic_operator import LogicOperatorRule +from app.dataset.rules.registry import get_registry + + +def test_python_none_reference(): + src = """def process(data): + if data is not None: + x = data.upper() + return x +""" + m = NoneReferenceRule().detect_and_mutate(src) + assert m is not None + assert m.defect_type == "null_pointer" + assert "if data is not None" not in m.mutated_source + + +def test_python_boundary_error(): + src = """def get(items, i): + if i < len(items): + return items[i] +""" + m = BoundaryErrorRule().detect_and_mutate(src) + assert m is not None + assert "<=" in m.mutated_source + + +def test_python_logic_operator(): + src = """def ok(a, b): + return a and b +""" + m = LogicOperatorRule().detect_and_mutate(src) + assert m is not None + assert " or " in m.mutated_source + + +def test_registry_discovers_rules(): + registry = get_registry() + assert "python" in registry.all_rules() + assert "java" in registry.all_rules() + assert "javascript" in registry.all_rules() diff --git a/backend/tests/test_runner.py b/backend/tests/test_runner.py new file mode 100644 index 0000000..2c1cf2e --- /dev/null +++ b/backend/tests/test_runner.py @@ -0,0 +1,62 @@ +"""Tests for experiment runner.""" + +from unittest import mock +from uuid import uuid4 + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from app.db import Base +from app.experiments.runner import ExperimentRunner +from app.model_adapters.base import ChatResponse +from app.models import ExperimentRun, PromptTemplate, PromptTemplateVersion, Result, Sample + + +def get_test_db(): + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(bind=engine) + Session = sessionmaker(bind=engine) + return Session() + + +@pytest.mark.asyncio +async def test_runner_executes_pending_run(): + db = get_test_db() + template = PromptTemplate(strategy_id="code_review", level="L1") + version = PromptTemplateVersion( + template=template, version_number=1, body="", variables_schema={} + ) + db.add(template) + db.commit() + + sample = Sample(repo="r", commit_sha="abc", language="python", diff="d") + db.add(sample) + db.commit() + + run = ExperimentRun( + id=uuid4(), + experiment_id=uuid4(), + sample_id=sample.id, + model_id="deepseek", + template_version_id=version.id, + repeat_index=1, + status="pending", + sampling_params={}, + ) + db.add(run) + db.commit() + + runner = ExperimentRunner(db) + with mock.patch("app.experiments.runner.create_adapter") as mock_factory: + mock_adapter = mock.AsyncMock() + mock_adapter.chat.return_value = ChatResponse( + text="- null_pointer", token_usage={}, latency_ms=100.0 + ) + mock_factory.return_value = mock_adapter + summary = await runner.run_experiment() + + assert summary["total"] == 1 + db.refresh(run) + assert run.status == "done" + assert run.result is not None diff --git a/backend/tests/test_stability.py b/backend/tests/test_stability.py new file mode 100644 index 0000000..a26c7b7 --- /dev/null +++ b/backend/tests/test_stability.py @@ -0,0 +1,51 @@ +"""Tests for stability metric.""" + +from uuid import uuid4 + +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker + +from app.analysis.stability import compute_stability_score +from app.db import Base +from app.models import ExperimentRun, PromptTemplate, PromptTemplateVersion, Result, Sample + + +def get_test_db(): + engine = create_engine("sqlite:///:memory:") + Base.metadata.create_all(bind=engine) + Session = sessionmaker(bind=engine) + return Session() + + +def test_stability_perfect(): + db = get_test_db() + template = PromptTemplate(strategy_id="code_review", level="L1") + version = PromptTemplateVersion( + template=template, version_number=1, body="", variables_schema={} + ) + db.add(template) + db.commit() + + sample = Sample(repo="r", commit_sha="abc", language="python", diff="d") + db.add(sample) + db.commit() + + experiment_id = uuid4() + for i in range(3): + run = ExperimentRun( + id=uuid4(), + experiment_id=experiment_id, + sample_id=sample.id, + model_id="deepseek", + template_version_id=version.id, + repeat_index=i + 1, + status="done", + ) + db.add(run) + db.flush() + result = Result(run_id=run.id, raw_output="- null_pointer") + db.add(result) + db.commit() + + score = compute_stability_score(db, str(experiment_id), "deepseek", "L1") + assert score == 1.0 diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..5a8d9de --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,49 @@ +services: + db: + image: postgres:16-alpine + restart: unless-stopped + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: promptcr + volumes: + - postgres_data:/var/lib/postgresql/data + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U postgres -d promptcr"] + interval: 5s + timeout: 5s + retries: 5 + + backend: + build: + context: . + dockerfile: backend/Dockerfile + restart: unless-stopped + environment: + DATABASE_URL: postgresql+psycopg://postgres:postgres@db:5432/promptcr + DEEPSEEK_API_KEY: ${DEEPSEEK_API_KEY:-} + KIMI_API_KEY: ${KIMI_API_KEY:-} + QWEN_API_KEY: ${QWEN_API_KEY:-} + ports: + - "8000:8000" + depends_on: + db: + condition: service_healthy + volumes: + - ./backend:/app/backend:ro + command: uvicorn app.api.main:app --host 0.0.0.0 --port 8000 --reload + + frontend: + build: + context: ./frontend + dockerfile: Dockerfile + restart: unless-stopped + ports: + - "80:80" + depends_on: + - backend + +volumes: + postgres_data: diff --git a/frontend/.gitignore b/frontend/.gitignore new file mode 100644 index 0000000..a547bf3 --- /dev/null +++ b/frontend/.gitignore @@ -0,0 +1,24 @@ +# Logs +logs +*.log +npm-debug.log* +yarn-debug.log* +yarn-error.log* +pnpm-debug.log* +lerna-debug.log* + +node_modules +dist +dist-ssr +*.local + +# Editor directories and files +.vscode/* +!.vscode/extensions.json +.idea +.DS_Store +*.suo +*.ntvs* +*.njsproj +*.sln +*.sw? diff --git a/frontend/.oxlintrc.json b/frontend/.oxlintrc.json new file mode 100644 index 0000000..6fa991d --- /dev/null +++ b/frontend/.oxlintrc.json @@ -0,0 +1,8 @@ +{ + "$schema": "./node_modules/oxlint/configuration_schema.json", + "plugins": ["react", "typescript", "oxc"], + "rules": { + "react/rules-of-hooks": "error", + "react/only-export-components": ["warn", { "allowConstantExport": true }] + } +} diff --git a/frontend/Dockerfile b/frontend/Dockerfile new file mode 100644 index 0000000..c3073fa --- /dev/null +++ b/frontend/Dockerfile @@ -0,0 +1,18 @@ +FROM node:20-alpine AS builder + +WORKDIR /app + +COPY package.json /app/package.json +RUN npm install + +COPY . /app +RUN npm run build + +FROM nginx:alpine + +COPY --from=builder /app/dist /usr/share/nginx/html +COPY nginx.conf /etc/nginx/conf.d/default.conf + +EXPOSE 80 + +CMD ["nginx", "-g", "daemon off;"] diff --git a/frontend/README.md b/frontend/README.md new file mode 100644 index 0000000..d6af7e3 --- /dev/null +++ b/frontend/README.md @@ -0,0 +1,32 @@ +# React + TypeScript + Vite + +This template provides a minimal setup to get React working in Vite with HMR and some Oxlint rules. + +Currently, two official plugins are available: + +- [@vitejs/plugin-react](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react) uses [Oxc](https://oxc.rs) +- [@vitejs/plugin-react-swc](https://github.com/vitejs/vite-plugin-react/blob/main/packages/plugin-react-swc) uses [SWC](https://swc.rs/) + +## React Compiler + +The React Compiler is not enabled on this template because of its impact on dev & build performances. To add it, see [this documentation](https://react.dev/learn/react-compiler/installation). + +## Expanding the Oxlint configuration + +If you are developing a production application, we recommend enabling type-aware lint rules by installing `oxlint-tsgolint` and editing `.oxlintrc.json`: + +```json +{ + "$schema": "./node_modules/oxlint/configuration_schema.json", + "plugins": ["react", "typescript", "oxc"], + "options": { + "typeAware": true + }, + "rules": { + "react/rules-of-hooks": "error", + "react/only-export-components": ["warn", { "allowConstantExport": true }] + } +} +``` + +See the [Oxlint rules documentation](https://oxc.rs/docs/guide/usage/linter/rules) for the full list of rules and categories. diff --git a/frontend/index.html b/frontend/index.html new file mode 100644 index 0000000..0fca6f0 --- /dev/null +++ b/frontend/index.html @@ -0,0 +1,13 @@ + + + + + + + frontend + + +
+ + + diff --git a/frontend/nginx.conf b/frontend/nginx.conf new file mode 100644 index 0000000..9811ea3 --- /dev/null +++ b/frontend/nginx.conf @@ -0,0 +1,18 @@ +server { + listen 80; + server_name localhost; + root /usr/share/nginx/html; + index index.html; + + location / { + try_files $uri $uri/ /index.html; + } + + location /api { + proxy_pass http://backend:8000; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } +} diff --git a/frontend/package.json b/frontend/package.json new file mode 100644 index 0000000..3a30ba1 --- /dev/null +++ b/frontend/package.json @@ -0,0 +1,41 @@ +{ + "name": "frontend", + "private": true, + "version": "0.0.0", + "type": "module", + "scripts": { + "dev": "vite", + "build": "tsc -b && vite build", + "lint": "oxlint", + "preview": "vite preview", + "test": "vitest run" + }, + "dependencies": { + "echarts": "^6.1.0", + "react": "^18.3.1", + "react-dom": "^18.3.1", + "react-router-dom": "^7.18.2", + "vitest": "^4.1.10" + }, + "devDependencies": { + "@radix-ui/react-slot": "^1.3.3", + "@testing-library/jest-dom": "^6.6.3", + "@testing-library/react": "^15.0.7", + "@types/node": "^24.13.3", + "@types/react": "^18.3.3", + "@types/react-dom": "^18.3.0", + "@vitejs/plugin-react": "^6.0.4", + "autoprefixer": "^10.5.4", + "class-variance-authority": "^0.7.1", + "clsx": "^2.1.1", + "jsdom": "^24.0.0", + "lucide-react": "^1.28.0", + "oxlint": "^1.75.0", + "postcss": "^8.5.25", + "tailwind-merge": "^3.6.0", + "tailwindcss": "^3.4.19", + "tailwindcss-animate": "^1.0.7", + "typescript": "~6.0.2", + "vite": "^8.2.0" + } +} diff --git a/frontend/postcss.config.js b/frontend/postcss.config.js new file mode 100644 index 0000000..2e7af2b --- /dev/null +++ b/frontend/postcss.config.js @@ -0,0 +1,6 @@ +export default { + plugins: { + tailwindcss: {}, + autoprefixer: {}, + }, +} diff --git a/frontend/public/favicon.svg b/frontend/public/favicon.svg new file mode 100644 index 0000000..6893eb1 --- /dev/null +++ b/frontend/public/favicon.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/public/icons.svg b/frontend/public/icons.svg new file mode 100644 index 0000000..e952219 --- /dev/null +++ b/frontend/public/icons.svg @@ -0,0 +1,24 @@ + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/frontend/src/App.css b/frontend/src/App.css new file mode 100644 index 0000000..f90339d --- /dev/null +++ b/frontend/src/App.css @@ -0,0 +1,184 @@ +.counter { + font-size: 16px; + padding: 5px 10px; + border-radius: 5px; + color: var(--accent); + background: var(--accent-bg); + border: 2px solid transparent; + transition: border-color 0.3s; + margin-bottom: 24px; + + &:hover { + border-color: var(--accent-border); + } + &:focus-visible { + outline: 2px solid var(--accent); + outline-offset: 2px; + } +} + +.hero { + position: relative; + + .base, + .framework, + .vite { + inset-inline: 0; + margin: 0 auto; + } + + .base { + width: 170px; + position: relative; + z-index: 0; + } + + .framework, + .vite { + position: absolute; + } + + .framework { + z-index: 1; + top: 34px; + height: 28px; + transform: perspective(2000px) rotateZ(300deg) rotateX(44deg) rotateY(39deg) + scale(1.4); + } + + .vite { + z-index: 0; + top: 107px; + height: 26px; + width: auto; + transform: perspective(2000px) rotateZ(300deg) rotateX(40deg) rotateY(39deg) + scale(0.8); + } +} + +#center { + display: flex; + flex-direction: column; + gap: 25px; + place-content: center; + place-items: center; + flex-grow: 1; + + @media (max-width: 1024px) { + padding: 32px 20px 24px; + gap: 18px; + } +} + +#next-steps { + display: flex; + border-top: 1px solid var(--border); + text-align: left; + + & > div { + flex: 1 1 0; + padding: 32px; + @media (max-width: 1024px) { + padding: 24px 20px; + } + } + + .icon { + margin-bottom: 16px; + width: 22px; + height: 22px; + } + + @media (max-width: 1024px) { + flex-direction: column; + text-align: center; + } +} + +#docs { + border-right: 1px solid var(--border); + + @media (max-width: 1024px) { + border-right: none; + border-bottom: 1px solid var(--border); + } +} + +#next-steps ul { + list-style: none; + padding: 0; + display: flex; + gap: 8px; + margin: 32px 0 0; + + .logo { + height: 18px; + } + + a { + color: var(--text-h); + font-size: 16px; + border-radius: 6px; + background: var(--social-bg); + display: flex; + padding: 6px 12px; + align-items: center; + gap: 8px; + text-decoration: none; + transition: box-shadow 0.3s; + + &:hover { + box-shadow: var(--shadow); + } + .button-icon { + height: 18px; + width: 18px; + } + } + + @media (max-width: 1024px) { + margin-top: 20px; + flex-wrap: wrap; + justify-content: center; + + li { + flex: 1 1 calc(50% - 8px); + } + + a { + width: 100%; + justify-content: center; + box-sizing: border-box; + } + } +} + +#spacer { + height: 88px; + border-top: 1px solid var(--border); + @media (max-width: 1024px) { + height: 48px; + } +} + +.ticks { + position: relative; + width: 100%; + + &::before, + &::after { + content: ''; + position: absolute; + top: -4.5px; + border: 5px solid transparent; + } + + &::before { + left: 0; + border-left-color: var(--border); + } + &::after { + right: 0; + border-right-color: var(--border); + } +} diff --git a/frontend/src/App.test.tsx b/frontend/src/App.test.tsx new file mode 100644 index 0000000..dc3a826 --- /dev/null +++ b/frontend/src/App.test.tsx @@ -0,0 +1,10 @@ +import { describe, it, expect } from "vitest" +import { render, screen } from "@testing-library/react" +import App from "./App" + +describe("App", () => { + it("renders platform title", () => { + render() + expect(screen.getByText("PromptCR-Lab")).toBeInTheDocument() + }) +}) diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx new file mode 100644 index 0000000..5a912f0 --- /dev/null +++ b/frontend/src/App.tsx @@ -0,0 +1,64 @@ +import { BrowserRouter, Routes, Route, NavLink } from "react-router-dom" +import { Button } from "@/components/ui/button" +import DatasetPage from "@/pages/DatasetPage" +import ExperimentPage from "@/pages/ExperimentPage" +import MonitorPage from "@/pages/MonitorPage" +import AnalysisPage from "@/pages/AnalysisPage" + +function Layout() { + return ( +
+
+

PromptCR-Lab

+ +
+
+ + } /> + } /> + } /> + } /> + +
+
+ ) +} + +function App() { + return ( + + + + ) +} + +export default App diff --git a/frontend/src/assets/hero.png b/frontend/src/assets/hero.png new file mode 100644 index 0000000000000000000000000000000000000000..02251f4b956c55af2d76fd0788124d7eee2b45eb GIT binary patch literal 13057 zcmV+cGycqpP)V|)f$;Qooc7=_G zlYe)HToTQIc!$)^+J1M1y0*T%w!p~7%ux`!eRhO?c80XDxKQ*R^lUUMnA>6NT^?feoZ8xxvP32D&s-9ow zqjcM}eesrC)NeDmsf)*P7wJ|K!&xP%Zy4iI8lF)Tv2!reW)tCzg_1=PmOwd1SQfxa z8;58t!=z~Ba7CYlNWVG>he8aRPY|+-JmozNhn!#9i#77Aa_Edt$ijyCWL#=~I>~2X zZNrQ8I0=D+NWD4pq=7~(i zhfThMNw|G>g^y9pGzxX7ZSApl@tIxFcs{p#MX{Ax&XZT+cR#U+OWc@S)pkIuI}dzu zH?^Q=<(y&Vq-oxSLfc0Zmq81bjZWf}RnssBaD6}2g-XJHLcN_|*IOu>m|x$nbm(?E zyNy!Zp=RroS;?Vg*kmoJYBi!n5{_^@rA!)=t#a^;N$8GL!*DsQb}`yvEuX!G@||An znOfUZAevPrkV_qjl|<~3QRZzG&h@C9Y5z zqpNH4xqbF_InIPh)kX}Vn^5kyed|mOuq+2>M;v~KO37a#yrEn3XDqtOl=rc6_KZ!; zreo)DFVB4|>1Zd(bvMI%8uM;3!)YMYu&cG?(PE!B~y@3yKBMt|R zAf=I16tFwPsl)!jDqvYkLHaAQ+f@W1m6F5aZvwhm4JL z{_l)@b;)mDSzle2gyFP5-r1x-5X{G}ot%VyWP@vEW80!Q=f%RTfpg>B*TA^pyWYUQ z<=xPtz}WcZ!;rFl4m1D&FFHv?K~#9!?A%+fn=lXt;9!Fc#kQ;zk~gZFsH z8e5iu@c_pzX&qb8&Dum*oXwB+fm6l6gFfC|o*wgEiy6tw~&co z9Vd_4)P%wP-KwQW7|lN-znGK#?N+j24U=$982myIBM+vsiKsc*@4-rwJxuAaHKna6 zT3wi!C~a4ZKH03qU}_1bKyx0&$CaK7_%Z+Kl$)fF5^op zZApQF2TvDav!s|krTjw-8US6ep z%!VmX4luub+fseQz_D9ATJQ?iQQwD}TZz{-yo#l12a%+7bT@E(X-hyaVS-5vuXc#^ zx^w;L21;NphGVoj*{s3f4dme0y2LC=G1-7THd`#z?;tuC{^9k(dM{Rf2GOxg7Jzho z7nSZHl7?M9kdalX`)YgoKEfiae5+;$(OGeN1eqxrv!ZCVKyH>xiyNqfe8xzY8*7)H zQls8KMp)F4D>ED;idMOU^^WhVF@q>ZSmeB0y~qC~|DB648hr%Sh|*T(4q|w2l?m2+ zvBVw3@7+Mz?^Yc#+se6KM;a<=(W-I>k)$-qL2V*t}VaW`;?P4)WqI%maIDq8!oUcSYAD`}wWjkSyAVsnF65#2zQ zZ>(K*TlS(E#4y$4Zq+e^_&}d)q20hCe3!LfLYP%nQpLJ~gM6a1hJlz3)aS<9C9me| zAcmJ#>tOwBy{HoP0Sm1&_(E+S@6 zgBIFUoei8zJmdpiq8q5=OY7t@`)JWxn_&GvKVr=Zdb_pEL_j|=?f;WK^U9Q0efd#K z9q7SfJTl4pmA$jsZ5oK8@O9#!I3Cv-kL)<8SalSsp#dcpvJ}Nz#G6FC0%9|7Fi#8; zGDJXtj!&GljT3*HE@0EE>G8Se&d)*nkqe}-?`3vPl&UqK?xG z!3XJ4M-x`EuQjhBbu?ik-)rmIt=DF_N?TVMP)8Gjn)TZ2V%H|zENbeix}kOxd@0}Q z>)HuH6Ean!uS#~4g2Ne2WsMGel|h%j9*W_quQheG^JqmKhc*RYzp0wKlGjBq2VzY_ zgOv8WC1+%W=W)k)Yp_`8kfE=uiiwOZTXi8Uj9YGr$f@yJcJ;#&-Nq~sJ7anE(@;QN z=~br%7%7`isKStX|7!1?L(apl^QvPKlrHV4S+6tNVQ*R1iGdC~WMNE1$a+=rpQmcB z>wxiLIBvOnm;u*;9Y!kJdy(T4lk|8>JAm(&wEsFIF1$_*{>2ZNd$V6DS=SfrGxAv0 zzKe377JI`&o9Ljr+VnS*EwehA{f&{cKZF(6*MG5!p5MvrFA3ll{fmRG*L@6^cb;o^ z3Wm8c?Sc6$`>~VEWw(c$Y?nRO;2Q$=ulpqPtM^=1IZx;@xK0PgO7rKQ^WHVLwtgUT z%|JF{^f(VH)wLKQ%dYiu2RmchBdxL0-M?wxxul_z*{h6ZZ`>-k(vizs((vW8Lt6Z6 zY;Dt?@JWyN`O`f;&d1Mb?e%9oyRK1ql?EE5XB2(W)|D1~Rx35$H6@6)$F?)7V|zEO zI}fu0-0}8W5=6sg$fPnZ~7=tTudl?Ecb@pxbo)vni%gP-?hL|%*?62C;x6?@E`VRnJv z?fTb;k4x;TS7Cu-z%J}uy}e-pwpLQ17Q@4DC+FCdAmNKklG$`I_pyw7E{fYmw~{Fj zi?6KcVy=Wrel)EB_DWO|0CKmI|13!gBV?X`Ozp7x>?6jr`>Qz=^4ea35!$*f}) zS$i+x_k+@P2q1RFUH^ZTTk7=n?cjfR>hTq3l3SY~#w+I8SSutXGyhw;Ws~=zMQ%Vc z>$On~47Ut?P*_!TOQ&PFmLAyJieB2X4_Fd_!WxI-AY`q1Lc-oK?+qcOTzlQ?@~x@OT}*9jTVNfl@3rGvZpWI=eKg>T zZb@6YWz)J=IhP7CF|c?G62vMEG%#U}?#86$0jR4sG~i(jRd#jmn`7b(O#?N;3a;1t zhXLssmUwGhp79luw#(*V8WL0|8+E z6=YZ_O@er~$LrD_PYGc(kJgB=;yw#+Z3X6LDUZ(NcwN=B-hjdiHm!JFar%m{(5bEW z@@_VEtG$5;`EJZ|OkJ@l&G9n((w@uNFwmU%bG|s#TbcJJos!{e+bjCjrCq_}LcN!UFgKtgg7siV*7# z!}1whTRRi*-avJPu->C}Z8EiuK$#886+H_#_!btv+rsiBbv2jAJvJ+O0{#}y(%L3H zfjU-kq_-L@2XrL*ae{{qYJkD{@dw%*bkh2P&YS-0!Xt!PRz7KHV0+~j(t9W8lAVWR zt@B*DgURgEz4>WuN>o?_iKcw$?k{||Pg7{Q2o4|VmJ)mg?{VQJA<}zEr^YAAS zgGm5RT4T3p)U;yz-tfBO^kw8?IoG!IVmc+Z3m#}AOQ?5MRa>)OcU!$N^_+yK6ayn? zK>~WK0!#ysuj^oNLakm)Zvu+J)OSubX^kv!c*xgdIvs;kln!rgG4*uZ;w0mQQO4XD zO9P{GNdv!=cQ(CAL{S(%KtuV^zC&Q{%g)PoXnp^gn^>c*`E>$hLYg2HjnbVGtWLa{7zHdG1jT@B{|Dm16 z7K2(jsfG+m*Zxof)iXxu+!H5Mo-0$pkyV3VV4B@Qms46M zuBxGRV@HxU7Wwx-6CB zaU*HO<_qn$5GH>&@?nRy1{z zkik!sLfWQ)r#75)vVwCBU*r_)Q6mp?!j85{#Xqse)ApRdE$V0%I0*~e(_{)5H)`Mk z#rExC>yjhZxuL@|+#v4#<Axw$+VpV zuT;!2Vww$je$DpAW`$FX_Ab|Ip%$;&T$-lW8jS~B$>G}rd>eQG+$h9lQx4Mx0w={m zx9?T6VU`>sR}XClkAhHEShOUe8awiq zmizhL+}5UKs3}6~It7vBTig9dfQ2Q8coo+Miiaw7n~>4ybv2Ptt0^^=VqX(t*Yya9 zr`FxxFX8(v*H=+uJ#JJWIB2A(==HDYx~^zZ2nu?2`}|Wsa*f3h3ixc+U|FDtAG$Y! z*lc_7se5Oso-Cgqe0){{!8H4g$3<8!R<6JOurD;((({c$1(pwb>(#TT!sge@4>r2@ zVL7>U`0`nsWAYErezk4(Z!gMI2?UTo{J3Ajo(u4)KYIRd>BRcG4BoS3G0EXyEp@tw z%P7__?A^a>Q&AKL@ayDO9D*Qkc!NHnO9l}kpp_6hXbMppYL(X1L?njdFT|-h2<_$; zAtDZ!1Rf%|yb!qbWKd}%0b`LzBeyNy43|QO(&h2mxQLUL)|0%agVOW)6TV!&Ip^Ls z`PG2cygM8)IecQx=Fc+nqYRo4hS^^-nM_&-y8?EJXUczP=DIw(GkTJdpEdh<_STs{ z|A)4n1GKdE=Wu!!nYoZHcUQ4S&R;oDOKX2lrkdF(mK>hz<$Pp>igjOcvoRIjlN=W8 zu8Gx5(roqn8$>gEE5vy{GiGeW8Tq{vnf3hS-V=$tZkQuftUVuU8o6k&dn=Yg3)6MOIH>nlK^-2+C6BZITr~1@So?NvG#TwL)|~=1YXGMTLpS<)ziK_CSOabe z=cB#5)yz|@0i9dSo?*CX)}UP=s6)B+F@~Em(u@Q(I9J9i_V{LmMu8BfXYMh~*oPP+ z!3~xTv|(>|=n6ZOtT~C@V!z!w%18*8T2t6}U2S##rC)mekBql&VsBX;$~ByGE$oA9 z`0Wzq8p?R{4)$l*on;!cLa}Dh^Xe?owiQZt9nH1fxxh$pN9K%CtOw?u3>85L7rr!d zXs)l{TZ{xXP&U8exz?9cv~dNNibOmt*K4I$?RxqIBZ0(?Mg-9FS{*9Bc49Qc1`=sIF-rye`aNT1G@4NwXcnyc@+bw_mTsR>5< zF<2;X0QesG_pw|TonqVBhRtfqI>ty(SIu&VOXd0CrLlfp+;WH7HYjhqnu^oAY!9cB z=B6#R?Rfz9BP`dJ=@v_?70s3HxQPk+{6Y+lM85f2NF^00*^OcM0~?JOZfR9ZPYF+# zYSs}(_BUYV8{n@2a1hD^SV41bwmi2uztR;PeBgF1F-`9>`zoNss-@3LaF2sjl~>OaaVmp7PNp+UT`6@}gR%uzqHDVeEZ14{Yt?n%JeQm+t(1_u zSc}oj^{b;+rlS|ME%+LjzSI&xu0Bblxo$MJ-J$kJ?Qu_XUXh}*@*-x@ny|}wVM%Lg z3tNB`yvr*}N?ClGL;H2cglcvErIccU3(eP7>@~4nOIcI~-`P8tSQnx=jI&{9)!1}l z;gQ%_h>ZlPSV@o@Azq1R$C6ja5!^ZGh;YRhhxs58qJWo9@Bceac&yy(pET1hnn`~7@}2L0&dfPKYs$ih7m2}R!25!(hxqA(!UIw; zK4+~Jowy3=RNC6nE=ncU{LH5?*9@W24lacJlvCZXB$CYtE@>c+~H zkV=(5I&gb{xn2!~f&fs2NQgAL6`p|kyt6kpWk}iVlqIp(H;ig`{_U9yxs1jzu^ETM z7~)Rg8C-NueqTYP&U8l{DY=Y47cR zOR@U%$KQV{mkRF|4)z9Y^t3K`@p>duY&QLUFeh6VoV`a`$U@)(z!-N*5Cj<11$EZW&hJLX83TO{lJYP74rlDZQPkm@t<=U^I)x@|UnHHkdQlh?!ltZwl92rE;;^ zZuIappj4dhld1}kttYYV-j|KF1Kus zWBnzttD^00%LFK(wrwNragFub6xiV8QE2rm<`&fcR4SLFcdtLxVuN!Aal-g6dE4%k zARZ}|xeo;K{0yf7@9aua%2j5o)CPcIOc6uLHFJOcgtB5owlcNAwyAHc0QB0Dts?c@ zUemG~j_E&W7R%+x-IO4FJl8e&*2Blmp1S#RA|)geVrxvP)NHdYuxi~g&Etn?QdNK8ZDKZ?QFLU?zh30G|t9G>a_X4zk}Ygw<^$7K!GIn(Io$>(d4ODJQ2XSd%jpK zm7>ptl$a3GyB}5-%p4>Q*p#VL^B{yQMuFCM^#l#+N!Ne z5_PrJWB=@Iy+t)H`g1lX`{bm($KE5I?0c(JEYm#t{F}j!xtsbob0{xu@0TB_*>G7w0ICn zr#VoBktqHZ~XxhiKD*lcG|b;H*|Ny3P^8ceV`sfBRfrhwZ!T+MFZ!F1Bt{q$8d9i6o?~ zODj^POr}&ivSa^R^YFIq7o0giLBKCycH_aU`F6)O6JX%nPTwh~Q`eq6*0iE#Srj2^ z*_hN3%*b83zfafy60@Cp3{J({RlSaEn&E?mrxRNC9GQ7#+f=s! z0KBf-9Ny_v2VbE%aB|Di)5kNJ^t&C`4D(>t7zYUWUFtbxt+Oq=!@O7BU)}>d*R72o zFF)3jQD_lLe4is&xzyJYC1-c{8TX$RU>&>P$%)ufpez0XSAukmh!xcekg`s$c<>-q zI#zn^JU0zzF}V60)o$_gY}PQH>b2M9&8fRZa#OauglPb zeQ@pMm&=!vNgos4CluQjLMV!pfkmxK+35bi^k&=k>9h02?l+u+m0agG;(h2|Jslc-llvtEwn~*w3bx7qnvZACG<8}AGeaDVvcHbKd2>3G^ zSFPULUn-?Pmo^-_`mLZr??uNH`2=I&yajlrF{DtUxMy#Nu}z=3y7qbUA;5`)hibMR zhXL@@uKyV0-2&A@t@!xyrBnMJl&^o@Gx$&5_q6?D=ji5grd-~=?dlg;ur(_V0wjh! zA=JV^C1m+DDkOsgr<%O9ZQFg!0}pD(#PSz4Dr_EyS5$`)VIAv);4n-SFP~YtC7sH= z7&*MfpH;gd*FHbkmD#)hVxb6xjc9~`t?_{=JS+@ip_cTicXxG<=7m9& zPX+Z8IC*GSAXuGCrZDHgR$r%jyk-fctis2Kx4HvZ|B~8uC@o)m^>Hy-O!&TKA?$&n zkP2Xc54w~!=z2?^NafyL*L0V9cbYrugHBBUj`xVyZmGFR&kvk#>1J*Z~i zNTz}?IAdJ$gkqd2!Gw(%LzE!O5s4C7q4%T~e_P{+z=DNDKrG**p=U`d5yg^vp`;Zn zsU=8gd0a9s4s0FPJePWR9eH5=+O^Kks&kC-iblNqTh2&Pw*^(4384f+D8N|fewZu_ zg2ejQ)ov;ztz;NQl7yj;A`(!H!XQu_$sqY9h_IrH*}_%1{L&_YLDvO?%R5Z-t+ClW z_qERbL?HKUZ!nt+!E9S`uoh^5A|DaIHe*_gf1`E_Vq+}{&T@t$EGhMnRjJ4z2w_W8 zp+qjs7as22^&S3wY1?+}^j-I=RcCE>#|39)g(lU7v_8;?=qK(9D8-*pPdiy)P3lIblG`+?%ea| zYoD3dopYt!tKgFicfNmNi(EWE=E4hC6(r|PYtanqJlmt57YOVrr2^tfrG(eG9C##X zu&1t@%L$RIvpj!wUA z8i>Pqot#_+Cnp6L2XPcZy1ar|9MnY+7eNvK1E)@Tr#2KsXq1*>)uUCozT7L##ok?o zhA6ofP4E|b*9tAfG?uf$#}>TIR&1A!yslP8}i7w-EzW(x#9VEvx18k%Tn=-$VV zkOtUr0b2!w3t>h?#8AZl^Az*(6KCGlD;4j~yx};`#2gN1_gv=%7KVzecIRakN{f*4 zeaI>yH;-o4OGhvGTU)(quWI)-q?V*(sVesSMv|wMUQ3hLEt=lBB$KZ9TyHr>)f7o%) zPYeU<3P)*P10*7vE)nA5#{c=6-E-_>r_u4e3i!I2+UksELwDqwMeBZ9FSP$;^Ajro z_@M#_Ss$?ejoB@!wN|kbGKs(0zLo%0QpQXW#t;oC$B0MZYZ&Ej?8~fNhcCVvPo3vo zFn0WWZaPliF^8_}yzb`*f@yg0uWv6HgNI)xa=pO%Ck(C<=-60l#uD3(wXP~c7!NoX z0&^6=N`zcc90F#qt@=Rn@r!3(*1v(Tl{B!m?Mc7yIA+nEHpY{YWr$=)F7rhR1P}(v zt{YhY#;jsW6G>#xhP*B`OCk|Pf+NN;ju1rxa*HAgoGq*rvqw&xe~;t1JA31$s?GBb z*g7&@cbKo4n<`>)!UlIAgR6q&))B0KYU8r66GbFj?8Guw4E%&}Qi_lT003LtoIZei zwD~=XZmeo+yZ2Pq3KYCF-R&11^p= z@H%s+=G`}wrbJ{()Mh71#2SP3Zy3m>l1n?0N-N1Q;z6?oSxr-G(H5m4EO>~&;}VKi zfY}3w+9z>vp#d)hVuu`)vG_aaH%3b=WKMnSu&c31;<3O;bz2iD=w+o4#oBb36 z5ZCF*Gu?zjZIR0S>_%pHY2$k8D^n7Sz_K8tCDeXM+dO<#LSg%h6`~dnVG1N@T7v&e z%wEd1!k{^zfz_1BTW{!$!B%g)J^2b87!9Y>>100X1SgT7s0z$o>^lAA=Gp_cC1(h=*5Tmf8z&LGJJ>$|K^~s`z9*OWz5MFUr?>Bi?_PGBB)#psD5?>n+q{o_ zz7~ez&;t#h8l$jwGPCC&xq2YetXYQT+0F3j(`xmNGf8dj#an|p#I*pvI*kwW4iuB> z+q3_7xB8y;pLzHG-S%+UHQA zvqp;$kmGJY>lLsN4C~&TcvAS1SErTcwcw0r@wngk zShAUA1M9b#g}^pL-zH7Q#z^&j#r9F8BTVfkR&qF<=e35goTu7c|GN)0mokj4m0%~0 zXJ8j4Hc_l;HJ&uU*Iw`8d_EscJ``s0tk9mkKo^&#TYXm-EoAzTQObxa@^u~g2t#T) zJz|rE!I_?i4dCJC=B8(_pZ{YR>|V?0iCcnU;E@$239^x?SYCfNaMHN;CtHIS_zHN9 zTkQc1v@O35okiFtq5_u+5FkY55ap@pi)O?}x0D1c*qB0KpYR}>Ul+B0Vmr}Z@+%mJ|As}sis_=ROPbov@*2thpE&?!V#Qgu$snYvCZ zrkhmkMU+fSf-s8(L37fPr&M*jRs{{THb!aXQu|P9l_-vJhHvLzMGH zE?1U0H_+PmNABp9`|KzkGfrrZ%XvdGo6*<{d5m9~L7 z_^`M;X6xDo=m6LY6RfvJEvsTK1!u8d2HPx|$S}p;sRy!I zWL55Yxu~_B`OP@~(q6&W3#)~I&+MGL%GWR$#udC151^wsswhqlii;rP9jJpiI7o&Z zAb})=HY7?4HA|re3ns`%$)FuvKCFWjhb~?IE)F6dF2K5}poj-NK6Gf;hw$t3=1txY zoxQxZWrQU6K!%|~!m?~Bnw-6Rr!F3BZ{u5!LqnZTDON}Coj9^@&le)V!NYrVwS~B% zEL+>Sr@}qGwGvu|HrOo|gSt__ezN^&%~{*)a=rf7y1HujUcr`zZB<4#l@T#eN)si} z)lZA<{=tKx8E%c9>A(##6}_p+~EZpKsl5a4pj`E*;_-6`ysiv zffA!7=MT1vCz}-m4~tjVey1b2KSR4OEtLd-(_DdUqYZ74LaDkhH?KFh?%WAOP2WbX zp@zT+Dx|5_f%JQiAGvVw!oh+g3e50u!aPfMxdC=E)XB{F5IcEZhePIM- zph6Y`$Oy?JBL<8Ex(SqEhLeQ@XcrdA>a?rx+_~HLA;l14)WmmpH}_w?Pg#HBZs0eS zwypwAW?M-x+3AU-(GGWSJ=ngxUEcEZ5OsX(Qlt!MQ zn^(`S{GHkAv(8@D`EAfSYig%Cxv?z!{=w^F#y)5_d7FuKZH7qlR-#5B0bt806%D0I zT7VdVP_?q*%Rq8UR;JkD4i^RXowt+E%#V2U>TfDqzZSDZ+dR!a#T3I>-z_$q9@k|m zy5~A*m~&JWP@E7a=pc}4kVHTc4h&R;Li7d@f`|hKMLkbb^uhOakNr3&FLjlm~i5NBM< zFaYI{;cpiHCNRdE0dg*>qIm(_t?#$h=(SCw?h3rJV2*ER8{O4^3#=dO)KwklZkoqU zS8i5c%YL*y*4;FY#D=XmkQnYj%LH)?02~gSJH`Qp1XY64g>%c_K$xseI&|e)7vRoL zAqRba$G@%fSGA7X7hQk%_3NVOYVS+$leU_!&6*5uN)8#5ZBz_6ASCA;azYS-Rt@ki zg2NWz(=;t}SC(~Ibl63$5C8FPmhXqb^)5#jaJ~I{Ex3xZ!+2h8$}}h_g@Be>HZ;72 z6#y#>AY3^skuVKF#0WxFBQ()5d5_nWb?c6c>EeMM|Mh+*&wEpPyxHCq{R-Gdr-`hN zF=1sxl&mBoK+#qRLl9#CEN|Fg8>nbmsTg3a1;#M9enQ$RgWk}kp#-5wh=EF&1tl%mJln2V^8o%Qv(*=zEuO7y z=m*8?xpUn-*@h5Cl_3BK3joiGkyaScK+>|MWdMRWm@RT!Q1piAlv5hL@B6>3&GI8) zP!xBc6}ZNIpJLL%2a8Y!+(<=f%WX>_uWVxlga9!D*oYt$l0cxRDMvqfU;Kq_mLK5k z)dvqYcgLa_Lz?3HyeF)@$%$&6lI?r4I>6W#M*<)vq{?&Oqrx``d`mhpVPr> z#q078F6gw_X<=?KR>8%^t%@wbITvNMu!hKiTSkCTJkw>1!e*Y{%31#_yMf=LW7{RJ zYoC^w$6%3cBtVG5)x#{Hg6IVTh9XEcM{gQwXk!R^y95^f-hZ`d{aVa+xW1EO4wDV4 zB?JgD7*?qkvc|$nIykTvNl2x0j3Q!MXoLL^)~}d7jcYf(H8D~c+?$pKL(px>Z3`eb z04RzS6_AgFT6Pn#iZAg$Sl_j8#;6ShF%&(Fag#E2asU@@LaN;=b=Wf7sgPKhfzhBM zC@eFL8^MrnA*9&Khe*Ab@CC9*uyJGXyi(;y2>lQLJZt;ShtJi?3Yf_t`F+$hY!+Q2Ndsx=U+bjTiAy7djLji>7k%k`$9&--f<*BNA3Hy&ZrHH|4 zG5H&9cB?O#zI1_OOf0Ce%mDfQxdtp3vU%(iY6yji3iISS61XLv#z|!zI_sZqza@B+ zyu9st5-h+`H7QUKx9}3w@oU@EO}&cEzG?fu!!bLO->%zkcg;i9^j`S~=WKMnDi1f= P00000NkvXXu0mjft=yBf literal 0 HcmV?d00001 diff --git a/frontend/src/assets/react.svg b/frontend/src/assets/react.svg new file mode 100644 index 0000000..6c87de9 --- /dev/null +++ b/frontend/src/assets/react.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/frontend/src/assets/vite.svg b/frontend/src/assets/vite.svg new file mode 100644 index 0000000..5101b67 --- /dev/null +++ b/frontend/src/assets/vite.svg @@ -0,0 +1 @@ +Vite diff --git a/frontend/src/components/ui/button.tsx b/frontend/src/components/ui/button.tsx new file mode 100644 index 0000000..11ba890 --- /dev/null +++ b/frontend/src/components/ui/button.tsx @@ -0,0 +1,53 @@ +import * as React from "react" +import { Slot } from "@radix-ui/react-slot" +import { cva, type VariantProps } from "class-variance-authority" + +import { cn } from "@/lib/utils" + +const buttonVariants = cva( + "inline-flex items-center justify-center gap-2 whitespace-nowrap rounded-md text-sm font-medium transition-colors focus-visible:outline-none focus-visible:ring-1 focus-visible:ring-ring disabled:pointer-events-none disabled:opacity-50 [&_svg]:pointer-events-none [&_svg]:size-4 [&_svg]:shrink-0", + { + variants: { + variant: { + default: "bg-primary text-primary-foreground shadow hover:bg-primary/90", + destructive: "bg-destructive text-destructive-foreground shadow-sm hover:bg-destructive/90", + outline: "border border-input bg-background shadow-sm hover:bg-accent hover:text-accent-foreground", + secondary: "bg-secondary text-secondary-foreground shadow-sm hover:bg-secondary/80", + ghost: "hover:bg-accent hover:text-accent-foreground", + link: "text-primary underline-offset-4 hover:underline", + }, + size: { + default: "h-9 px-4 py-2", + sm: "h-8 rounded-md px-3 text-xs", + lg: "h-10 rounded-md px-8", + icon: "h-9 w-9", + }, + }, + defaultVariants: { + variant: "default", + size: "default", + }, + } +) + +export interface ButtonProps + extends React.ButtonHTMLAttributes, + VariantProps { + asChild?: boolean +} + +const Button = React.forwardRef( + ({ className, variant, size, asChild = false, ...props }, ref) => { + const Comp = asChild ? Slot : "button" + return ( + + ) + } +) +Button.displayName = "Button" + +export { Button, buttonVariants } diff --git a/frontend/src/hooks/useApi.ts b/frontend/src/hooks/useApi.ts new file mode 100644 index 0000000..759e13a --- /dev/null +++ b/frontend/src/hooks/useApi.ts @@ -0,0 +1,64 @@ +import { useEffect, useState } from "react" + +const API_BASE = "/api" + +export type ApiState = + | { status: "loading"; data: null; error: null } + | { status: "success"; data: T; error: null } + | { status: "error"; data: null; error: string } + | { status: "empty"; data: null; error: null } + +export async function apiFetch(path: string, options?: RequestInit): Promise { + const res = await fetch(`${API_BASE}${path}`, options) + if (!res.ok) { + let message = `API error ${res.status}` + try { + const body = await res.json() + message = body.detail || message + } catch { + // ignore + } + throw new Error(message) + } + return res.json() +} + +export function useApi(path: string | null, deps: unknown[] = []) { + const [state, setState] = useState>({ + status: "loading", + data: null, + error: null, + }) + + useEffect(() => { + if (!path) { + setState({ status: "empty", data: null, error: null }) + return + } + + let cancelled = false + setState({ status: "loading", data: null, error: null }) + + apiFetch(path) + .then((data) => { + if (!cancelled) { + if (Array.isArray(data) && data.length === 0) { + setState({ status: "empty", data: null, error: null }) + } else { + setState({ status: "success", data, error: null }) + } + } + }) + .catch((err) => { + if (!cancelled) { + setState({ status: "error", data: null, error: err.message }) + } + }) + + return () => { + cancelled = true + } + }, deps) + + return state +} diff --git a/frontend/src/index.css b/frontend/src/index.css new file mode 100644 index 0000000..10c2d37 --- /dev/null +++ b/frontend/src/index.css @@ -0,0 +1,59 @@ +@tailwind base; +@tailwind components; +@tailwind utilities; + +@layer base { + :root { + --background: 0 0% 100%; + --foreground: 222.2 84% 4.9%; + --card: 0 0% 100%; + --card-foreground: 222.2 84% 4.9%; + --popover: 0 0% 100%; + --popover-foreground: 222.2 84% 4.9%; + --primary: 222.2 47.4% 11.2%; + --primary-foreground: 210 40% 98%; + --secondary: 210 40% 96.1%; + --secondary-foreground: 222.2 47.4% 11.2%; + --muted: 210 40% 96.1%; + --muted-foreground: 215.4 16.3% 46.9%; + --accent: 210 40% 96.1%; + --accent-foreground: 222.2 47.4% 11.2%; + --destructive: 0 84.2% 60.2%; + --destructive-foreground: 210 40% 98%; + --border: 214.3 31.8% 91.4%; + --input: 214.3 31.8% 91.4%; + --ring: 222.2 84% 4.9%; + --radius: 0.5rem; + } + + .dark { + --background: 222.2 84% 4.9%; + --foreground: 210 40% 98%; + --card: 222.2 84% 4.9%; + --card-foreground: 210 40% 98%; + --popover: 222.2 84% 4.9%; + --popover-foreground: 210 40% 98%; + --primary: 210 40% 98%; + --primary-foreground: 222.2 47.4% 11.2%; + --secondary: 217.2 32.6% 17.5%; + --secondary-foreground: 210 40% 98%; + --muted: 217.2 32.6% 17.5%; + --muted-foreground: 215 20.2% 65.1%; + --accent: 217.2 32.6% 17.5%; + --accent-foreground: 210 40% 98%; + --destructive: 0 62.8% 30.6%; + --destructive-foreground: 210 40% 98%; + --border: 217.2 32.6% 17.5%; + --input: 217.2 32.6% 17.5%; + --ring: 212.7 26.8% 83.9%; + } +} + +@layer base { + * { + @apply border-border; + } + body { + @apply bg-background text-foreground; + } +} diff --git a/frontend/src/main.tsx b/frontend/src/main.tsx new file mode 100644 index 0000000..bef5202 --- /dev/null +++ b/frontend/src/main.tsx @@ -0,0 +1,10 @@ +import { StrictMode } from 'react' +import { createRoot } from 'react-dom/client' +import './index.css' +import App from './App.tsx' + +createRoot(document.getElementById('root')!).render( + + + , +) diff --git a/frontend/src/pages/AnalysisPage.tsx b/frontend/src/pages/AnalysisPage.tsx new file mode 100644 index 0000000..56b7942 --- /dev/null +++ b/frontend/src/pages/AnalysisPage.tsx @@ -0,0 +1,163 @@ +import { useEffect, useRef, useState } from "react" +import * as echarts from "echarts" +import { useApi, apiFetch } from "@/hooks/useApi" +import { Button } from "@/components/ui/button" + +type ExperimentSummary = { + id: string + name: string + status: string + run_count: number +} + +type ChartData = { + heatmap: string + boxplot: string + grouped_bar: string +} + +type AggregateData = { + detection_rate: { mean: number; std: number; min: number; max: number; median: number } + false_positive_rate: { mean: number; std: number; min: number; max: number; median: number } + coverage_rate: { mean: number; std: number; min: number; max: number; median: number } +} + +export default function AnalysisPage() { + const experimentsState = useApi("/experiments") + const [selectedExperimentId, setSelectedExperimentId] = useState(null) + const chartsState = useApi( + selectedExperimentId ? `/analysis/${selectedExperimentId}/charts` : null, + [selectedExperimentId] + ) + const aggregateState = useApi( + selectedExperimentId ? `/analysis/${selectedExperimentId}/aggregate` : null, + [selectedExperimentId] + ) + const heatmapRef = useRef(null) + const barRef = useRef(null) + + useEffect(() => { + if (chartsState.status !== "success" || !heatmapRef.current || !barRef.current) return + + const heatmapChart = echarts.init(heatmapRef.current) + // We render the base64 PNGs as simple images for now; full ECharts data binding + // could be added with a JSON endpoint returning series data. + heatmapChart.setOption({ + title: { text: "检测率热力图" }, + xAxis: { type: "category", data: ["L1", "L2", "L3"] }, + yAxis: { type: "category", data: ["deepseek", "kimi", "qwen"] }, + visualMap: { min: 0, max: 1 }, + series: [], + }) + + const barChart = echarts.init(barRef.current) + barChart.setOption({ + title: { text: "分组柱状图" }, + xAxis: { type: "category", data: ["deepseek", "kimi", "qwen"] }, + yAxis: { type: "value" }, + series: [], + }) + + return () => { + heatmapChart.dispose() + barChart.dispose() + } + }, [chartsState.status]) + + async function handleLikert(runId: string, score: number) { + await apiFetch(`/analysis/likert/${runId}?score=${score}`, { method: "POST" }) + alert("李克特打分已保存") + } + + return ( +
+

结果分析

+ + {experimentsState.status === "success" && ( +
+ {experimentsState.data.map((e) => ( + + ))} +
+ )} + + {aggregateState.status === "success" && ( +
+ {Object.entries(aggregateState.data).map(([key, stats]) => ( +
+

{key}

+

均值: {stats.mean}

+

标准差: {stats.std}

+

中位数: {stats.median}

+
+ ))} +
+ )} + + {chartsState.status === "success" && ( +
+
+

热力图

+ heatmap +
+
+

分组柱状图

+ grouped bar +
+
+

箱线图

+ boxplot +
+
+ )} + +
+

李克特打分示例

+

+ 选择 run ID 并输入 1-5 分（实际页面可结合实验监控选择） +

+
+ + + +
+
+ +
+
+
+ ) +} diff --git a/frontend/src/pages/DatasetPage.tsx b/frontend/src/pages/DatasetPage.tsx new file mode 100644 index 0000000..707d67d --- /dev/null +++ b/frontend/src/pages/DatasetPage.tsx @@ -0,0 +1,124 @@ +import { useState } from "react" +import { useApi, apiFetch, type ApiState } from "@/hooks/useApi" +import { Button } from "@/components/ui/button" + +type SampleSummary = { + id: string + repo: string + commit_sha: string + language: string + defect_count: number +} + +type SampleDetail = SampleSummary & { + diff: string + defects: { + id: string + defect_type: string + line_start: number | null + line_end: number | null + reference_fix: string + }[] +} + +function ApiStatus({ state }: { state: ApiState }) { + if (state.status === "loading") return

加载中...

+ if (state.status === "error") return

API 错误：{state.error}

+ if (state.status === "empty") return

暂无数据

+ return null +} + +export default function DatasetPage() { + const samplesState = useApi("/datasets/samples") + const [selectedId, setSelectedId] = useState(null) + const detailState = useApi( + selectedId ? `/datasets/samples/${selectedId}` : null, + [selectedId] + ) + const [repoPath, setRepoPath] = useState("") + + async function handleBuild() { + if (!repoPath) return + await apiFetch("/datasets/build", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ repo_path: repoPath }), + }) + window.location.reload() + } + + return ( +
+

数据集管理

+ +
+ setRepoPath(e.target.value)} + placeholder="输入 Git 仓库本地路径" + className="border rounded px-3 py-2 flex-1" + /> + +
+ + + {samplesState.status === "success" && ( +
+
+ + + + + + + + + + {samplesState.data.map((s) => ( + setSelectedId(s.id)} + > + + + + + ))} + +
仓库语言缺陷数
{s.repo}{s.language}{s.defect_count}
+
+ +
+ {selectedId ? ( + <> + + {detailState.status === "success" && ( +
+

Ground Truth

+ 
+                      {detailState.data.diff}
+
+
    + {detailState.data.defects.map((d) => ( +
  • +

    + {d.defect_type} @ {d.line_start}-{d.line_end} +

    +

    {d.reference_fix}

    +
    • + ))} +
+
+ )} + + ) : ( +

选择左侧样本查看详情

+ )} +
+
+ )} +
+ ) +} diff --git a/frontend/src/pages/ExperimentPage.tsx b/frontend/src/pages/ExperimentPage.tsx new file mode 100644 index 0000000..5ce5751 --- /dev/null +++ b/frontend/src/pages/ExperimentPage.tsx @@ -0,0 +1,196 @@ +import { useEffect, useState } from "react" +import { useApi, apiFetch } from "@/hooks/useApi" +import { Button } from "@/components/ui/button" + +type SampleSummary = { + id: string + repo: string + commit_sha: string + language: string + defect_count: number +} + +type PromptTemplate = { + id: string + strategy_id: string + level: string + version_count: number +} + +type PromptVersion = { + id: string + version_number: number + body: string + variables_schema: Record +} + +export default function ExperimentPage() { + const samplesState = useApi("/datasets/samples") + useApi("/prompts") + const [name, setName] = useState("exp-1") + const [selectedModels, setSelectedModels] = useState(["deepseek"]) + const [selectedLevels, setSelectedLevels] = useState(["L1"]) + const [selectedSamples, setSelectedSamples] = useState([]) + const [repeats, setRepeats] = useState(3) + + const [editStrategy] = useState("code_review") + const [editLevel, setEditLevel] = useState("L1") + const [editBody, setEditBody] = useState("") + const [versions, setVersions] = useState([]) + + const unitCount = selectedModels.length * selectedLevels.length * selectedSamples.length * repeats + + function toggle(list: T[], value: T): T[] { + return list.includes(value) ? list.filter((v) => v !== value) : [...list, value] + } + + async function handleCreate() { + await apiFetch("/experiments", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + name, + models: selectedModels, + levels: selectedLevels, + sample_ids: selectedSamples, + repeats, + }), + }) + alert("实验已创建") + } + + async function loadVersions(strategyId: string, level: string) { + const data = await apiFetch(`/prompts/${strategyId}/${level}/versions`) + setVersions(data) + const latest = [...data].sort((a, b) => b.version_number - a.version_number)[0] + setEditBody(latest?.body ?? "") + } + + useEffect(() => { + loadVersions(editStrategy, editLevel) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []) + + async function handleSaveTemplate() { + await apiFetch("/prompts/versions", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + strategy_id: editStrategy, + level: editLevel, + body: editBody, + }), + }) + await loadVersions(editStrategy, editLevel) + alert("模板新版本已保存") + } + + return ( +
+

实验配置

+ +
+
+

实验参数

+ setName(e.target.value)} + className="border rounded px-3 py-2 w-full" + placeholder="实验名称" + /> +
+

模型

+
+ {["deepseek", "kimi", "qwen"].map((m) => ( + + ))} +
+
+
+

级别

+
+ {["L1", "L2", "L3"].map((l) => ( + + ))} +
+
+
+

样本

+ {samplesState.status === "success" && ( +
+ {samplesState.data.map((s) => ( + + ))} +
+ )} +
+
+

重复次数: {repeats}

+ setRepeats(Number(e.target.value))} + className="w-full" + /> +
+

+ 预计单元数: {unitCount} +

+ +
+ +
+

提示词模板编辑

+
+ + + 当前版本数: {versions.length} + +
+