first commit

backend/tests/conftest.py

@@ -0,0 +1,19 @@
+"""Shared pytest fixtures."""
+
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from app.db import Base
+
+
+@pytest.fixture
+def db():
+    engine = create_engine("sqlite:///:memory:")
+    Base.metadata.create_all(bind=engine)
+    Session = sessionmaker(bind=engine)
+    session = Session()
+    try:
+        yield session
+    finally:
+        session.close()

backend/tests/test_analysis.py

@@ -0,0 +1,28 @@
+"""Tests for analysis parser and metrics."""
+
+from app.analysis.parser import compare_findings, parse_output
+
+
+def test_parse_l1_output():
+    output = "- null pointer\n- off by one"
+    findings = parse_output(output, "L1")
+    assert len(findings) == 2
+    assert findings[0].defect_type == "null pointer"
+
+
+def test_parse_l2_output():
+    output = """- Type: null_pointer
+  Lines: 5-6
+  Explanation: missing guard"""
+    findings = parse_output(output, "L2")
+    assert len(findings) == 1
+    assert findings[0].line_start == 5
+    assert findings[0].line_end == 6
+
+
+def test_compare_findings():
+    findings = parse_output("- null_pointer", "L1")
+    gt = [{"id": "1", "defect_type": "null_pointer", "line_start": 5, "line_end": 5}]
+    metrics = compare_findings(findings, gt)
+    assert metrics["detection_rate"] == 1.0
+    assert metrics["coverage_rate"] == 1.0

backend/tests/test_analysis_service.py

@@ -0,0 +1,61 @@
+"""Tests for analysis service."""
+
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from app.analysis.service import AnalysisService
+from app.db import Base
+from app.models import Defect, ExperimentRun, PromptTemplate, PromptTemplateVersion, Result, Sample
+
+
+def get_test_db():
+    engine = create_engine("sqlite:///:memory:")
+    Base.metadata.create_all(bind=engine)
+    Session = sessionmaker(bind=engine)
+    return Session()
+
+
+def test_compute_metrics_for_run():
+    db = get_test_db()
+    sample = Sample(repo="r", commit_sha="abc", language="python", diff="diff")
+    defect = Defect(
+        sample=sample,
+        defect_type="null_pointer",
+        language="python",
+        line_start=5,
+        line_end=5,
+    )
+    db.add(sample)
+    db.commit()
+
+    from uuid import uuid4
+
+    template = PromptTemplate(strategy_id="code_review", level="L1")
+    version = PromptTemplateVersion(
+        template=template, version_number=1, body="", variables_schema={}
+    )
+    db.add(template)
+    db.commit()
+
+    run = ExperimentRun(
+        id=uuid4(),
+        experiment_id=uuid4(),
+        sample_id=sample.id,
+        model_id="deepseek",
+        template_version_id=version.id,
+        repeat_index=1,
+        status="done",
+    )
+    db.add(run)
+    db.flush()
+    result = Result(
+        run_id=run.id,
+        raw_output="- null_pointer",
+    )
+    db.add(result)
+    db.commit()
+
+    service = AnalysisService(db)
+    metrics = service.compute_metrics_for_run(str(run.id))
+    assert metrics["detection_rate"] == 1.0

backend/tests/test_api.py

@@ -0,0 +1,42 @@
+"""Tests for FastAPI endpoints."""
+
+import pytest
+from fastapi.testclient import TestClient
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from app.api.main import app
+from app.db import Base, get_db
+
+
+def get_test_db():
+    engine = create_engine("sqlite:///:memory:")
+    Base.metadata.create_all(bind=engine)
+    Session = sessionmaker(bind=engine)
+    db = Session()
+    try:
+        yield db
+    finally:
+        db.close()
+
+
+app.dependency_overrides[get_db] = get_test_db
+client = TestClient(app)
+
+
+def test_health():
+    response = client.get("/health")
+    assert response.status_code == 200
+    assert response.json()["status"] == "ok"
+
+
+def test_list_samples_empty():
+    response = client.get("/api/datasets/samples")
+    assert response.status_code == 200
+    assert response.json() == []
+
+
+def test_list_models():
+    response = client.get("/api/models")
+    assert response.status_code == 200
+    assert "deepseek" in response.json()["models"]

backend/tests/test_charts.py

@@ -0,0 +1,21 @@
+"""Tests for chart generation."""
+
+from app.analysis.charts import boxplot, grouped_bar, heatmap
+
+
+def test_heatmap_returns_base64():
+    data = {"m1": {"L1": 0.5, "L2": 0.8}}
+    b64 = heatmap(data, title="Test")
+    assert b64.startswith("iVBOR") or b64.startswith("/9j")
+
+
+def test_boxplot_returns_base64():
+    data = {"m1-L1": [0.1, 0.2, 0.3]}
+    b64 = boxplot(data, title="Test")
+    assert isinstance(b64, str) and len(b64) > 0
+
+
+def test_grouped_bar_returns_base64():
+    data = {"m1": {"L1": 0.5}}
+    b64 = grouped_bar(data, title="Test")
+    assert isinstance(b64, str) and len(b64) > 0

backend/tests/test_experiments.py

@@ -0,0 +1,46 @@
+"""Tests for experiment matrix and runner."""
+
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from app.db import Base
+from app.experiments.matrix import generate_full_factorial_matrix
+from app.model_adapters.factory import create_adapter, list_models
+from app.prompts.defaults import seed_default_templates
+
+
+def get_test_db():
+    engine = create_engine("sqlite:///:memory:")
+    Base.metadata.create_all(bind=engine)
+    Session = sessionmaker(bind=engine)
+    return Session()
+
+
+def test_list_models():
+    assert "deepseek" in list_models()
+
+
+def test_create_adapter_requires_key():
+    with pytest.raises(ValueError):
+        create_adapter("unknown")
+
+
+def test_generate_matrix():
+    db = get_test_db()
+    seed_default_templates(db)
+    from app.models import Sample
+
+    sample = Sample(repo="test", commit_sha="abc", language="python", diff="diff")
+    db.add(sample)
+    db.commit()
+
+    experiment = generate_full_factorial_matrix(
+        db,
+        name="test-exp",
+        models=["deepseek"],
+        levels=["L1"],
+        sample_ids=[str(sample.id)],
+        repeats=2,
+    )
+    assert len(experiment.runs) == 2

backend/tests/test_git_parser.py

@@ -0,0 +1,40 @@
+"""Tests for Git parser and candidate selection."""
+
+import os
+from pathlib import Path
+
+import pytest
+
+from app.dataset.git_parser import score_commit, select_candidates
+
+
+def _make_repo(tmp_path: Path) -> str:
+    repo = tmp_path / "repo"
+    repo.mkdir()
+    os.system(
+        f'cd {repo} && git init -q && git config user.email "test@example.com" && git config user.name "Test"'
+    )
+    (repo / "main.py").write_text("print('hello')\n")
+    os.system(f'cd {repo} && git add . && git commit -q -m "init"')
+    (repo / "main.py").write_text("print('world')\n")
+    os.system(f'cd {repo} && git add . && git commit -q -m "update main"')
+    (repo / "app.java").write_text("class App {}\n")
+    os.system(f'cd {repo} && git add . && git commit -q -m "add java app"')
+    return str(repo)
+
+
+def test_score_commit_prefers_message_and_size(tmp_path):
+    repo = _make_repo(tmp_path)
+    from app.dataset.git_parser import list_commits
+
+    commits = list_commits(repo)
+    assert len(commits) >= 2
+    scores = [score_commit(c) for c in commits]
+    assert all(0 <= s <= 1 for s in scores)
+
+
+def test_select_candidates(tmp_path):
+    repo = _make_repo(tmp_path)
+    candidates = select_candidates(repo, count=2, languages=["python", "java"])
+    assert len(candidates) <= 2
+    assert all(c.repo == "repo" for c in candidates)

backend/tests/test_model_adapters.py

@@ -0,0 +1,26 @@
+"""Tests for model adapters."""
+
+import pytest
+
+from app.model_adapters.base import ModelAdapter, ChatResponse
+from app.model_adapters.providers import DeepSeekAdapter
+
+
+def test_adapter_payload_and_extraction():
+    adapter = DeepSeekAdapter(api_key="test", model="deepseek-chat", base_url="https://x")
+    payload = adapter._build_payload("hello", {"temperature": 0.5, "max_tokens": 100})
+    assert payload["model"] == "deepseek-chat"
+    assert payload["messages"][0]["content"] == "hello"
+
+    response_text = adapter._extract_text(
+        {"choices": [{"message": {"content": "hi"}}]}
+    )
+    assert response_text == "hi"
+
+
+def test_adapter_token_usage():
+    adapter = DeepSeekAdapter(api_key="test", model="m", base_url="https://x")
+    usage = adapter._extract_token_usage(
+        {"usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}}
+    )
+    assert usage["total_tokens"] == 15

backend/tests/test_prompts.py

@@ -0,0 +1,36 @@
+"""Tests for prompt template service."""
+
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from app.db import Base, init_db
+from app.prompts.defaults import seed_default_templates
+from app.prompts.service import PromptService
+
+
+def get_test_db():
+    engine = create_engine("sqlite:///:memory:")
+    Base.metadata.create_all(bind=engine)
+    Session = sessionmaker(bind=engine)
+    return Session()
+
+
+def test_seed_and_render():
+    db = get_test_db()
+    seed_default_templates(db)
+    service = PromptService(db)
+    rendered = service.render("code_review", None, {"language": "python", "diff": "+x"})
+    assert "python" in rendered
+    assert "+x" in rendered
+
+
+def test_create_version_increments():
+    db = get_test_db()
+    service = PromptService(db)
+    v1 = service.create_version("test", "L1", "hello {{ name }}")
+    v2 = service.create_version("test", "L1", "hello {{ name }} v2")
+    assert v1.version_number == 1
+    assert v2.version_number == 2
+    versions = service.list_versions("test", "L1")
+    assert len(versions) == 2

backend/tests/test_rules.py

@@ -0,0 +1,44 @@
+"""Tests for mutation rules."""
+
+from app.dataset.rules.python.null_pointer import NoneReferenceRule
+from app.dataset.rules.python.boundary_error import BoundaryErrorRule
+from app.dataset.rules.python.logic_operator import LogicOperatorRule
+from app.dataset.rules.registry import get_registry
+
+
+def test_python_none_reference():
+    src = """def process(data):
+    if data is not None:
+        x = data.upper()
+        return x
+"""
+    m = NoneReferenceRule().detect_and_mutate(src)
+    assert m is not None
+    assert m.defect_type == "null_pointer"
+    assert "if data is not None" not in m.mutated_source
+
+
+def test_python_boundary_error():
+    src = """def get(items, i):
+    if i < len(items):
+        return items[i]
+"""
+    m = BoundaryErrorRule().detect_and_mutate(src)
+    assert m is not None
+    assert "<=" in m.mutated_source
+
+
+def test_python_logic_operator():
+    src = """def ok(a, b):
+    return a and b
+"""
+    m = LogicOperatorRule().detect_and_mutate(src)
+    assert m is not None
+    assert " or " in m.mutated_source
+
+
+def test_registry_discovers_rules():
+    registry = get_registry()
+    assert "python" in registry.all_rules()
+    assert "java" in registry.all_rules()
+    assert "javascript" in registry.all_rules()

backend/tests/test_runner.py

@@ -0,0 +1,62 @@
+"""Tests for experiment runner."""
+
+from unittest import mock
+from uuid import uuid4
+
+import pytest
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from app.db import Base
+from app.experiments.runner import ExperimentRunner
+from app.model_adapters.base import ChatResponse
+from app.models import ExperimentRun, PromptTemplate, PromptTemplateVersion, Result, Sample
+
+
+def get_test_db():
+    engine = create_engine("sqlite:///:memory:")
+    Base.metadata.create_all(bind=engine)
+    Session = sessionmaker(bind=engine)
+    return Session()
+
+
+@pytest.mark.asyncio
+async def test_runner_executes_pending_run():
+    db = get_test_db()
+    template = PromptTemplate(strategy_id="code_review", level="L1")
+    version = PromptTemplateVersion(
+        template=template, version_number=1, body="", variables_schema={}
+    )
+    db.add(template)
+    db.commit()
+
+    sample = Sample(repo="r", commit_sha="abc", language="python", diff="d")
+    db.add(sample)
+    db.commit()
+
+    run = ExperimentRun(
+        id=uuid4(),
+        experiment_id=uuid4(),
+        sample_id=sample.id,
+        model_id="deepseek",
+        template_version_id=version.id,
+        repeat_index=1,
+        status="pending",
+        sampling_params={},
+    )
+    db.add(run)
+    db.commit()
+
+    runner = ExperimentRunner(db)
+    with mock.patch("app.experiments.runner.create_adapter") as mock_factory:
+        mock_adapter = mock.AsyncMock()
+        mock_adapter.chat.return_value = ChatResponse(
+            text="- null_pointer", token_usage={}, latency_ms=100.0
+        )
+        mock_factory.return_value = mock_adapter
+        summary = await runner.run_experiment()
+
+    assert summary["total"] == 1
+    db.refresh(run)
+    assert run.status == "done"
+    assert run.result is not None

backend/tests/test_stability.py

@@ -0,0 +1,51 @@
+"""Tests for stability metric."""
+
+from uuid import uuid4
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import sessionmaker
+
+from app.analysis.stability import compute_stability_score
+from app.db import Base
+from app.models import ExperimentRun, PromptTemplate, PromptTemplateVersion, Result, Sample
+
+
+def get_test_db():
+    engine = create_engine("sqlite:///:memory:")
+    Base.metadata.create_all(bind=engine)
+    Session = sessionmaker(bind=engine)
+    return Session()
+
+
+def test_stability_perfect():
+    db = get_test_db()
+    template = PromptTemplate(strategy_id="code_review", level="L1")
+    version = PromptTemplateVersion(
+        template=template, version_number=1, body="", variables_schema={}
+    )
+    db.add(template)
+    db.commit()
+
+    sample = Sample(repo="r", commit_sha="abc", language="python", diff="d")
+    db.add(sample)
+    db.commit()
+
+    experiment_id = uuid4()
+    for i in range(3):
+        run = ExperimentRun(
+            id=uuid4(),
+            experiment_id=experiment_id,
+            sample_id=sample.id,
+            model_id="deepseek",
+            template_version_id=version.id,
+            repeat_index=i + 1,
+            status="done",
+        )
+        db.add(run)
+        db.flush()
+        result = Result(run_id=run.id, raw_output="- null_pointer")
+        db.add(result)
+    db.commit()
+
+    score = compute_stability_score(db, str(experiment_id), "deepseek", "L1")
+    assert score == 1.0