63 lines
1.7 KiB
Python
63 lines
1.7 KiB
Python
"""Tests for experiment runner."""
|
|
|
|
from unittest import mock
|
|
from uuid import uuid4
|
|
|
|
import pytest
|
|
from sqlalchemy import create_engine
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
from app.db import Base
|
|
from app.experiments.runner import ExperimentRunner
|
|
from app.model_adapters.base import ChatResponse
|
|
from app.models import ExperimentRun, PromptTemplate, PromptTemplateVersion, Result, Sample
|
|
|
|
|
|
def get_test_db():
|
|
engine = create_engine("sqlite:///:memory:")
|
|
Base.metadata.create_all(bind=engine)
|
|
Session = sessionmaker(bind=engine)
|
|
return Session()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_runner_executes_pending_run():
|
|
db = get_test_db()
|
|
template = PromptTemplate(strategy_id="code_review", level="L1")
|
|
version = PromptTemplateVersion(
|
|
template=template, version_number=1, body="", variables_schema={}
|
|
)
|
|
db.add(template)
|
|
db.commit()
|
|
|
|
sample = Sample(repo="r", commit_sha="abc", language="python", diff="d")
|
|
db.add(sample)
|
|
db.commit()
|
|
|
|
run = ExperimentRun(
|
|
id=uuid4(),
|
|
experiment_id=uuid4(),
|
|
sample_id=sample.id,
|
|
model_id="deepseek",
|
|
template_version_id=version.id,
|
|
repeat_index=1,
|
|
status="pending",
|
|
sampling_params={},
|
|
)
|
|
db.add(run)
|
|
db.commit()
|
|
|
|
runner = ExperimentRunner(db)
|
|
with mock.patch("app.experiments.runner.create_adapter") as mock_factory:
|
|
mock_adapter = mock.AsyncMock()
|
|
mock_adapter.chat.return_value = ChatResponse(
|
|
text="- null_pointer", token_usage={}, latency_ms=100.0
|
|
)
|
|
mock_factory.return_value = mock_adapter
|
|
summary = await runner.run_experiment()
|
|
|
|
assert summary["total"] == 1
|
|
db.refresh(run)
|
|
assert run.status == "done"
|
|
assert run.result is not None
|