52 lines
1.4 KiB
Python
52 lines
1.4 KiB
Python
"""Tests for stability metric."""
|
|
|
|
from uuid import uuid4
|
|
|
|
from sqlalchemy import create_engine
|
|
from sqlalchemy.orm import sessionmaker
|
|
|
|
from app.analysis.stability import compute_stability_score
|
|
from app.db import Base
|
|
from app.models import ExperimentRun, PromptTemplate, PromptTemplateVersion, Result, Sample
|
|
|
|
|
|
def get_test_db():
|
|
engine = create_engine("sqlite:///:memory:")
|
|
Base.metadata.create_all(bind=engine)
|
|
Session = sessionmaker(bind=engine)
|
|
return Session()
|
|
|
|
|
|
def test_stability_perfect():
|
|
db = get_test_db()
|
|
template = PromptTemplate(strategy_id="code_review", level="L1")
|
|
version = PromptTemplateVersion(
|
|
template=template, version_number=1, body="", variables_schema={}
|
|
)
|
|
db.add(template)
|
|
db.commit()
|
|
|
|
sample = Sample(repo="r", commit_sha="abc", language="python", diff="d")
|
|
db.add(sample)
|
|
db.commit()
|
|
|
|
experiment_id = uuid4()
|
|
for i in range(3):
|
|
run = ExperimentRun(
|
|
id=uuid4(),
|
|
experiment_id=experiment_id,
|
|
sample_id=sample.id,
|
|
model_id="deepseek",
|
|
template_version_id=version.id,
|
|
repeat_index=i + 1,
|
|
status="done",
|
|
)
|
|
db.add(run)
|
|
db.flush()
|
|
result = Result(run_id=run.id, raw_output="- null_pointer")
|
|
db.add(result)
|
|
db.commit()
|
|
|
|
score = compute_stability_score(db, str(experiment_id), "deepseek", "L1")
|
|
assert score == 1.0
|