long/ai-safety-platform
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first commit

Browse Source
This commit is contained in:
baozaotumao2025
2026-07-18 21:00:26 +08:00
commit 14722be770
105 changed files with 25004 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_api_contracts.py
+753
View File
@@ -0,0 +1,753 @@
import pytest
from fastapi.testclient import TestClient
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from sqlalchemy.pool import StaticPool
from app.api.deps import get_database, get_provider_factory
from app.api.v1.endpoints import runs
from app.db.base import Base
from app.db.repositories.repositories import (
ResultRepository,
TestRunRepository as RunRepository,
UserRepository,
)
from app.main import app
from app.providers.model_provider import ChatResponse
from app.services.auth_service import token_digest
class FakeProvider:
endpoint = "https://example.test/v1/chat/completions"
model = "fake-model"
async def list_models(self):
return ["fake-model"]
async def chat(self, messages, tools=None):
content = messages[-1]["content"]
if tools:
return ChatResponse(content="", tool_calls=[{"id": "call-1"}], raw={})
if isinstance(content, list):
return ChatResponse(content="蓝色圆形", tool_calls=[], raw={})
if "TEXT_OK" in content:
return ChatResponse(content="TEXT_OK", tool_calls=[], raw={})
if "SYSTEM_OK" in messages[0]["content"]:
return ChatResponse(content="SYSTEM_OK", tool_calls=[], raw={})
if "编号是什么" in content:
return ChatResponse(content="4837", tool_calls=[], raw={})
if "status" in content:
return ChatResponse(content='{"status":"ok"}', tool_calls=[], raw={})
return ChatResponse(content="OK", tool_calls=[], raw={})
class FakeFactory:
def create(self, _):
return FakeProvider()
@pytest.fixture
def client(monkeypatch):
engine = create_engine(
"sqlite://", connect_args={"check_same_thread": False}, poolclass=StaticPool
)
Base.metadata.create_all(engine)
session = Session(engine)
async def skip_background(*_):
return None
app.dependency_overrides[get_database] = lambda: session
app.dependency_overrides[get_provider_factory] = lambda: FakeFactory()
app.state.session_factory = lambda: session
UserRepository(session).create(
username="auditor", password="correct-horse-battery-staple", is_admin=True
)
monkeypatch.setattr(runs, "execute_run", skip_background)
try:
yield TestClient(app)
finally:
app.dependency_overrides.clear()
del app.state.session_factory
session.close()
def test_health_contract(client):
response = client.get("/api/v1/health")
assert response.status_code == 200
assert response.json()["status"] == "ok"
def test_authentication_protects_api_and_login_issues_bearer_token(client):
unauthenticated = client.get("/api/v1/providers/target/models")
assert unauthenticated.status_code == 401
assert unauthenticated.headers["WWW-Authenticate"] == "Bearer"
assert unauthenticated.headers["X-Request-ID"]
assert client.get("/api/v1/health").status_code == 200
invalid = client.post(
"/api/v1/auth/login", json={"username": "auditor", "password": "wrong-password"}
)
assert invalid.status_code == 401
assert invalid.json()["error"]["code"] == "UNAUTHORIZED"
login = client.post(
"/api/v1/auth/login",
json={"username": "auditor", "password": "correct-horse-battery-staple"},
)
assert login.status_code == 200
token = login.json()["access_token"]
assert login.json()["token_type"] == "bearer"
assert login.json()["refresh_token"]
assert login.json()["refresh_expires_at"] > login.json()["expires_at"]
allowed = client.get(
"/api/v1/providers/target/models", headers={"Authorization": f"Bearer {token}"}
)
assert allowed.status_code == 200
def test_refresh_rotates_token_and_idempotently_replays_the_same_response(client, monkeypatch):
now = 1_000_000
monkeypatch.setattr("app.api.v1.endpoints.auth.time.time", lambda: now)
login = client.post(
"/api/v1/auth/login",
json={"username": "auditor", "password": "correct-horse-battery-staple"},
).json()
assert login["expires_at"] == now + 3600
assert login["refresh_expires_at"] == now + 28800
request = {"refresh_token": login["refresh_token"]}
headers = {"Idempotency-Key": "refresh-attempt-1"}
now += 1800
first = client.post("/api/v1/auth/refresh", json=request, headers=headers)
replay = client.post("/api/v1/auth/refresh", json=request, headers=headers)
assert first.status_code == replay.status_code == 200
assert first.json() == replay.json()
assert first.json()["expires_at"] == now + 3600
assert first.json()["refresh_expires_at"] == now + 28800
assert first.json()["refresh_expires_at"] > login["refresh_expires_at"]
assert first.json()["refresh_token"] != login["refresh_token"]
assert (
client.get(
"/api/v1/auth/me",
headers={"Authorization": f"Bearer {first.json()['access_token']}"},
).status_code
== 200
)
def test_refresh_requires_idempotency_key_and_replay_revokes_the_session(client, monkeypatch):
now = 1_000_000
monkeypatch.setattr("app.api.v1.endpoints.auth.time.time", lambda: now)
login = client.post(
"/api/v1/auth/login",
json={"username": "auditor", "password": "correct-horse-battery-staple"},
).json()
request = {"refresh_token": login["refresh_token"]}
assert client.post("/api/v1/auth/refresh", json=request).status_code == 422
now += 1800
rotated = client.post(
"/api/v1/auth/refresh",
json=request,
headers={"Idempotency-Key": "refresh-attempt-1"},
).json()
replay_attack = client.post(
"/api/v1/auth/refresh",
json=request,
headers={"Idempotency-Key": "different-attempt"},
)
assert replay_attack.status_code == 401
assert (
client.post(
"/api/v1/auth/refresh",
json={"refresh_token": rotated["refresh_token"]},
headers={"Idempotency-Key": "refresh-attempt-2"},
).status_code
== 401
)
assert (
client.get(
"/api/v1/auth/me",
headers={"Authorization": f"Bearer {rotated['access_token']}"},
).status_code
== 401
)
def test_refresh_is_rejected_before_window_without_consuming_token(client, monkeypatch):
now = 1_000_000
monkeypatch.setattr("app.api.v1.endpoints.auth.time.time", lambda: now)
login = client.post(
"/api/v1/auth/login",
json={"username": "auditor", "password": "correct-horse-battery-staple"},
).json()
request = {"refresh_token": login["refresh_token"]}
early = client.post(
"/api/v1/auth/refresh",
json=request,
headers={"Idempotency-Key": "early-attempt"},
)
now += 1800
due = client.post(
"/api/v1/auth/refresh",
json=request,
headers={"Idempotency-Key": "due-attempt"},
)
assert early.status_code == 409
assert early.json()["error"] == {
"code": "REFRESH_NOT_DUE",
"message": "访问令牌尚未进入续约窗口",
"details": {"refresh_after": 1_001_800},
"request_id": early.headers["X-Request-ID"],
}
assert due.status_code == 200
def test_refresh_slides_session_expiry_past_original_login_deadline(client, monkeypatch):
now = 1_000_000
monkeypatch.setattr("app.api.v1.endpoints.auth.time.time", lambda: now)
login = client.post(
"/api/v1/auth/login",
json={"username": "auditor", "password": "correct-horse-battery-staple"},
).json()
original_deadline = login["refresh_expires_at"]
now += 1800
first = client.post(
"/api/v1/auth/refresh",
json={"refresh_token": login["refresh_token"]},
headers={"Idempotency-Key": "first-renewal"},
).json()
now = original_deadline + 1
second = client.post(
"/api/v1/auth/refresh",
json={"refresh_token": first["refresh_token"]},
headers={"Idempotency-Key": "past-original-deadline"},
)
assert second.status_code == 200
assert second.json()["expires_at"] == now + 3600
assert second.json()["refresh_expires_at"] == now + 28800
assert second.json()["refresh_expires_at"] > original_deadline
def test_logout_revokes_refresh_token_family(client):
login = client.post(
"/api/v1/auth/login",
json={"username": "auditor", "password": "correct-horse-battery-staple"},
).json()
headers = {"Authorization": f"Bearer {login['access_token']}"}
assert client.post("/api/v1/auth/logout", headers=headers).status_code == 204
assert (
client.post(
"/api/v1/auth/refresh",
json={"refresh_token": login["refresh_token"]},
headers={"Idempotency-Key": "after-logout"},
).status_code
== 401
)
def test_expired_refresh_token_is_rejected(client):
login = client.post(
"/api/v1/auth/login",
json={"username": "auditor", "password": "correct-horse-battery-staple"},
).json()
db = app.state.session_factory()
row = UserRepository(db).get_refresh_token(token_digest(login["refresh_token"]))
row.expires_at = 0
db.commit()
assert (
client.post(
"/api/v1/auth/refresh",
json={"refresh_token": login["refresh_token"]},
headers={"Idempotency-Key": "expired-token"},
).status_code
== 401
)
@pytest.mark.parametrize(
("method", "path"),
[
("get", "/api/v1/auth/me"),
("patch", "/api/v1/auth/password"),
("post", "/api/v1/providers/target/check"),
("patch", "/api/v1/auth/users/1/password"),
("get", "/api/v1/providers/target/models"),
("post", "/api/v1/runs"),
("get", "/api/v1/runs"),
("patch", "/api/v1/runs/1"),
("delete", "/api/v1/runs/1"),
("post", "/api/v1/runs/1/resume"),
("post", "/api/v1/runs/1/retry-errors"),
("post", "/api/v1/runs/1/results/R0001/retry"),
("get", "/api/v1/runs/1"),
("get", "/api/v1/runs/1/results"),
("get", "/api/v1/reports"),
("get", "/api/v1/reports/1"),
],
)
def test_every_business_api_requires_authentication(client, method, path):
assert getattr(client, method)(path).status_code == 401
def test_admin_can_manage_provider_configs(client, auth_headers):
inherited = client.get("/api/v1/providers", headers=auth_headers).json()
assert {item["provider_id"] for item in inherited} == {"target", "judge"}
assert all(item["source"] == "env" for item in inherited)
payload = {
"provider_id": "target",
"base_url": "https://models.example/v1",
"chat_path": "/chat/completions",
"models_path": "/models",
"model_name": "safe-model",
"auth_type": "bearer",
"api_key": "secret-value",
"auth_header": "Authorization",
"auth_prefix": "Bearer",
}
created = client.post("/api/v1/providers", headers=auth_headers, json=payload)
assert created.status_code == 201
assert created.json()["api_key_configured"] is True
assert "api_key" not in created.json()
provider_id = created.json()["provider_id"]
assert client.post("/api/v1/providers", headers=auth_headers, json=payload).status_code == 409
effective = client.get("/api/v1/providers", headers=auth_headers).json()
target = next(item for item in effective if item["provider_id"] == "target")
assert target["source"] == "database"
assert client.get(f"/api/v1/providers/{provider_id}", headers=auth_headers).status_code == 200
updated = client.patch(
f"/api/v1/providers/{provider_id}",
headers=auth_headers,
json={"model_name": "safe-model-v2"},
)
assert updated.status_code == 200
assert updated.json()["model_name"] == "safe-model-v2"
assert updated.json()["api_key_configured"] is True
assert (
client.delete(f"/api/v1/providers/{provider_id}", headers=auth_headers).status_code == 204
)
fallback = client.get(f"/api/v1/providers/{provider_id}", headers=auth_headers)
assert fallback.status_code == 200
assert fallback.json()["source"] == "env"
def test_provider_writes_require_admin(client, auth_headers):
created = client.post(
"/api/v1/auth/users",
headers=auth_headers,
json={"username": "operator", "password": "maxta2026", "is_admin": False},
)
assert created.status_code == 201
token = client.post(
"/api/v1/auth/login", json={"username": "operator", "password": "maxta2026"}
).json()["access_token"]
response = client.post(
"/api/v1/providers",
headers={"Authorization": f"Bearer {token}"},
json={
"provider_id": "judge",
"base_url": "https://models.example/v1",
"model_name": "judge-model",
"auth_type": "none",
},
)
assert response.status_code == 403
def test_admin_can_manage_users(client, auth_headers):
created = client.post(
"/api/v1/auth/users",
headers=auth_headers,
json={"username": "operator", "password": "maxta2026", "is_admin": False},
)
assert created.status_code == 201
user_id = created.json()["id"]
assert created.json()["username"] == "operator"
listed = client.get("/api/v1/auth/users", headers=auth_headers)
assert listed.status_code == 200
assert {user["username"] for user in listed.json()} == {"auditor", "operator"}
auditor_id = next(user["id"] for user in listed.json() if user["username"] == "auditor")
operator_login = client.post(
"/api/v1/auth/login",
json={"username": "operator", "password": "maxta2026"},
)
assert operator_login.status_code == 200
assert (
client.get(
"/api/v1/auth/users",
headers={"Authorization": f"Bearer {operator_login.json()['access_token']}"},
).status_code
== 403
)
disabled = client.patch(
f"/api/v1/auth/users/{user_id}", headers=auth_headers, json={"is_active": False}
)
assert disabled.status_code == 200
assert disabled.json()["is_active"] is False
assert client.delete(f"/api/v1/auth/users/{user_id}", headers=auth_headers).status_code == 204
assert client.delete(f"/api/v1/auth/users/{user_id}", headers=auth_headers).status_code == 404
assert "operator" not in {
user["username"] for user in client.get("/api/v1/auth/users", headers=auth_headers).json()
}
assert (
client.post(
"/api/v1/auth/login",
json={"username": "operator", "password": "maxta2026"},
).status_code
== 401
)
assert (
client.delete(f"/api/v1/auth/users/{auditor_id}", headers=auth_headers).status_code == 409
)
def test_authenticated_user_can_read_own_profile(client, auth_headers):
response = client.get("/api/v1/auth/me", headers=auth_headers)
assert response.status_code == 200
assert response.json() == {
"id": 1,
"username": "auditor",
"is_active": True,
"is_admin": True,
}
def test_user_can_change_own_password_and_existing_tokens_are_revoked(client, auth_headers):
client.post(
"/api/v1/auth/users",
headers=auth_headers,
json={"username": "operator", "password": "old-password", "is_admin": False},
)
old_login = client.post(
"/api/v1/auth/login", json={"username": "operator", "password": "old-password"}
).json()
old_token = old_login["access_token"]
headers = {"Authorization": f"Bearer {old_token}"}
rejected = client.patch(
"/api/v1/auth/password",
headers=headers,
json={"current_password": "wrong-password", "new_password": "new-password"},
)
assert rejected.status_code == 401
assert rejected.json()["error"]["code"] == "UNAUTHORIZED"
changed = client.patch(
"/api/v1/auth/password",
headers=headers,
json={"current_password": "old-password", "new_password": "new-password"},
)
assert changed.status_code == 204
assert client.get("/api/v1/auth/me", headers=headers).status_code == 401
assert (
client.post(
"/api/v1/auth/refresh",
json={"refresh_token": old_login["refresh_token"]},
headers={"Idempotency-Key": "after-password-change"},
).status_code
== 401
)
assert (
client.post(
"/api/v1/auth/login", json={"username": "operator", "password": "old-password"}
).status_code
== 401
)
assert (
client.post(
"/api/v1/auth/login", json={"username": "operator", "password": "new-password"}
).status_code
== 200
)
def test_admin_can_reset_password_and_existing_tokens_are_revoked(client, auth_headers):
created = client.post(
"/api/v1/auth/users",
headers=auth_headers,
json={"username": "operator", "password": "old-password", "is_admin": False},
)
user_id = created.json()["id"]
old_token = client.post(
"/api/v1/auth/login", json={"username": "operator", "password": "old-password"}
).json()["access_token"]
reset = client.patch(
f"/api/v1/auth/users/{user_id}/password",
headers=auth_headers,
json={"password": "new-password"},
)
assert reset.status_code == 204
assert (
client.post(
"/api/v1/auth/login", json={"username": "operator", "password": "old-password"}
).status_code
== 401
)
assert (
client.post(
"/api/v1/auth/login", json={"username": "operator", "password": "new-password"}
).status_code
== 200
)
assert (
client.get(
"/api/v1/providers", headers={"Authorization": f"Bearer {old_token}"}
).status_code
== 401
)
def test_reset_password_requires_admin(client, auth_headers):
created = client.post(
"/api/v1/auth/users",
headers=auth_headers,
json={"username": "operator", "password": "old-password", "is_admin": False},
)
token = client.post(
"/api/v1/auth/login", json={"username": "operator", "password": "old-password"}
).json()["access_token"]
assert (
client.patch(
f"/api/v1/auth/users/{created.json()['id']}/password",
headers={"Authorization": f"Bearer {token}"},
json={"password": "new-password"},
).status_code
== 403
)
def test_logout_revokes_current_token(client, auth_headers):
assert client.post("/api/v1/auth/logout", headers=auth_headers).status_code == 204
assert client.get("/api/v1/providers/target/models", headers=auth_headers).status_code == 401
@pytest.fixture
def auth_headers(client):
token = client.post(
"/api/v1/auth/login",
json={"username": "auditor", "password": "correct-horse-battery-staple"},
).json()["access_token"]
return {"Authorization": f"Bearer {token}"}
def test_provider_contracts(client, auth_headers):
check = client.post("/api/v1/providers/target/check", headers=auth_headers)
models = client.get("/api/v1/providers/judge/models", headers=auth_headers)
assert check.status_code == models.status_code == 200
assert check.json()["model"] == "fake-model"
assert models.json()["models"] == ["fake-model"]
def test_run_and_report_contracts(client, auth_headers):
created = client.post(
"/api/v1/runs",
json={"profile": "smoke", "auto_judge": False},
headers={**auth_headers, "Idempotency-Key": "api-contract-run"},
)
run_id = created.json()["run_id"]
status = client.get(f"/api/v1/runs/{run_id}", headers=auth_headers)
listed = client.get("/api/v1/runs", headers=auth_headers)
results = client.get(f"/api/v1/runs/{run_id}/results", headers=auth_headers)
reports = client.get("/api/v1/reports", headers=auth_headers)
report = client.get(f"/api/v1/reports/{run_id}", headers=auth_headers)
resumed = client.post(f"/api/v1/runs/{run_id}/resume", headers=auth_headers)
assert created.status_code == resumed.status_code == 202
assert (
status.status_code
== listed.status_code
== results.status_code
== reports.status_code
== report.status_code
== 200
)
assert listed.json()[0]["run_id"] == run_id
assert results.json() == []
assert reports.json()[0]["run_id"] == run_id
assert report.json()["run_id"] == run_id
def test_reports_are_derived_read_only_resources(client, auth_headers):
assert client.get("/api/v1/reports/999", headers=auth_headers).status_code == 404
assert client.post("/api/v1/reports", headers=auth_headers).status_code == 405
assert client.patch("/api/v1/reports/1", headers=auth_headers).status_code == 405
assert client.delete("/api/v1/reports/1", headers=auth_headers).status_code == 405
def test_run_cancel_and_delete_contract(client, auth_headers):
created = client.post(
"/api/v1/runs",
json={"profile": "smoke", "auto_judge": False},
headers=auth_headers,
)
run_id = created.json()["run_id"]
assert client.delete(f"/api/v1/runs/{run_id}", headers=auth_headers).status_code == 409
cancelled = client.patch(
f"/api/v1/runs/{run_id}",
json={"status": "cancelled"},
headers=auth_headers,
)
assert cancelled.status_code == 200
assert cancelled.json()["status"] == "cancelled"
assert cancelled.json()["terminal"] is True
assert (
client.patch(
f"/api/v1/runs/{run_id}",
json={"status": "cancelled"},
headers=auth_headers,
).status_code
== 409
)
assert client.delete(f"/api/v1/runs/{run_id}", headers=auth_headers).status_code == 204
assert client.delete(f"/api/v1/runs/{run_id}", headers=auth_headers).status_code == 404
assert client.get(f"/api/v1/runs/{run_id}", headers=auth_headers).status_code == 404
def test_retry_errors_keeps_successes_and_requeues_only_errors(client, auth_headers):
created = client.post(
"/api/v1/runs",
json={"profile": "smoke", "auto_judge": False},
headers=auth_headers,
)
run_id = created.json()["run_id"]
db = app.state.session_factory()
run_repo = RunRepository(db)
run = run_repo.get(run_id)
run.selected_count = 2
run.completed_count = 1
run.error_count = 1
run_repo.update_status(
run,
"completed_with_errors",
summary={"phase": "finished", "auto_judge": False},
)
for execution_id, execution_status in (("case-ok", "completed"), ("case-error", "error")):
ResultRepository(db).add(
run_id=run_id,
execution_id=execution_id,
case_kind="control",
interaction_mode="single_turn",
execution_status=execution_status,
verdict=None,
model_response="ok" if execution_status == "completed" else "",
judge_result_json="{}",
error_message="boom" if execution_status == "error" else "",
)
assert client.post(f"/api/v1/runs/{run_id}/resume", headers=auth_headers).status_code == 409
retried = client.post(f"/api/v1/runs/{run_id}/retry-errors", headers=auth_headers)
assert retried.status_code == 202
assert retried.json()["retry_count"] == 1
assert [row.execution_status for row in ResultRepository(db).list_by_run(run_id)] == [
"completed"
]
run = RunRepository(db).get(run_id)
assert (run.status, run.completed_count, run.error_count) == ("pending", 1, 0)
assert (
client.post(f"/api/v1/runs/{run_id}/retry-errors", headers=auth_headers).status_code == 409
)
run = RunRepository(db).get(run_id)
RunRepository(db).update_status(
run,
"completed_with_errors",
summary={"phase": "finished", "auto_judge": False},
)
assert (
client.post(f"/api/v1/runs/{run_id}/retry-errors", headers=auth_headers).status_code == 409
)
def test_retry_one_result_requeues_only_selected_execution(client, auth_headers):
created = client.post(
"/api/v1/runs",
json={"profile": "smoke", "auto_judge": False},
headers=auth_headers,
)
run_id = created.json()["run_id"]
db = app.state.session_factory()
run_repo = RunRepository(db)
run = run_repo.get(run_id)
run.selected_count = run.completed_count = 2
run_repo.update_status(
run,
"completed",
summary={"phase": "finished", "auto_judge": False},
)
for execution_id in ("case-keep", "case-retry"):
ResultRepository(db).add(
run_id=run_id,
execution_id=execution_id,
case_kind="control",
interaction_mode="single_turn",
execution_status="completed",
verdict="pass",
model_response="ok",
judge_result_json='{"verdict":"pass"}',
error_message="",
)
assert (
client.post(
f"/api/v1/runs/{run_id}/results/missing/retry",
headers=auth_headers,
).status_code
== 404
)
retried = client.post(
f"/api/v1/runs/{run_id}/results/case-retry/retry",
headers=auth_headers,
)
assert retried.status_code == 202
assert retried.json() == {
"run_id": run_id,
"status": "pending",
"selected_count": 2,
"execution_id": "case-retry",
}
assert [row.execution_id for row in ResultRepository(db).list_by_run(run_id)] == ["case-keep"]
run = run_repo.get(run_id)
assert (run.status, run.completed_count, run.error_count) == ("pending", 1, 0)
assert (
client.post(
f"/api/v1/runs/{run_id}/results/case-retry/retry",
headers=auth_headers,
).status_code
== 409
)
def test_run_contract_errors(client, auth_headers):
assert client.get("/api/v1/runs/999", headers=auth_headers).status_code == 404
assert client.get("/api/v1/runs/999/results", headers=auth_headers).status_code == 404
assert (
client.post("/api/v1/runs/999/results/R0001/retry", headers=auth_headers).status_code == 404
)
assert (
client.post("/api/v1/runs", json={"profile": "invalid"}, headers=auth_headers).status_code
== 422
)
tests/test_auth.py
+91
View File
@@ -0,0 +1,91 @@
import pytest
from app.core.config import Settings
from app.services.auth_service import (
derive_refresh_token,
hash_password,
issue_refresh_token,
issue_token,
verify_password,
verify_token,
)
def settings(**overrides):
return Settings(
target_base_url="https://target.test",
judge_base_url="https://judge.test",
**overrides,
)
def test_password_hash_is_salted_and_verifiable():
encoded = hash_password("correct-horse-battery-staple")
assert encoded.startswith("pbkdf2_sha256$600000$")
assert "correct-horse-battery-staple" not in encoded
assert verify_password("correct-horse-battery-staple", encoded)
assert not verify_password("wrong", encoded)
assert not verify_password("wrong", "not-a-password-hash")
def test_signed_token_rejects_tampering_and_requires_secret_outside_tests():
token, expires_at = issue_token(7, settings(auth_token_secret="x" * 32))
assert expires_at > 0
assert verify_token(token, settings(auth_token_secret="x" * 32)) == 7
assert verify_token(token + "x", settings(auth_token_secret="x" * 32)) is None
with pytest.raises(ValueError):
issue_token(7, settings(app_env="production", auth_token_secret=""))
with pytest.raises(ValueError):
issue_token(7, settings(app_env="production", auth_token_secret="too-short"))
def test_refresh_tokens_are_random_but_idempotent_rotation_is_stable():
config = settings(auth_token_secret="x" * 32)
first, second = issue_refresh_token(), issue_refresh_token()
assert first != second
assert derive_refresh_token(first, "attempt-1", config) == derive_refresh_token(
first, "attempt-1", config
)
assert derive_refresh_token(first, "attempt-1", config) != derive_refresh_token(
first, "attempt-2", config
)
def test_session_timing_configuration_requires_refresh_window_before_expiry():
assert settings().auth_token_ttl_seconds == 3600
assert settings().auth_token_refresh_window_seconds == 1800
assert settings().auth_refresh_token_ttl_seconds == 28800
with pytest.raises(ValueError):
settings(auth_token_ttl_seconds=1800, auth_token_refresh_window_seconds=1800)
def test_create_user_cli_prompts_and_never_accepts_password_argument(monkeypatch):
from app import create_user
created = {}
class FakeDatabase:
def __enter__(self):
return self
def __exit__(self, *_):
return None
class FakeRepository:
def __init__(self, _):
pass
def create(self, *, username, password, is_admin):
created.update(username=username, password=password, is_admin=is_admin)
monkeypatch.setattr(create_user, "SessionLocal", FakeDatabase)
monkeypatch.setattr(create_user, "UserRepository", FakeRepository)
monkeypatch.setattr(create_user.getpass, "getpass", lambda _: "correct-horse-battery-staple")
monkeypatch.setattr("sys.argv", ["create_user", "auditor"])
create_user.main()
assert created == {
"username": "auditor",
"password": "correct-horse-battery-staple",
"is_admin": True,
}
tests/test_export_run.py
+26
View File
@@ -0,0 +1,26 @@
from pathlib import Path
from runpy import run_path
render_markdown = run_path(Path(__file__).parents[1] / "scripts/export_run.py")["render_markdown"]
def test_render_markdown_contains_input_output_and_judgement():
text = render_markdown(
{"run_id": 4, "status": "completed", "terminal": True},
[
{
"execution_id": "R0001",
"case_kind": "risk",
"interaction_mode": "single_turn",
"execution_status": "completed",
"verdict": "fail",
"model_input": {"messages": [{"role": "user", "content": "测试输入"}]},
"model_response": "模型输出",
"judge_result": {"verdict": "fail", "score": 0.9, "reason": "仲裁理由"},
"error_message": "",
}
],
)
assert all(value in text for value in ["测试输入", "模型输出", "fail", "0.9", "仲裁理由"])
tests/test_health.py
+117
View File
@@ -0,0 +1,117 @@
from fastapi.testclient import TestClient
from app.main import app
def test_health():
client = TestClient(app)
response = client.get("/api/v1/health")
assert response.status_code == 200
assert response.json()["status"] == "ok"
def test_openapi_available():
client = TestClient(app)
response = client.get("/openapi.json")
assert response.status_code == 200
assert "/api/v1/health" in response.json()["paths"]
assert "/api/v1/test-plans" not in response.json()["paths"]
def test_openapi_declares_bearer_authorization_for_protected_operations():
schema = TestClient(app).get("/openapi.json").json()
bearer = schema["components"]["securitySchemes"]["BearerAuth"]
assert bearer == {"type": "http", "scheme": "bearer"}
assert schema["paths"]["/api/v1/providers/{provider_id}/check"]["post"]["security"] == [
{"BearerAuth": []}
]
assert schema["paths"]["/api/v1/auth/users"]["get"]["security"] == [{"BearerAuth": []}]
assert schema["paths"]["/api/v1/auth/users/{user_id}"]["delete"]["security"] == [
{"BearerAuth": []}
]
assert schema["paths"]["/api/v1/auth/me"]["get"]["security"] == [{"BearerAuth": []}]
assert schema["paths"]["/api/v1/auth/password"]["patch"]["security"] == [{"BearerAuth": []}]
assert "security" not in schema["paths"]["/api/v1/health"]["get"]
assert "security" not in schema["paths"]["/api/v1/auth/login"]["post"]
def test_self_service_auth_openapi_is_documented():
schema = TestClient(app).get("/openapi.json").json()
assert "is_admin" in schema["components"]["schemas"]["UserResponse"]["properties"]
change = schema["components"]["schemas"]["ChangePasswordRequest"]
assert set(change["required"]) == {"current_password", "new_password"}
assert "重新登录" in schema["paths"]["/api/v1/auth/password"]["patch"]["description"]
def test_provider_check_openapi_is_documented():
schema = TestClient(app).get("/openapi.json").json()
operation = schema["paths"]["/api/v1/providers/{provider_id}/check"]["post"]
assert "/api/v1/providers/check" not in schema["paths"]
assert "模型提供商" in operation["summary"]
assert (
operation["responses"]["502"]["content"]["application/json"]["schema"]["$ref"]
== "#/components/schemas/ErrorResponse"
)
provider = operation["parameters"][0]["schema"]
assert provider["enum"] == ["target", "judge"]
def test_capability_probe_is_internal_to_runs():
schema = TestClient(app).get("/openapi.json").json()
assert "/api/v1/capabilities/probe" not in schema["paths"]
assert "能力探测" in schema["paths"]["/api/v1/runs"]["post"]["description"]
assert {"get", "post"} <= set(schema["paths"]["/api/v1/runs"])
assert {"get", "patch", "delete"} <= set(schema["paths"]["/api/v1/runs/{run_id}"])
assert "/api/v1/runs/{run_id}/retry-errors" in schema["paths"]
assert "/api/v1/runs/{run_id}/results/{execution_id}/retry" in schema["paths"]
def test_start_run_openapi_explains_usage():
schema = TestClient(app).get("/openapi.json").json()
operation = schema["paths"]["/api/v1/runs"]["post"]
response = schema["components"]["schemas"]["RunResponse"]
assert '"profile": "smoke"' in operation["description"]
assert "单轮、多轮、工具和图片各选取 1 条" in operation["description"]
assert (
"单轮、多轮、工具和图片各执行 1 条"
in schema["components"]["schemas"]["StartRunRequest"]["properties"]["profile"][
"description"
]
)
assert "completed_with_errors" in operation["description"]
assert "REQUEST_RETRY_COUNT" in operation["description"]
assert (
"completed_count + error_count"
in schema["paths"]["/api/v1/runs/{run_id}"]["get"]["description"]
)
assert "terminal" in schema["paths"]["/api/v1/runs/{run_id}"]["get"]["description"]
assert (
"needs_human_review"
in schema["paths"]["/api/v1/runs/{run_id}/results"]["get"]["description"]
)
assert "model_input" in schema["components"]["schemas"]["ResultItem"]["properties"]
result_properties = schema["components"]["schemas"]["ResultItem"]["properties"]
assert {"execution_status", "verdict"} <= set(result_properties)
assert "status" not in result_properties
assert "/runs/{run_id}/results" in operation["description"]
assert response["examples"][0]["run_id"] == 42
assert response["examples"][0]["status"] == "pending"
assert "202" in operation["responses"]
assert "Idempotency-Key" in operation["description"]
assert "409" in operation["responses"]
resume = schema["paths"]["/api/v1/runs/{run_id}/resume"]["post"]
assert "不会再次请求模型" in resume["description"]
assert "skipped_count" in schema["components"]["schemas"]["ResumeRunResponse"]["properties"]
def test_reports_openapi_declares_derived_read_only_resource():
schema = TestClient(app).get("/openapi.json").json()
assert set(schema["paths"]["/api/v1/reports"]) == {"get"}
assert set(schema["paths"]["/api/v1/reports/{run_id}"]) == {"get"}
assert "实时计算" in schema["paths"]["/api/v1/reports"]["get"]["description"]
assert "404" in schema["paths"]["/api/v1/reports/{run_id}"]["get"]["responses"]
tests/test_init_db.py
+26
View File
@@ -0,0 +1,26 @@
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session
from app.core.config import get_settings
from app.db.models import User
from app.services.auth_service import verify_password
def test_initialize_fresh_database_creates_default_admin(monkeypatch, tmp_path):
database = tmp_path / "platform.db"
monkeypatch.setenv("DATABASE_URL", f"sqlite:///{database}")
get_settings.cache_clear()
try:
from app.init_db import initialize
initialize()
initialize()
with Session(create_engine(f"sqlite:///{database}")) as db:
users = list(db.scalars(select(User)))
assert len(users) == 1
assert users[0].username == "gly"
assert users[0].is_admin is True
assert verify_password("maxta2026", users[0].password_hash)
finally:
get_settings.cache_clear()
tests/test_migrations.py
+89
View File
@@ -0,0 +1,89 @@
from alembic import command
from alembic.config import Config
from sqlalchemy import create_engine, inspect, text
from app.core.config import get_settings
def test_upgrade_from_0005_preserves_duplicate_model_profiles(monkeypatch, tmp_path):
database = tmp_path / "existing.db"
monkeypatch.setenv("DATABASE_URL", f"sqlite:///{database}")
get_settings.cache_clear()
config = Config("alembic.ini")
try:
command.upgrade(config, "0005")
engine = create_engine(f"sqlite:///{database}")
with engine.begin() as connection:
for profile_id in (1, 2):
connection.execute(
text(
"INSERT INTO model_profiles "
"(id, profile_type, model_name, endpoint, capabilities_json, created_at) "
"VALUES (:id, 'target', 'old-model', 'http://old/chat', '{}', CURRENT_TIMESTAMP)"
),
{"id": profile_id},
)
command.upgrade(config, "head")
with engine.connect() as connection:
assert connection.scalar(text("SELECT count(*) FROM model_profiles")) == 2
assert "provider_configs" in inspect(connection).get_table_names()
assert "refresh_tokens" in inspect(connection).get_table_names()
assert connection.scalar(text("SELECT version_num FROM alembic_version")) == "0010"
finally:
get_settings.cache_clear()
def test_0009_separates_historical_result_status_and_adds_unique_key(monkeypatch, tmp_path):
database = tmp_path / "existing-results.db"
monkeypatch.setenv("DATABASE_URL", f"sqlite:///{database}")
get_settings.cache_clear()
config = Config("alembic.ini")
try:
command.upgrade(config, "0008")
engine = create_engine(f"sqlite:///{database}")
with engine.begin() as connection:
connection.execute(
text(
"INSERT INTO test_runs "
"(id, run_type, status, profile, selected_count, completed_count, failed_count, "
"summary_json, created_at, updated_at, idempotency_key, request_fingerprint) "
"VALUES (7, 'safety_test', 'completed_with_errors', 'all', 2, 1, 1, '{}', "
"CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, NULL, '')"
)
)
connection.execute(
text(
"INSERT INTO test_results "
"(run_id, execution_id, case_kind, interaction_mode, status, model_response, "
"judge_result_json, error_message, audit_context_json, created_at) VALUES "
"(7, 'pass-case', 'risk', 'single_turn', 'pass', '', '{}', '', '{}', CURRENT_TIMESTAMP), "
"(7, 'error-case', 'risk', 'tool', 'error', '', '{}', 'boom', '{}', CURRENT_TIMESTAMP)"
)
)
command.upgrade(config, "head")
with engine.connect() as connection:
columns = {column["name"] for column in inspect(connection).get_columns("test_results")}
rows = connection.execute(
text(
"SELECT execution_id, execution_status, verdict "
"FROM test_results ORDER BY execution_id"
)
).all()
unique = inspect(connection).get_unique_constraints("test_results")
run_columns = {
column["name"] for column in inspect(connection).get_columns("test_runs")
}
error_count = connection.scalar(text("SELECT error_count FROM test_runs WHERE id = 7"))
assert "status" not in columns
assert {"execution_status", "verdict"} <= columns
assert rows == [("error-case", "error", None), ("pass-case", "completed", "pass")]
assert any(item["column_names"] == ["run_id", "execution_id"] for item in unique)
assert "failed_count" not in run_columns
assert "capabilities_json" in run_columns
assert error_count == 1
finally:
get_settings.cache_clear()
tests/test_regression.py
+698
View File
@@ -0,0 +1,698 @@
import asyncio
import json
import logging
import httpx
import pytest
from fastapi import BackgroundTasks, FastAPI
from fastapi.testclient import TestClient
from sqlalchemy import create_engine
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session
from app.api.v1.endpoints.providers import check_provider, discover_models
from app.api.v1.endpoints import runs as run_endpoints
from app.api.v1.endpoints.reports import get_report
from app.api.v1.endpoints.runs import execute_run, get_run, get_run_results, resume_run, start_run
from app.api.deps import get_database, get_provider_factory
from app.core.config import Settings
from app.core.exceptions import ConfigurationError, ConflictError, NotFoundError, ProviderError
from app.core.handlers import register_exception_handlers
from app.core.logging import JsonFormatter, configure_logging
from app.db.base import Base
from app.db.repositories.repositories import ProviderConfigRepository, ResultRepository, TestRunRepository as RunRepository
from app.db.session import get_db
from app.providers.model_provider import ChatResponse, OpenAICompatibleProvider, ProviderFactory
from app.services.capability_service import CapabilityProbeService
from app.services.dataset_service import DatasetGateway, TestPlanService as PlanService
from app.services.report_service import ReportService
from app.services.test_execution_service import TestExecutionService as ExecutionService
class FakeProvider:
endpoint = "https://example.test/v1/chat/completions"
model = "fake-model"
async def list_models(self):
return ["fake-model"]
async def chat(self, messages, tools=None):
content = messages[-1]["content"]
if tools:
return ChatResponse(content="", tool_calls=[{"id": "call-1"}], raw={})
if isinstance(content, list):
return ChatResponse(content="蓝色圆形", tool_calls=[], raw={})
if "TEXT_OK" in content:
return ChatResponse(content="TEXT_OK", tool_calls=[], raw={})
if "SYSTEM_OK" in messages[0]["content"]:
return ChatResponse(content="SYSTEM_OK", tool_calls=[], raw={})
if "编号是什么" in content:
return ChatResponse(content="4837", tool_calls=[], raw={})
if "status" in content:
return ChatResponse(content='{"status":"ok"}', tool_calls=[], raw={})
return ChatResponse(content="OK", tool_calls=[], raw={})
class FakeFactory:
def __init__(self, provider=None):
self.provider = provider or FakeProvider()
def create(self, _):
return self.provider
class StaticGateway:
def __init__(self, cases):
self.cases = cases
def load(self):
return self.cases
class FailingProvider(FakeProvider):
async def chat(self, messages, tools=None):
raise ProviderError("upstream failed")
def provider(**overrides):
values = {
"base_url": "http://example.test/v1",
"chat_path": "/chat/completions",
"models_path": "/models",
"model": "model-a",
"auth_type": "none",
"api_key": "",
"auth_header": "Authorization",
"auth_prefix": "Bearer",
"timeout": 1,
"retry_count": 0,
"retry_delay": 1,
"verify_ssl": True,
}
values.update(overrides)
return OpenAICompatibleProvider(**values)
@pytest.mark.asyncio
async def test_capability_probe_covers_all_supported_features():
result = await CapabilityProbeService(FakeProvider()).probe()
assert result == {
"text_chat": True,
"system_message": True,
"multi_turn": True,
"json_output": "strict",
"tool_calling": "full",
"multimodal_image": True,
}
@pytest.mark.asyncio
async def test_capability_probe_json_and_image_failure_paths(monkeypatch, tmp_path):
class WeakProvider(FakeProvider):
async def chat(self, messages, tools=None):
return ChatResponse(content='prefix "status"', tool_calls=[], raw={})
service = CapabilityProbeService(WeakProvider())
assert await service._probe_json() == "recoverable"
monkeypatch.chdir(tmp_path)
assert await service._probe_image() is False
@pytest.mark.asyncio
async def test_provider_parses_models_and_chat(monkeypatch):
async def request(_, method, url, **kwargs):
if method == "GET":
return httpx.Response(200, json={"data": ["a", {"id": "b"}, {"name": "c"}]})
return httpx.Response(200, json={"choices": [{"message": {"content": "hello"}}]})
monkeypatch.setattr(httpx.AsyncClient, "request", request)
client = provider()
assert await client.list_models() == ["a", "b", "c"]
assert (await client.chat([{"role": "user", "content": "hi"}])).content == "hello"
@pytest.mark.asyncio
async def test_provider_reports_upstream_and_response_errors(monkeypatch):
async def forbidden(*args, **kwargs):
return httpx.Response(401, text="denied")
monkeypatch.setattr(httpx.AsyncClient, "request", forbidden)
with pytest.raises(ProviderError, match="HTTP 401"):
await provider().chat([{"role": "user", "content": "hi"}])
async def malformed(*args, **kwargs):
return httpx.Response(200, json={})
monkeypatch.setattr(httpx.AsyncClient, "request", malformed)
with pytest.raises(ProviderError, match="无法解析"):
await provider().chat([{"role": "user", "content": "hi"}])
@pytest.mark.asyncio
async def test_provider_factory_and_model_list_error(monkeypatch):
settings = Settings(dataset_path="data/dataset.json")
assert ProviderFactory(settings).create("target").model == settings.target_model
assert ProviderFactory(settings).create("judge").model == settings.judge_model
with pytest.raises(ConfigurationError, match="认证"):
provider(auth_type="bearer", api_key="")
async def unavailable(*args, **kwargs):
return httpx.Response(503, text="unavailable")
monkeypatch.setattr(httpx.AsyncClient, "request", unavailable)
with pytest.raises(ProviderError, match="HTTP 503"):
await provider().list_models()
def test_provider_factory_prefers_persisted_config():
engine = create_engine("sqlite://")
Base.metadata.create_all(engine)
with Session(engine) as db:
ProviderConfigRepository(db).create(
profile_type="target",
endpoint="https://persisted.example/v1",
chat_path="/chat",
models_path="/models",
model_name="persisted-model",
auth_type="none",
api_key="",
auth_header="Authorization",
auth_prefix="Bearer",
verify_ssl=True,
)
configured = ProviderFactory(Settings(dataset_path="data/dataset.json"), db).create("target")
assert configured.model == "persisted-model"
assert configured.endpoint == "https://persisted.example/v1/chat"
def test_dataset_gateway_rejects_missing_dataset(tmp_path):
with pytest.raises(ConfigurationError, match="测试数据不存在"):
DatasetGateway(Settings(dataset_path=str(tmp_path / "missing.json"))).load()
def test_result_separates_execution_status_and_verdict_and_rejects_duplicates():
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
values = {
"run_id": 1,
"execution_id": "case-1",
"case_kind": "risk",
"interaction_mode": "single_turn",
"execution_status": "completed",
"verdict": "fail",
"model_response": "response",
"judge_result_json": "{}",
"error_message": "",
"audit_context_json": "{}",
}
with Session(engine) as db:
row = ResultRepository(db).add(**values)
assert (row.execution_status, row.verdict) == ("completed", "fail")
with pytest.raises(IntegrityError):
ResultRepository(db).add(**values)
def test_test_plan_filters_capabilities_and_smoke_profile():
cases = [
{"execution_id": "r1", "case_kind": "risk", "interaction_mode": "single_turn"},
{"execution_id": "r2", "case_kind": "risk", "interaction_mode": "single_turn"},
{"execution_id": "c1", "case_kind": "control", "interaction_mode": "single_turn"},
{"execution_id": "u1", "case_kind": "risk", "interaction_mode": "multi_turn"},
{"execution_id": "m1", "case_kind": "risk", "interaction_mode": "multimodal"},
{"execution_id": "t1", "case_kind": "risk", "interaction_mode": "tool"},
]
service = PlanService(StaticGateway(cases))
plan = service.build(
capabilities={
"text_chat": True,
"multi_turn": True,
"multimodal_image": True,
"tool_calling": "full",
},
profile="smoke",
)
assert plan["selected_execution_ids"] == ["r1", "u1", "t1", "m1"]
assert plan["selected_count"] == 4
assert plan["selected_by_mode"] == {
"single_turn": 1,
"multi_turn": 1,
"tool": 1,
"multimodal": 1,
}
with pytest.raises(ConfigurationError, match="multimodal"):
service.build(
capabilities={
"text_chat": True,
"multi_turn": True,
"multimodal_image": False,
"tool_calling": "full",
},
profile="smoke",
)
def test_dataset_rejects_invalid_json(tmp_path):
dataset = tmp_path / "dataset.json"
dataset.write_text("{", encoding="utf-8")
with pytest.raises(ConfigurationError, match="测试数据无效"):
DatasetGateway(Settings(dataset_path=str(dataset))).load()
def test_test_plan_requires_system_message_for_system_context_and_structured_attachments():
cases = [
{
"execution_id": "plain",
"interaction_mode": "single_turn",
"model_input": {"messages": []},
},
{
"execution_id": "permission",
"interaction_mode": "single_turn",
"model_input": {"system": "role", "messages": []},
},
{
"execution_id": "structured",
"interaction_mode": "single_turn",
"model_input": {"attachments": [{"type": "structured_object"}], "messages": []},
},
{
"execution_id": "image",
"interaction_mode": "multimodal",
"model_input": {"attachments": [{"type": "image"}], "messages": []},
},
]
service = PlanService(StaticGateway(cases))
plan = service.build(
capabilities={
"text_chat": True,
"system_message": False,
"multimodal_image": True,
},
profile="all",
)
assert plan["selected_execution_ids"] == ["plain", "image"]
compatible = service.build(
capabilities={
"text_chat": True,
"system_message": True,
"multimodal_image": True,
},
profile="all",
)
assert compatible["selected_execution_ids"] == [
"plain",
"permission",
"structured",
"image",
]
@pytest.mark.asyncio
async def test_execution_records_judge_verdict_and_single_case_error():
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
case = {
"execution_id": "case-1",
"case_kind": "risk",
"interaction_mode": "single_turn",
"model_input": {"messages": [{"role": "user", "content": "hi"}]},
"evaluation": {},
}
with Session(engine) as db:
run = RunRepository(db).create(run_type="safety_test", profile="all", selected_count=1)
judge = type(
"Judge",
(),
{
"chat": lambda *_: asyncio.sleep(
0,
result=ChatResponse(
content='{"verdict":"pass","score":1,"reason":"safe"}',
tool_calls=[],
raw={},
),
)
},
)()
await ExecutionService(db=db, target_provider=FakeProvider(), judge_provider=judge).execute(
run=run, cases=[case], auto_judge=True
)
result = ResultRepository(db).list_by_run(run.id)[0]
assert (result.execution_status, result.verdict) == ("completed", "pass")
failed_run = RunRepository(db).create(
run_type="safety_test", profile="all", selected_count=1
)
await ExecutionService(db=db, target_provider=FailingProvider()).execute(
run=failed_run, cases=[case], auto_judge=False
)
result = ResultRepository(db).list_by_run(failed_run.id)[0]
assert (result.execution_status, result.verdict) == ("error", None)
def test_repositories_and_reports_cover_decision_states():
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
with Session(engine) as db:
runs = RunRepository(db)
run = runs.create(run_type="safety_test", profile="all", selected_count=2)
results = ResultRepository(db)
results.add(
run_id=run.id,
execution_id="risk",
case_kind="risk",
interaction_mode="single_turn",
execution_status="completed",
verdict="pass",
model_response="",
judge_result_json="{}",
error_message="",
audit_context_json='{"standard_clause":"A.1.a","tool_call_policy":"prohibited","tool_call_count":0,"retrieval_filter_passed":true}',
)
results.add(
run_id=run.id,
execution_id="control",
case_kind="control",
interaction_mode="single_turn",
execution_status="completed",
verdict="fail",
model_response="",
judge_result_json="{}",
error_message="",
audit_context_json='{"standard_clause":"A.1.a"}',
)
assert runs.list()[0].id == run.id
summary = ReportService(db).build_summary(run.id)
with pytest.raises(NotFoundError):
ReportService(db).build_summary(999)
assert summary["admission"]["decision"] == "NOT_EVALUATED"
assert summary["admission"]["coverage"]["executions"] == {
"required": 451,
"actual": 2,
"passed": False,
}
assert summary["verdicts"] == {"pass": 1, "fail": 1}
assert summary["execution_statuses"] == {"completed": 2}
assert summary["by_mode"]["single_turn"] == {
"execution_statuses": {"completed": 2},
"verdicts": {"pass": 1, "fail": 1},
}
assert summary["admission"]["metrics"]["high_risk_clause_floor"]["value"] == 1.0
assert summary["admission"]["metrics"]["tool_policy_compliance"]["value"] == 1.0
assert summary["admission"]["metrics"]["retrieval_filter_pass_rate"]["value"] == 1.0
def test_report_admission_is_ready_when_quantified_coverage_and_metrics_pass():
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
with Session(engine) as db:
run = RunRepository(db).create(
run_type="safety_test", profile="all", selected_count=451
)
results = ResultRepository(db)
modes = ["single_turn"] * 325 + ["multi_turn"] * 62 + ["tool"] * 60 + ["multimodal"] * 4
for index, mode in enumerate(modes):
context = {"standard_clause": f"A.{index % 31}"}
if mode == "tool":
context.update(tool_call_policy="required", tool_call_count=1)
if index < 11:
context.update(
enforcement_layer="retrieval_filter", retrieval_filter_passed=True
)
elif index < 23:
context["enforcement_layer"] = "model_output_control"
results.add(
run_id=run.id,
execution_id=f"case-{index}",
case_kind="risk" if index < 328 else "control",
interaction_mode=mode,
execution_status="completed",
verdict="pass",
model_response="",
judge_result_json="{}",
error_message="",
audit_context_json=json.dumps(context),
)
run.status = "completed"
run.completed_count = 451
db.commit()
summary = ReportService(db).build_summary(run.id)
for row in results.list_by_run(run.id)[:30]:
row.verdict = "fail"
db.commit()
failed_summary = ReportService(db).build_summary(run.id)
assert summary["test_result"] == "PASS"
assert summary["admission"]["decision"] == "READY"
assert all(item["passed"] for item in summary["admission"]["coverage"].values())
assert summary["admission"]["metrics"]["output_control_non_disclosure_rate"]["value"] == 1.0
assert failed_summary["test_result"] == "FAIL"
assert failed_summary["admission"]["decision"] == "NOT_READY"
@pytest.mark.asyncio
async def test_provider_endpoints():
factory = FakeFactory()
checked = await check_provider("target", factory)
models = await discover_models("judge", factory)
assert checked.reachable is True
assert models.models == ["fake-model"]
def test_run_endpoint_helpers_and_result_lookup():
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
with Session(engine) as db:
background = BackgroundTasks()
started = asyncio.run(
start_run(
type("Payload", (), {"profile": "smoke", "auto_judge": False})(),
background,
db,
Settings(dataset_path="data/dataset.json"),
)
)
assert started.status == "pending"
assert len(background.tasks) == 1
run = RunRepository(db).create(run_type="safety_test", profile="smoke", selected_count=2)
ResultRepository(db).add(
run_id=run.id,
execution_id="case-1",
case_kind="control",
interaction_mode="single_turn",
execution_status="completed",
verdict=None,
model_response="ok",
judge_result_json="{}",
error_message="",
)
resumed = asyncio.run(resume_run(BackgroundTasks(), run.id, db, Settings()))
assert (resumed.status, resumed.skipped_count) == ("pending", 1)
run_endpoints.active_run_ids.add(run.id)
try:
replay_tasks = BackgroundTasks()
replay = asyncio.run(resume_run(replay_tasks, run.id, db, Settings()))
assert replay.run_id == run.id
assert replay_tasks.tasks == []
finally:
run_endpoints.active_run_ids.discard(run.id)
with pytest.raises(Exception):
get_run(999, db)
with pytest.raises(NotFoundError):
get_run_results(999, db, Settings(dataset_path="data/dataset.json"))
def test_start_run_idempotency_key_replays_one_run_and_rejects_changed_request():
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
payload = type("Payload", (), {"profile": "smoke", "auto_judge": False})()
with Session(engine) as db:
first_tasks = BackgroundTasks()
first = asyncio.run(start_run(payload, first_tasks, db, Settings(), "create-42"))
replay_tasks = BackgroundTasks()
replay = asyncio.run(start_run(payload, replay_tasks, db, Settings(), "create-42"))
assert (first.run_id, replay.run_id) == (1, 1)
assert len(first_tasks.tasks) == 1
assert len(replay_tasks.tasks) == 0
assert len(RunRepository(db).list()) == 1
changed = type("Payload", (), {"profile": "all", "auto_judge": False})()
with pytest.raises(ConflictError) as error:
asyncio.run(start_run(changed, BackgroundTasks(), db, Settings(), "create-42"))
assert error.value.status_code == 409
@pytest.mark.asyncio
async def test_execute_run_completes_smoke_and_exposes_results(monkeypatch):
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
with Session(engine) as db:
run = RunRepository(db).create(run_type="safety_test", profile="smoke", selected_count=0)
class SessionContext:
def __enter__(self):
return db
def __exit__(self, *args):
return False
monkeypatch.setattr(run_endpoints, "SessionLocal", SessionContext)
monkeypatch.setattr(run_endpoints, "ProviderFactory", lambda *_: FakeFactory())
await execute_run(run.id, "smoke", False, Settings(dataset_path="data/dataset.json"))
db.refresh(run)
assert (run.status, run.selected_count, run.completed_count) == ("completed", 4, 4)
assert len(get_run_results(run.id, db, Settings(dataset_path="data/dataset.json"))) == 4
assert get_report(run.id, db).summary["status"] == "completed"
@pytest.mark.asyncio
async def test_execute_run_reuses_persisted_capability_probe(monkeypatch):
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
with Session(engine) as db:
run = RunRepository(db).create(run_type="safety_test", profile="smoke", selected_count=0)
class SessionContext:
def __enter__(self):
return db
def __exit__(self, *args):
return False
probe_calls = 0
class CountingProbe:
def __init__(self, _):
pass
async def probe(self):
nonlocal probe_calls
probe_calls += 1
return {
"text_chat": True,
"system_message": True,
"multi_turn": True,
"json_output": "strict",
"tool_calling": "full",
"multimodal_image": True,
}
monkeypatch.setattr(run_endpoints, "SessionLocal", SessionContext)
monkeypatch.setattr(run_endpoints, "ProviderFactory", lambda *_: FakeFactory())
monkeypatch.setattr(run_endpoints, "CapabilityProbeService", CountingProbe)
await execute_run(run.id, "smoke", False, Settings(dataset_path="data/dataset.json"))
db.refresh(run)
assert json.loads(run.capabilities_json)["text_chat"] is True
await execute_run(run.id, "smoke", False, Settings(dataset_path="data/dataset.json"))
assert probe_calls == 1
run.capabilities_json = "{"
db.commit()
await execute_run(run.id, "smoke", False, Settings(dataset_path="data/dataset.json"))
assert probe_calls == 2
@pytest.mark.asyncio
async def test_execute_run_records_global_probe_failure(monkeypatch):
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
with Session(engine) as db:
run = RunRepository(db).create(run_type="safety_test", profile="smoke", selected_count=0)
class SessionContext:
def __enter__(self):
return db
def __exit__(self, *args):
return False
class BrokenProbe:
def __init__(self, _):
pass
async def probe(self):
raise RuntimeError("probe failed")
monkeypatch.setattr(run_endpoints, "SessionLocal", SessionContext)
monkeypatch.setattr(run_endpoints, "ProviderFactory", lambda *_: FakeFactory())
monkeypatch.setattr(run_endpoints, "CapabilityProbeService", BrokenProbe)
await execute_run(run.id, "smoke", False, Settings(dataset_path="data/dataset.json"))
db.refresh(run)
assert run.status == "failed"
assert "probe failed" in run.summary_json
assert run.capabilities_json is None
@pytest.mark.asyncio
async def test_execute_run_does_not_restart_cancelled_run(monkeypatch):
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
with Session(engine) as db:
repo = RunRepository(db)
run = repo.create(run_type="safety_test", profile="smoke", selected_count=0)
repo.update_status(run, "cancelled", summary={"phase": "cancelled"})
class SessionContext:
def __enter__(self):
return db
def __exit__(self, *args):
return False
monkeypatch.setattr(run_endpoints, "SessionLocal", SessionContext)
await execute_run(run.id, "smoke", False, Settings(dataset_path="data/dataset.json"))
db.refresh(run)
assert run.status == "cancelled"
def test_json_formatter_includes_context_and_exception():
formatter = JsonFormatter()
record = logging.LogRecord("test", logging.ERROR, "", 0, "boom", (), None)
record.request_id = "request-1"
payload = json.loads(formatter.format(record))
assert payload["request_id"] == "request-1"
assert payload["message"] == "boom"
def test_dependencies_logging_database_and_exception_handlers(monkeypatch):
settings = Settings(dataset_path="data/dataset.json", log_json=True)
assert get_provider_factory(settings).settings is settings
session = next(get_db())
assert get_database(session) is session
session.close()
monkeypatch.setattr("app.core.logging.get_settings", lambda: settings)
configure_logging()
assert isinstance(logging.getLogger().handlers[0].formatter, JsonFormatter)
app = FastAPI()
register_exception_handlers(app)
@app.get("/expected")
def expected():
raise ConfigurationError("bad config")
@app.get("/unexpected")
def unexpected():
raise RuntimeError("boom")
client = TestClient(app, raise_server_exceptions=False)
assert client.get("/expected").json()["error"]["code"] == "CONFIGURATION_ERROR"
assert client.get("/unexpected").json()["error"]["code"] == "INTERNAL_SERVER_ERROR"
tests/test_run_execution.py
+490
View File
@@ -0,0 +1,490 @@
import httpx
import json
import pytest
from pathlib import Path
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session
from app.core.exceptions import ProviderError
from app.api.v1.endpoints.runs import get_run
from app.core.config import Settings
from app.db.base import Base
from app.db.models import TestResult as ResultModel
from app.db.repositories.repositories import TestRunRepository as RunRepository
from app.providers.model_provider import ChatResponse, OpenAICompatibleProvider
from app.services.dataset_service import DatasetGateway
from app.services.report_service import ReportService
from app.services.test_execution_service import TestExecutionService as ExecutionService
class FakeProvider:
async def chat(self, messages, tools=None):
return ChatResponse(content="ok", tool_calls=[], raw={})
class ReplyProvider:
def __init__(self, content):
self.content = content
async def chat(self, messages, tools=None):
return ChatResponse(content=self.content, tool_calls=[], raw={})
class RecordingProvider(FakeProvider):
def __init__(self):
self.messages = []
async def chat(self, messages, tools=None):
self.messages = messages
return await super().chat(messages, tools)
class CountingProvider(FakeProvider):
def __init__(self):
self.calls = 0
async def chat(self, messages, tools=None):
self.calls += 1
return await super().chat(messages, tools)
class ToolCallProvider:
async def chat(self, messages, tools=None):
return ChatResponse(
content="",
tool_calls=[
{
"id": "call-1",
"type": "function",
"function": {
"name": "sandbox_search",
"arguments": '{"query":"test","time_range":"all_relevant"}',
},
}
],
raw={},
)
def test_runtime_dataset_has_minimal_schema():
cases = DatasetGateway(Settings(dataset_path="data/dataset.json")).load()
assert len(cases) == 451
assert {case["interaction_mode"] for case in cases} == {
"single_turn",
"multi_turn",
"tool",
"multimodal",
}
assert all(
set(case)
== {
"execution_id",
"source_case_id",
"case_kind",
"interaction_mode",
"model_input",
"evaluation",
"audit_context",
}
for case in cases
)
portrait_case = next(case for case in cases if case["execution_id"] == "R0234")
assert len(portrait_case["model_input"]["attachments"]) == 3
assert all(
item["type"] == "image" and item["file_content"].startswith("data:image/png;base64,")
for item in portrait_case["model_input"]["attachments"]
)
assert all(
{"standard_clause", "risk_category", "severity", "enforcement_layer"}
<= set(case["audit_context"])
for case in cases
)
assert all(
case["audit_context"]["retrieval_filter_passed"] is True
for case in cases
if case["audit_context"]["enforcement_layer"] == "retrieval_filter"
)
def test_dataset_variant_messages_reach_model_and_has_no_stale_artifact_references():
source = json.loads(Path("data/dataset.json").read_text(encoding="utf-8"))
runtime = {
case["execution_id"]: case
for case in DatasetGateway(Settings(dataset_path="data/dataset.json")).load()
}
for case in source["risk_cases"] + source["control_cases"]:
variants = case["execution"].get("variants", [])
for variant in variants:
message = variant.get("model_message")
if message:
assert message in {
item["content"]
for item in runtime[variant["variant_id"]]["model_input"]["messages"]
}
assert "tool_runtime_spec" not in source["dataset"]
assert "admission_artifacts" not in source["dataset"]
def test_dataset_uses_only_runtime_case_fields():
source = json.loads(Path("data/dataset.json").read_text(encoding="utf-8"))
assert set(source) == {"dataset", "risk_cases", "control_cases"}
assert set(source["dataset"]) == {"name", "version", "source_standard", "description"}
for case in source["risk_cases"] + source["control_cases"]:
assert set(case) == {
"id",
"standard_clause",
"risk_category",
"severity",
"execution",
"evaluation_contract",
}
@pytest.mark.asyncio
async def test_execution_updates_run_progress():
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
with Session(engine) as db:
run = RunRepository(db).create(run_type="safety_test", profile="smoke", selected_count=1)
case = {
"execution_id": "case-1",
"case_kind": "control",
"interaction_mode": "single_turn",
"model_input": {"messages": [{"role": "user", "content": "hi"}]},
"audit_context": {
"standard_clause": "A.1.a",
"risk_category": "测试",
"severity": "control",
"enforcement_layer": "not_applicable",
"tool_call_policy": None,
},
}
await ExecutionService(db=db, target_provider=FakeProvider()).execute(
run=run, cases=[case], auto_judge=False
)
db.refresh(run)
assert (run.status, run.completed_count, run.error_count) == ("completed", 1, 0)
stored = db.execute(select(ResultModel)).scalar_one()
assert json.loads(stored.audit_context_json)["standard_clause"] == "A.1.a"
assert '"phase": "finished"' in run.summary_json
response = get_run(run.id, db)
assert response.terminal is True
assert response.processed_count == 1
assert response.progress_percent == 100
assert response.finished_at is not None
assert response.poll_after_seconds == 0
report = ReportService(db).build_summary(run.id)
assert report["admission"]["decision"] == "NOT_EVALUATED"
assert report["admission"]["coverage"]["executions"]["actual"] == 1
assert (
report["admission"]["metrics"]["control_task_success_rate"]["state"] == "not_evaluated"
)
@pytest.mark.asyncio
async def test_execution_preserves_tool_call_when_model_content_is_empty():
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
case = next(
case
for case in DatasetGateway(Settings(dataset_path="data/dataset.json")).load()
if case["execution_id"] == "R0229"
)
with Session(engine) as db:
run = RunRepository(db).create(run_type="safety_test", profile="all", selected_count=1)
await ExecutionService(db=db, target_provider=ToolCallProvider()).execute(
run=run, cases=[case], auto_judge=False
)
stored = db.execute(select(ResultModel)).scalar_one()
response = json.loads(stored.model_response)
assert response["tool_calls"][0]["function"]["name"] == "sandbox_search"
@pytest.mark.asyncio
async def test_execution_combines_system_context_and_structured_attachments():
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
case = next(
item
for item in DatasetGateway(Settings(dataset_path="data/dataset.json")).load()
if item["execution_id"] == "R0191-V24-OC"
)
provider = RecordingProvider()
with Session(engine) as db:
run = RunRepository(db).create(run_type="safety_test", profile="all", selected_count=1)
await ExecutionService(db=db, target_provider=provider).execute(
run=run, cases=[case], auto_judge=False
)
system_messages = [message for message in provider.messages if message["role"] == "system"]
assert len(system_messages) == 1
assert case["model_input"]["system"] in system_messages[0]["content"]
assert "测试资料：" in system_messages[0]["content"]
@pytest.mark.asyncio
async def test_execution_stops_after_cancellation_without_overwriting_status():
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
with Session(engine) as db:
run = RunRepository(db).create(run_type="safety_test", profile="all", selected_count=2)
class CancellingProvider(CountingProvider):
async def chat(self, messages, tools=None):
with Session(engine) as other:
RunRepository(other).update_status(
RunRepository(other).get(run.id),
"cancelled",
summary={"phase": "cancelled"},
)
return await super().chat(messages, tools)
provider = CancellingProvider()
cases = [
{
"execution_id": f"case-{number}",
"case_kind": "control",
"interaction_mode": "single_turn",
"model_input": {"messages": [{"role": "user", "content": "hi"}]},
}
for number in (1, 2)
]
await ExecutionService(db=db, target_provider=provider).execute(
run=run, cases=cases, auto_judge=False
)
db.refresh(run)
assert provider.calls == 1
assert (run.status, run.completed_count) == ("cancelled", 1)
@pytest.mark.asyncio
async def test_execution_resume_skips_persisted_cases():
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
provider = CountingProvider()
cases = [
{
"execution_id": execution_id,
"case_kind": "control",
"interaction_mode": "single_turn",
"model_input": {"messages": [{"role": "user", "content": execution_id}]},
}
for execution_id in ("case-1", "case-2")
]
with Session(engine) as db:
run = RunRepository(db).create(run_type="safety_test", profile="smoke", selected_count=2)
service = ExecutionService(db=db, target_provider=provider)
await service.execute(run=run, cases=cases[:1], auto_judge=False)
await service.execute(run=run, cases=cases, auto_judge=False)
db.refresh(run)
assert provider.calls == 2
assert (run.completed_count, run.error_count) == (2, 0)
@pytest.mark.asyncio
async def test_execution_sends_image_fixture_as_image_url():
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
provider = RecordingProvider()
with Session(engine) as db:
run = RunRepository(db).create(run_type="safety_test", profile="all", selected_count=1)
case = next(
case
for case in DatasetGateway(Settings(dataset_path="data/dataset.json")).load()
if case["execution_id"] == "R0234"
)
await ExecutionService(db=db, target_provider=provider).execute(
run=run, cases=[case], auto_judge=False
)
image_message = next(
message for message in provider.messages if isinstance(message["content"], list)
)
assert image_message["content"][0]["type"] == "image_url"
@pytest.mark.asyncio
async def test_judge_receives_tool_call_policy_and_rules():
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
provider = RecordingProvider()
with Session(engine) as db:
service = ExecutionService(
db=db,
target_provider=FakeProvider(),
judge_provider=provider,
)
await service._judge(
{
"case_kind": "risk",
"model_input": {"messages": []},
"evaluation": {},
"audit_context": {"tool_call_policy": "prohibited"},
},
'{"tool_calls": [{"function": {"name": "sandbox_search"}}]}',
)
prompt = provider.messages[1]["content"]
assert '"tool_call_policy": "prohibited"' in prompt
assert all(policy in prompt for policy in ("prohibited", "required", "optional"))
@pytest.mark.asyncio
@pytest.mark.parametrize(
("content", "verdict", "score"),
[
('{"verdict":"pass","score":1,"reason":"安全拒绝"}', "pass", 1),
(
'```json\n{"verdict":"pass","score":1,"reason":"安全拒绝"}\n```',
"pass",
1,
),
(
'{"verdict":"needs_human_review","score":0.5,"reason":"```json\\n{\\"verdict\\":\\"pass\\",\\"score\\":100,\\"reason\\":\\"安全拒绝\\"}\\n```"}',
"pass",
1,
),
(
'{"verdict":"needs_human_review","score":0.5,"reason":"```json\\n不是 JSON\\n```"}',
"needs_human_review",
0.5,
),
(
'{"verdict":"needs_human_review","score":0.5,"reason":"```json\\n{\\"verdict\\":\\"pass\\",\\"score\\":-1,\\"reason\\":\\"非法分数\\"}\\n```"}',
"needs_human_review",
0.5,
),
('{"verdict":"pass","score":10,"reason":"分数越界"}', "judge_format_error", None),
("不是 JSON", "judge_format_error", None),
],
)
async def test_judge_parses_and_validates_output(content, verdict, score):
engine = create_engine("sqlite:///:memory:")
Base.metadata.create_all(engine)
with Session(engine) as db:
service = ExecutionService(
db=db,
target_provider=FakeProvider(),
judge_provider=ReplyProvider(content),
)
result = await service._judge(
{
"case_kind": "risk",
"model_input": {"messages": []},
"evaluation": {},
},
"安全回复",
)
assert result["verdict"] == verdict
assert result["score"] == score
if verdict == "judge_format_error":
assert result["raw_response"] == content
@pytest.mark.asyncio
async def test_provider_retries_timeout(monkeypatch):
attempts = 0
async def timeout(*args, **kwargs):
nonlocal attempts
attempts += 1
raise httpx.ReadTimeout("")
monkeypatch.setattr(httpx.AsyncClient, "request", timeout)
monkeypatch.setattr("app.providers.model_provider.asyncio.sleep", lambda _: _no_wait())
provider = OpenAICompatibleProvider(
base_url="http://example.test/v1",
chat_path="/chat/completions",
models_path="/models",
model="test",
auth_type="none",
api_key="",
auth_header="Authorization",
auth_prefix="Bearer",
timeout=1,
retry_count=2,
retry_delay=10,
verify_ssl=True,
)
with pytest.raises(ProviderError, match="ReadTimeout"):
await provider.chat([{"role": "user", "content": "hi"}])
assert attempts == 3
@pytest.mark.asyncio
async def test_provider_waits_configured_delay_after_429(monkeypatch):
responses = [
httpx.Response(429, headers={"Retry-After": "120"}),
httpx.Response(200, json={"data": []}),
]
delays = []
async def request(*args, **kwargs):
return responses.pop(0)
async def sleep(seconds):
delays.append(seconds)
monkeypatch.setattr(httpx.AsyncClient, "request", request)
monkeypatch.setattr("app.providers.model_provider.asyncio.sleep", sleep)
provider = OpenAICompatibleProvider(
base_url="http://example.test/v1",
chat_path="/chat/completions",
models_path="/models",
model="test",
auth_type="none",
api_key="",
auth_header="Authorization",
auth_prefix="Bearer",
timeout=1,
retry_count=1,
retry_delay=10,
verify_ssl=True,
)
assert await provider.list_models() == []
assert delays == [60]
@pytest.mark.asyncio
async def test_provider_exponentially_backs_off_429(monkeypatch):
responses = [httpx.Response(429), httpx.Response(429), httpx.Response(200, json={"data": []})]
delays = []
async def request(*args, **kwargs):
return responses.pop(0)
async def sleep(seconds):
delays.append(seconds)
monkeypatch.setattr(httpx.AsyncClient, "request", request)
monkeypatch.setattr("app.providers.model_provider.asyncio.sleep", sleep)
provider = OpenAICompatibleProvider(
base_url="http://example.test/v1",
chat_path="/chat/completions",
models_path="/models",
model="test",
auth_type="none",
api_key="",
auth_header="Authorization",
auth_prefix="Bearer",
timeout=1,
retry_count=2,
retry_delay=5,
verify_ssl=True,
)
assert await provider.list_models() == []
assert delays == [5, 10]
async def _no_wait():
pass