first commit

app/api/deps.py

@@ -0,0 +1,16 @@
+from fastapi import Depends
+from sqlalchemy.orm import Session
+
+from app.core.config import Settings, get_settings
+from app.db.session import get_db
+from app.providers.model_provider import ProviderFactory
+
+
+def get_provider_factory(
+    settings: Settings = Depends(get_settings), db: Session = Depends(get_db)
+) -> ProviderFactory:
+    return ProviderFactory(settings, db)
+
+
+def get_database(db: Session = Depends(get_db)) -> Session:
+    return db

app/api/security.py

@@ -0,0 +1,4 @@
+from fastapi.security import HTTPBearer
+
+# OpenAPI declaration only; AuthenticationMiddleware remains the single enforcement point.
+bearer_scheme = HTTPBearer(scheme_name="BearerAuth", auto_error=False)

app/api/v1/endpoints/auth.py

@@ -0,0 +1,317 @@
+import logging
+import time
+from typing import Annotated
+from uuid import uuid4
+
+from fastapi import APIRouter, Depends, Header, Request, Response, Security, status
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from app.api.deps import get_database
+from app.api.security import bearer_scheme
+from app.core.config import Settings, get_settings
+from app.core.exceptions import (
+    ConfigurationError,
+    ConflictError,
+    ForbiddenError,
+    NotFoundError,
+    RefreshNotDueError,
+    UnauthorizedError,
+)
+from app.db.repositories.repositories import UserRepository
+from app.schemas.api import (
+    ChangePasswordRequest,
+    CreateUserRequest,
+    ErrorResponse,
+    LoginRequest,
+    ResetPasswordRequest,
+    RefreshTokenRequest,
+    TokenResponse,
+    UpdateUserRequest,
+    UserResponse,
+)
+from app.services.auth_service import (
+    derive_refresh_token,
+    issue_refresh_token,
+    issue_token,
+    token_digest,
+    verify_password,
+)
+
+logger = logging.getLogger(__name__)
+router = APIRouter()
+
+
+@router.post("/auth/login", response_model=TokenResponse, summary="登录并获取访问令牌")
+def login(
+    payload: LoginRequest,
+    db: Session = Depends(get_database),
+    settings: Settings = Depends(get_settings),
+) -> TokenResponse:
+    user = UserRepository(db).get_active_by_username(payload.username)
+    if not user or not verify_password(payload.password, user.password_hash):
+        logger.warning("Login failed username=%s", payload.username)
+        raise UnauthorizedError("用户名或密码错误")
+    try:
+        family_id = str(uuid4())
+        refresh_token = issue_refresh_token()
+        refresh_expires_at = int(time.time()) + settings.auth_refresh_token_ttl_seconds
+        token, expires_at = issue_token(user.id, settings, user.token_version, family_id)
+        UserRepository(db).create_refresh_token(
+            user_id=user.id,
+            family_id=family_id,
+            token_digest=token_digest(refresh_token),
+            token_version=user.token_version,
+            expires_at=refresh_expires_at,
+            access_expires_at=expires_at,
+        )
+    except ValueError as exc:
+        raise ConfigurationError("认证令牌配置无效") from exc
+    logger.info("Login succeeded user_id=%s", user.id)
+    return TokenResponse(
+        access_token=token,
+        expires_at=expires_at,
+        refresh_token=refresh_token,
+        refresh_expires_at=refresh_expires_at,
+    )
+
+
+def _token_response(row, refresh_token: str, settings: Settings) -> TokenResponse:
+    access_token, _ = issue_token(
+        row.user_id,
+        settings,
+        row.token_version,
+        row.family_id,
+        row.access_expires_at,
+    )
+    return TokenResponse(
+        access_token=access_token,
+        expires_at=row.access_expires_at,
+        refresh_token=refresh_token,
+        refresh_expires_at=row.expires_at,
+    )
+
+
+@router.post(
+    "/auth/refresh",
+    response_model=TokenResponse,
+    summary="轮换刷新令牌并签发新的访问令牌",
+    responses={
+        401: {"model": ErrorResponse, "description": "刷新令牌或会话已失效。"},
+        409: {"model": ErrorResponse, "description": "访问令牌尚未进入续约窗口。"},
+    },
+)
+def refresh(
+    payload: RefreshTokenRequest,
+    idempotency_key: Annotated[str, Header(alias="Idempotency-Key", min_length=1, max_length=128)],
+    db: Session = Depends(get_database),
+    settings: Settings = Depends(get_settings),
+) -> TokenResponse:
+    repo = UserRepository(db)
+    digest = token_digest(payload.refresh_token)
+    row = repo.get_refresh_token(digest)
+    now = int(time.time())
+    user = repo.get_active(row.user_id) if row else None
+    if (
+        not row
+        or not user
+        or row.revoked_at is not None
+        or row.expires_at <= now
+        or row.access_expires_at is None
+        or user.token_version != row.token_version
+    ):
+        raise UnauthorizedError("需要有效的刷新令牌")
+
+    replacement = derive_refresh_token(payload.refresh_token, idempotency_key, settings)
+    replacement_digest = token_digest(replacement)
+    if row.consumed_at is not None:
+        if row.idempotency_key == idempotency_key and row.replacement_digest == replacement_digest:
+            return _token_response(row, replacement, settings)
+        repo.revoke_refresh_family(row.family_id, now)
+        raise UnauthorizedError("刷新令牌已被重放，会话已撤销")
+
+    refresh_after = row.access_expires_at - settings.auth_token_refresh_window_seconds
+    if now < refresh_after:
+        raise RefreshNotDueError(
+            "访问令牌尚未进入续约窗口", details={"refresh_after": refresh_after}
+        )
+
+    access_expires_at = now + settings.auth_token_ttl_seconds
+    refresh_expires_at = now + settings.auth_refresh_token_ttl_seconds
+    if repo.rotate_refresh_token(
+        row,
+        idempotency_key=idempotency_key,
+        replacement_digest=replacement_digest,
+        access_expires_at=access_expires_at,
+        refresh_expires_at=refresh_expires_at,
+        consumed_at=now,
+    ):
+        row.access_expires_at = access_expires_at
+        row.expires_at = refresh_expires_at
+        return _token_response(row, replacement, settings)
+
+    row = repo.get_refresh_token(digest)
+    if (
+        row
+        and row.idempotency_key == idempotency_key
+        and row.replacement_digest == replacement_digest
+    ):
+        return _token_response(row, replacement, settings)
+    if row:
+        repo.revoke_refresh_family(row.family_id, now)
+    raise UnauthorizedError("刷新令牌已被重放，会话已撤销")
+
+
+@router.get(
+    "/auth/me",
+    response_model=UserResponse,
+    summary="读取当前登录用户",
+    description="返回当前 Bearer 令牌对应的已启用用户及其管理员标志。",
+    dependencies=[Security(bearer_scheme)],
+)
+def get_current_user(request: Request, db: Session = Depends(get_database)) -> UserResponse:
+    user = UserRepository(db).get_active(request.state.user_id)
+    if not user:
+        raise UnauthorizedError("需要有效的访问令牌")
+    return UserResponse.model_validate(user, from_attributes=True)
+
+
+@router.patch(
+    "/auth/password",
+    status_code=status.HTTP_204_NO_CONTENT,
+    summary="修改当前登录用户密码",
+    description=(
+        "校验当前密码后修改本人密码。成功后该用户已签发的所有 Bearer 令牌"
+        "立即失效，客户端应使用新密码重新登录。"
+    ),
+    dependencies=[Security(bearer_scheme)],
+)
+def change_password(
+    payload: ChangePasswordRequest,
+    request: Request,
+    db: Session = Depends(get_database),
+) -> Response:
+    repo = UserRepository(db)
+    user = repo.get_active(request.state.user_id)
+    if not user or not verify_password(payload.current_password, user.password_hash):
+        raise UnauthorizedError("当前密码错误")
+    repo.reset_password(user, payload.new_password)
+    logger.info("User changed own password user_id=%s", user.id)
+    return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+
+def _admin(request: Request, db: Session) -> UserRepository:
+    repo = UserRepository(db)
+    user = repo.get_active(request.state.user_id)
+    if not user or not user.is_admin:
+        raise ForbiddenError("需要管理员权限")
+    return repo
+
+
+@router.get(
+    "/auth/users",
+    response_model=list[UserResponse],
+    summary="列出本地用户",
+    dependencies=[Security(bearer_scheme)],
+)
+def list_users(request: Request, db: Session = Depends(get_database)) -> list[UserResponse]:
+    return [
+        UserResponse.model_validate(user, from_attributes=True)
+        for user in _admin(request, db).list()
+    ]
+
+
+@router.post(
+    "/auth/users",
+    status_code=status.HTTP_201_CREATED,
+    response_model=UserResponse,
+    summary="创建本地用户",
+    dependencies=[Security(bearer_scheme)],
+)
+def create_user(
+    payload: CreateUserRequest, request: Request, db: Session = Depends(get_database)
+) -> UserResponse:
+    repo = _admin(request, db)
+    try:
+        user = repo.create(**payload.model_dump())
+    except IntegrityError as exc:
+        db.rollback()
+        raise ConflictError("用户名已存在") from exc
+    logger.info("User created user_id=%s by_user_id=%s", user.id, request.state.user_id)
+    return UserResponse.model_validate(user, from_attributes=True)
+
+
+@router.patch(
+    "/auth/users/{user_id}",
+    response_model=UserResponse,
+    summary="启用或禁用本地用户",
+    dependencies=[Security(bearer_scheme)],
+)
+def update_user(
+    user_id: int, payload: UpdateUserRequest, request: Request, db: Session = Depends(get_database)
+) -> UserResponse:
+    repo = _admin(request, db)
+    if user_id == request.state.user_id and not payload.is_active:
+        raise ConflictError("不能禁用当前登录的管理员")
+    user = repo.get(user_id)
+    if not user:
+        raise NotFoundError("用户不存在")
+    updated = repo.set_active(user, payload.is_active)
+    logger.info(
+        "User active state changed user_id=%s by_user_id=%s", user_id, request.state.user_id
+    )
+    return UserResponse.model_validate(updated, from_attributes=True)
+
+
+@router.patch(
+    "/auth/users/{user_id}/password",
+    status_code=status.HTTP_204_NO_CONTENT,
+    summary="重置本地用户密码",
+    dependencies=[Security(bearer_scheme)],
+)
+def reset_password(
+    user_id: int,
+    payload: ResetPasswordRequest,
+    request: Request,
+    db: Session = Depends(get_database),
+) -> Response:
+    repo = _admin(request, db)
+    user = repo.get(user_id)
+    if not user:
+        raise NotFoundError("用户不存在")
+    repo.reset_password(user, payload.password)
+    logger.info("User password reset user_id=%s by_user_id=%s", user_id, request.state.user_id)
+    return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+
+@router.delete(
+    "/auth/users/{user_id}",
+    status_code=status.HTTP_204_NO_CONTENT,
+    summary="删除本地用户",
+    dependencies=[Security(bearer_scheme)],
+)
+def delete_user(user_id: int, request: Request, db: Session = Depends(get_database)) -> Response:
+    repo = _admin(request, db)
+    if user_id == request.state.user_id:
+        raise ConflictError("不能删除当前登录的管理员")
+    user = repo.get(user_id)
+    if not user:
+        raise NotFoundError("用户不存在")
+    repo.delete(user)
+    logger.info("User deleted user_id=%s by_user_id=%s", user_id, request.state.user_id)
+    return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+
+@router.post(
+    "/auth/logout",
+    status_code=status.HTTP_204_NO_CONTENT,
+    summary="撤销当前访问令牌",
+    dependencies=[Security(bearer_scheme)],
+)
+def logout(request: Request, db: Session = Depends(get_database)) -> Response:
+    repo = UserRepository(db)
+    repo.revoke_token(token_digest(request.headers["Authorization"][7:]))
+    if request.state.session_id:
+        repo.revoke_refresh_family(request.state.session_id, int(time.time()))
+    logger.info("Token revoked user_id=%s", request.state.user_id)
+    return Response(status_code=status.HTTP_204_NO_CONTENT)

app/api/v1/endpoints/health.py

@@ -0,0 +1,20 @@
+from fastapi import APIRouter, Depends
+
+from app.core.config import Settings, get_settings
+from app.schemas.api import HealthResponse
+
+router = APIRouter()
+
+
+@router.get(
+    "/health",
+    response_model=HealthResponse,
+    summary="健康检查",
+    description="用于负载均衡器、容器探针和运维监控。",
+)
+def health(settings: Settings = Depends(get_settings)) -> HealthResponse:
+    return HealthResponse(
+        status="ok",
+        environment=settings.app_env,
+        version="6.0.0",
+    )

app/api/v1/endpoints/providers.py

@@ -0,0 +1,232 @@
+import logging
+from typing import Literal
+
+from fastapi import APIRouter, Depends, Path, Request, Response, status
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from app.api.deps import get_database, get_provider_factory
+from app.core.config import Settings, get_settings
+from app.core.exceptions import ConflictError, ForbiddenError, NotFoundError
+from app.db.repositories.repositories import ProviderConfigRepository, UserRepository
+from app.providers.model_provider import ProviderFactory
+from app.schemas.api import (
+    DiscoverModelsResponse,
+    ErrorResponse,
+    ProviderCheckResponse,
+    ProviderConfigCreate,
+    ProviderConfigResponse,
+    ProviderConfigUpdate,
+)
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+
+
+def _response(row) -> ProviderConfigResponse:
+    return ProviderConfigResponse(
+        provider_id=row.profile_type,
+        source="database",
+        base_url=row.endpoint,
+        chat_path=row.chat_path,
+        models_path=row.models_path,
+        model_name=row.model_name,
+        auth_type=row.auth_type,
+        auth_header=row.auth_header,
+        auth_prefix=row.auth_prefix,
+        verify_ssl=row.verify_ssl,
+        api_key_configured=bool(row.api_key),
+        created_at=row.created_at,
+        updated_at=row.updated_at,
+    )
+
+
+def _env_response(settings: Settings, profile_type: Literal["target", "judge"]):
+    prefix = f"{profile_type}_"
+    return ProviderConfigResponse(
+        provider_id=profile_type,
+        source="env",
+        base_url=getattr(settings, prefix + "base_url"),
+        chat_path=getattr(settings, prefix + "chat_path"),
+        models_path=getattr(settings, prefix + "models_path"),
+        model_name=getattr(settings, prefix + "model"),
+        auth_type=getattr(settings, prefix + "auth_type"),
+        auth_header=getattr(settings, prefix + "auth_header"),
+        auth_prefix=getattr(settings, prefix + "auth_prefix").strip(),
+        verify_ssl=settings.request_verify_ssl,
+        api_key_configured=bool(getattr(settings, prefix + "api_key")),
+        created_at=None,
+        updated_at=None,
+    )
+
+
+def _admin(request: Request, db: Session) -> None:
+    user = UserRepository(db).get_active(request.state.user_id)
+    if not user or not user.is_admin:
+        raise ForbiddenError("需要管理员权限")
+
+
+@router.get(
+    "/providers",
+    response_model=list[ProviderConfigResponse],
+    summary="列出当前生效的模型提供商配置",
+)
+def list_provider_configs(
+    db: Session = Depends(get_database), settings: Settings = Depends(get_settings)
+):
+    persisted = {row.profile_type: row for row in ProviderConfigRepository(db).list()}
+    return [
+        _response(persisted[profile_type])
+        if profile_type in persisted
+        else _env_response(settings, profile_type)
+        for profile_type in ("target", "judge")
+    ]
+
+
+@router.get(
+    "/providers/{provider_id}",
+    response_model=ProviderConfigResponse,
+    summary="读取模型提供商配置",
+)
+def get_provider_config(
+    provider_id: Literal["target", "judge"] = Path(description="模型提供商配置 ID。"),
+    db: Session = Depends(get_database),
+    settings: Settings = Depends(get_settings),
+):
+    row = ProviderConfigRepository(db).get_by_type(provider_id)
+    return _response(row) if row else _env_response(settings, provider_id)
+
+
+@router.post(
+    "/providers", status_code=status.HTTP_201_CREATED,
+    response_model=ProviderConfigResponse, summary="创建模型提供商配置"
+)
+def create_provider_config(
+    payload: ProviderConfigCreate, request: Request, db: Session = Depends(get_database)
+):
+    _admin(request, db)
+    values = payload.model_dump()
+    values["profile_type"] = values.pop("provider_id")
+    values["endpoint"] = values.pop("base_url")
+    try:
+        row = ProviderConfigRepository(db).create(**values)
+    except IntegrityError as exc:
+        db.rollback()
+        raise ConflictError(f"{payload.provider_id} 模型提供商配置已存在") from exc
+    logger.info("Provider config created db_id=%s provider_id=%s by_user_id=%s", row.id, row.profile_type, request.state.user_id)
+    return _response(row)
+
+
+@router.patch(
+    "/providers/{provider_id}",
+    response_model=ProviderConfigResponse,
+    summary="更新模型提供商配置",
+)
+def update_provider_config(
+    provider_id: Literal["target", "judge"], payload: ProviderConfigUpdate, request: Request,
+    db: Session = Depends(get_database),
+):
+    _admin(request, db)
+    repo = ProviderConfigRepository(db)
+    row = repo.get_by_type(provider_id)
+    if not row:
+        raise NotFoundError("数据库中不存在该模型提供商配置；请先创建覆盖配置")
+    values = payload.model_dump(exclude_unset=True)
+    if "base_url" in values:
+        values["endpoint"] = values.pop("base_url")
+    auth_type = values.get("auth_type", row.auth_type)
+    api_key = values.get("api_key", row.api_key)
+    if auth_type != "none" and not api_key:
+        raise ConflictError("启用认证时必须提供 API Key")
+    row = repo.update(row, **values)
+    logger.info("Provider config updated db_id=%s provider_id=%s by_user_id=%s", row.id, row.profile_type, request.state.user_id)
+    return _response(row)
+
+
+@router.delete(
+    "/providers/{provider_id}",
+    status_code=status.HTTP_204_NO_CONTENT,
+    summary="删除模型提供商配置",
+)
+def delete_provider_config(
+    provider_id: Literal["target", "judge"], request: Request, db: Session = Depends(get_database)
+) -> Response:
+    _admin(request, db)
+    repo = ProviderConfigRepository(db)
+    row = repo.get_by_type(provider_id)
+    if not row:
+        raise NotFoundError("数据库中不存在该模型提供商配置；环境变量配置不能删除")
+    profile_type = row.profile_type
+    repo.delete(row)
+    logger.info("Provider config deleted db_id=%s provider_id=%s by_user_id=%s", row.id, profile_type, request.state.user_id)
+    return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+PROVIDER_CHECK_RESPONSES = {
+    500: {
+        "model": ErrorResponse,
+        "description": "服务端配置缺失，例如启用了认证但未设置对应的 API Key。",
+    },
+    502: {
+        "model": ErrorResponse,
+        "description": "上游模型服务不可达、认证失败或未返回可解析的模型响应。",
+    },
+}
+
+
+@router.post(
+    "/providers/{provider_id}/check",
+    response_model=ProviderCheckResponse,
+    summary="检查模型提供商连接",
+    description="""检查目标模型或裁判模型的认证、连通性和最小推理。
+
+路径中的 `provider_id` 指定配置：`target` 使用数据库覆盖或 `TARGET_*` 回退，`judge` 使用
+数据库覆盖或 `JUDGE_*` 回退。两者均支持以下配置项：
+
+* `*_BASE_URL`：服务根地址（例如 `https://host/v1`）
+* `*_CHAT_PATH`：聊天接口路径，默认 `/chat/completions`
+* `*_MODEL`：本次最小推理使用的模型名
+* `*_AUTH_TYPE`：认证类型；`none` 时不发送认证头
+* `*_API_KEY`、`*_AUTH_HEADER`、`*_AUTH_PREFIX`：认证头由 prefix 与 key 组合；密钥不会出现在响应中
+
+还可通过 `REQUEST_TIMEOUT_SECONDS` 和 `REQUEST_VERIFY_SSL` 控制超时和 TLS 校验。
+修改 `.env` 后需重启服务，配置才会重新加载。成功表示服务接受认证，并能完成一次
+`只回复 OK` 的最小聊天请求。""",
+    responses=PROVIDER_CHECK_RESPONSES,
+)
+async def check_provider(
+    provider_id: Literal["target", "judge"] = Path(description="模型提供商配置 ID。"),
+    factory: ProviderFactory = Depends(get_provider_factory),
+):
+    provider = factory.create(provider_id)
+    reply = await provider.chat([{"role": "user", "content": "只回复 OK"}])
+    return ProviderCheckResponse(
+        provider_id=provider_id,
+        endpoint=provider.endpoint,
+        authentication={"configured": True},
+        model=provider.model,
+        reachable=True,
+        response_preview=reply.content[:200],
+    )
+
+
+@router.get(
+    "/providers/{provider_id}/models",
+    response_model=DiscoverModelsResponse,
+    summary="查询模型提供商的可用模型",
+    description=(
+        "查询 `target` 或 `judge` 配置对应的上游 `/models` 接口。"
+        "结果为上游声明的模型 ID，不代表本平台已验证其可用性。"
+    ),
+    responses=PROVIDER_CHECK_RESPONSES,
+)
+async def discover_models(
+    provider_id: Literal["target", "judge"] = Path(
+        description="模型提供商配置 ID：`target` 为被测模型，`judge` 为裁判模型。"
+    ),
+    factory: ProviderFactory = Depends(get_provider_factory),
+):
+    provider = factory.create(provider_id)
+    return DiscoverModelsResponse(
+        provider_id=provider_id,
+        models=await provider.list_models(),
+    )

app/api/v1/endpoints/reports.py

@@ -0,0 +1,44 @@
+from fastapi import APIRouter, Depends, Path, Query
+
+from sqlalchemy.orm import Session
+
+from app.api.deps import get_database
+from app.db.repositories.repositories import TestRunRepository
+from app.schemas.api import ErrorResponse, ReportResponse
+from app.services.report_service import ReportService
+
+router = APIRouter()
+
+
+@router.get(
+    "/reports",
+    response_model=list[ReportResponse],
+    summary="列出测试汇总报告",
+    description="按运行创建时间倒序返回实时计算的报告；`limit` 默认 50，最大 200。",
+)
+def list_reports(
+    limit: int = Query(default=50, ge=1, le=200),
+    db: Session = Depends(get_database),
+):
+    service = ReportService(db)
+    return [
+        ReportResponse(run_id=run.id, summary=service.build_summary(run.id))
+        for run in TestRunRepository(db).list(limit)
+    ]
+
+
+@router.get(
+    "/reports/{run_id}",
+    response_model=ReportResponse,
+    summary="查询测试汇总报告",
+    description="实时计算并返回某次测试运行的结构化汇总；运行不存在时返回 404。",
+    responses={404: {"model": ErrorResponse, "description": "指定的测试运行不存在。"}},
+)
+def get_report(
+    run_id: int = Path(description="要生成报告的测试运行 ID，必须为正整数。", gt=0),
+    db: Session = Depends(get_database),
+):
+    return ReportResponse(
+        run_id=run_id,
+        summary=ReportService(db).build_summary(run_id),
+    )

app/api/v1/endpoints/runs.py

@@ -0,0 +1,521 @@
+import json
+import logging
+from hashlib import sha256
+from typing import Annotated
+
+from fastapi import APIRouter, BackgroundTasks, Depends, Header, Path, Query, Response, status
+
+from sqlalchemy.exc import IntegrityError
+from sqlalchemy.orm import Session
+
+from app.api.deps import get_database
+from app.core.config import Settings, get_settings
+from app.core.exceptions import ConflictError, NotFoundError
+from app.db.repositories.repositories import ResultRepository, TestRunRepository
+from app.db.session import SessionLocal
+from app.providers.model_provider import ProviderFactory
+from app.schemas.api import (
+    ErrorResponse,
+    ResultItem,
+    RetryErrorsResponse,
+    RetryResultResponse,
+    RunDetailResponse,
+    RunResponse,
+    ResumeRunResponse,
+    StartRunRequest,
+    UpdateRunRequest,
+)
+from app.services.capability_service import CapabilityProbeService
+from app.services.dataset_service import DatasetGateway, TestPlanService
+from app.services.test_execution_service import TestExecutionService
+
+router = APIRouter()
+logger = logging.getLogger(__name__)
+active_run_ids: set[int] = set()
+TERMINAL_STATUSES = {"cancelled", "completed", "completed_with_errors", "failed"}
+
+
+def _run_detail(run) -> RunDetailResponse:
+    summary = json.loads(run.summary_json or "{}")
+    terminal = run.status in TERMINAL_STATUSES
+    processed = run.completed_count + run.error_count
+    return RunDetailResponse(
+        run_id=run.id,
+        status=run.status,
+        terminal=terminal,
+        phase=summary.get("phase", run.status),
+        selected_count=run.selected_count,
+        processed_count=processed,
+        completed_count=run.completed_count,
+        error_count=run.error_count,
+        progress_percent=(
+            round(processed / run.selected_count * 100, 2)
+            if run.selected_count
+            else (100.0 if terminal and run.status not in {"failed", "cancelled"} else 0.0)
+        ),
+        current_execution_id=summary.get("current_execution_id"),
+        started_at=run.created_at,
+        finished_at=run.updated_at if terminal else None,
+        error_message=summary.get("error") if run.status == "failed" else None,
+        poll_after_seconds=0 if terminal else 2,
+        summary=summary,
+    )
+
+
+async def execute_run(run_id: int, profile: str, auto_judge: bool, settings: Settings) -> None:
+    if run_id in active_run_ids:
+        return
+    active_run_ids.add(run_id)
+    try:
+        await _execute_run(run_id, profile, auto_judge, settings)
+    finally:
+        active_run_ids.discard(run_id)
+
+
+async def _execute_run(run_id: int, profile: str, auto_judge: bool, settings: Settings) -> None:
+    with SessionLocal() as db:
+        repo = TestRunRepository(db)
+        run = repo.get(run_id)
+        if not run or run.status == "cancelled":
+            return
+        try:
+            factory = ProviderFactory(settings, db)
+            target = factory.create("target")
+            try:
+                capabilities = json.loads(run.capabilities_json) if run.capabilities_json else None
+            except json.JSONDecodeError:
+                logger.warning("Invalid capability cache; probing again run_id=%s", run_id)
+                capabilities = None
+            if capabilities is None:
+                repo.update_status(
+                    run, "probing", summary={"phase": "probing", "auto_judge": auto_judge}
+                )
+                capabilities = await CapabilityProbeService(target).probe()
+                repo.save_capabilities(run, capabilities)
+            db.refresh(run)
+            if run.status == "cancelled":
+                return
+            plan = TestPlanService(DatasetGateway(settings)).build(
+                capabilities=capabilities,
+                profile=profile,
+            )
+            cases = plan["selected_cases"]
+            run.selected_count = len(cases)
+            repo.update_status(
+                run,
+                "running",
+                summary={
+                    "phase": "executing",
+                    "completed": 0,
+                    "errors": 0,
+                    "selected": len(cases),
+                    "auto_judge": auto_judge,
+                },
+            )
+            await TestExecutionService(
+                db=db,
+                target_provider=target,
+                judge_provider=factory.create("judge") if auto_judge else None,
+            ).execute(run=run, cases=cases, auto_judge=auto_judge)
+        except Exception as exc:
+            db.rollback()
+            run = repo.get(run_id)
+            if run:
+                message = str(exc) or type(exc).__name__
+                repo.update_status(
+                    run,
+                    "failed",
+                    summary={"phase": "failed", "error": message, "auto_judge": auto_judge},
+                )
+            logger.exception("Background run failed run_id=%s", run_id)
+
+
+@router.post(
+    "/runs",
+    status_code=status.HTTP_202_ACCEPTED,
+    response_model=RunResponse,
+    summary="启动安全测试",
+    description=(
+        "对当前生效的 `target` 模型提供商配置启动一次后台安全测试。"
+        "接口创建运行记录后立即返回 HTTP 202，能力探测、样例执行和裁判在后台进行。\n\n"
+        "### 请求示例\n\n"
+        '```json\n{\n  "profile": "smoke",\n  "auto_judge": true\n}\n```\n\n'
+        "* `profile=smoke`：单轮、多轮、工具和图片各选取 1 条；四类都可执行才启动。\n"
+        "* `profile=all`：执行当前模型能力支持的全部样例。\n"
+        "* `auto_judge=true`：每条回复调用当前生效的 `judge` 配置，输出 `pass`、`fail` 或 "
+        "`needs_human_review`；工具题会同时参考 `tool_call_policy` 与实际工具调用。"
+        "设为 `false` 则只保存被测模型原始回复。\n\n"
+        "### 执行与返回\n\n"
+        "返回 `202 Accepted` 与 `{run_id, status, selected_count}`。创建时 `status=pending`、"
+        "`selected_count=0`；能力筛选完成后才写入实际样例数。\n\n"
+        "### 幂等创建\n\n"
+        "客户端重试 `POST /runs` 时应携带 `Idempotency-Key` 请求头（最多 128 字符）。"
+        "相同键和相同请求参数返回同一个运行且不会重复投递后台任务；同一个键用于不同参数返回 409。"
+        "未提供该请求头时，每次调用都会创建一次新测试。\n\n"
+        "### 状态规则\n\n"
+        "| status | 含义 |\n| --- | --- |\n"
+        "| `pending` | 运行记录已创建，后台任务待启动 |\n"
+        "| `probing` | 正在探测目标模型能力 |\n"
+        "| `running` | 正在逐条执行样例及可选裁判 |\n"
+        "| `cancelled` | 已请求取消，当前上游请求结束后停止后续样例 |\n"
+        "| `completed` | 所有样例调用成功 |\n"
+        "| `completed_with_errors` | 整体已结束，但至少一条样例调用失败 |\n"
+        "| `failed` | 能力探测、数据集或全局配置阶段失败 |\n\n"
+        "### 重试规则\n\n"
+        "模型请求超时由 `REQUEST_TIMEOUT_SECONDS` 控制；网络异常、HTTP 408/429/5xx "
+        "会按 `REQUEST_RETRY_COUNT` 额外重试。例如配置为 5 时，最多发起 6 次请求。"
+        "HTTP 429、408 和 5xx 按指数退避等待，单次最多由 `REQUEST_RETRY_MAX_DELAY_SECONDS` 限制；"
+        "HTTP 429 的 `Retry-After` 头会优先采用。其他 4xx 不重试。\n\n"
+        "### 调用顺序\n\n"
+        "1. 每 1～2 秒调用 `GET /api/v1/runs/{run_id}` 查询进度。\n"
+        "2. 进入最终状态后停止轮询。\n"
+        "3. 调用 `GET /api/v1/runs/{run_id}/results` 查看逐条结果，或"
+        " `GET /api/v1/reports/{run_id}` 查看汇总。"
+    ),
+    responses={
+        202: {"description": "运行已创建，后台任务已提交。"},
+        409: {"model": ErrorResponse, "description": "幂等键已用于不同请求参数。"},
+        500: {"model": ErrorResponse, "description": "运行记录创建失败。"},
+    },
+)
+async def start_run(
+    payload: StartRunRequest,
+    background_tasks: BackgroundTasks,
+    db: Session = Depends(get_database),
+    settings: Settings = Depends(get_settings),
+    idempotency_key: Annotated[str | None, Header(alias="Idempotency-Key", max_length=128)] = None,
+):
+    repo = TestRunRepository(db)
+    fingerprint = sha256(
+        json.dumps(
+            {"profile": payload.profile, "auto_judge": payload.auto_judge},
+            sort_keys=True,
+            separators=(",", ":"),
+        ).encode()
+    ).hexdigest()
+    if idempotency_key:
+        existing = repo.get_by_idempotency_key(idempotency_key)
+        if existing:
+            if existing.request_fingerprint != fingerprint:
+                raise ConflictError("幂等键不能复用于不同请求")
+            return RunResponse(
+                run_id=existing.id,
+                status=existing.status,
+                selected_count=existing.selected_count,
+            )
+    try:
+        run = repo.create(
+            run_type="safety_test",
+            profile=payload.profile,
+            selected_count=0,
+            idempotency_key=idempotency_key,
+            request_fingerprint=fingerprint,
+        )
+    except IntegrityError:
+        db.rollback()
+        existing = repo.get_by_idempotency_key(idempotency_key or "")
+        if existing and existing.request_fingerprint == fingerprint:
+            return RunResponse(
+                run_id=existing.id,
+                status=existing.status,
+                selected_count=existing.selected_count,
+            )
+        raise
+    background_tasks.add_task(
+        execute_run,
+        run.id,
+        payload.profile,
+        payload.auto_judge,
+        settings,
+    )
+    return RunResponse(
+        run_id=run.id,
+        status=run.status,
+        selected_count=run.selected_count,
+    )
+
+
+@router.get(
+    "/runs",
+    response_model=list[RunDetailResponse],
+    summary="列出测试运行",
+    description="按创建时间倒序返回测试运行；`limit` 默认 50，最大 200。",
+)
+def list_runs(
+    limit: int = Query(default=50, ge=1, le=200),
+    db: Session = Depends(get_database),
+):
+    return [_run_detail(run) for run in TestRunRepository(db).list(limit)]
+
+
+@router.post(
+    "/runs/{run_id}/resume",
+    status_code=status.HTTP_202_ACCEPTED,
+    response_model=ResumeRunResponse,
+    summary="从已保存结果恢复测试运行",
+    description=(
+        "仅用于已中断且确认原后台任务不再运行的测试。已持久化结果的样例不会再次请求模型，"
+        "包括 `execution_status=error` 的样例；该接口只补跑尚未落库的样例。若同一运行仍在当前进程执行，"
+        "重复调用仅返回当前运行，不会再次投递后台任务。`completed_with_errors` 应调用 "
+        "`retry-errors`，`completed` 无需恢复。"
+    ),
+    responses={404: {"model": ErrorResponse}, 409: {"model": ErrorResponse}},
+)
+async def resume_run(
+    background_tasks: BackgroundTasks,
+    run_id: int = Path(description="要恢复的测试运行 ID，必须为正整数。", gt=0),
+    db: Session = Depends(get_database),
+    settings: Settings = Depends(get_settings),
+):
+    run = TestRunRepository(db).get(run_id)
+    if not run:
+        raise NotFoundError("测试运行不存在")
+    skipped_count = len(ResultRepository(db).execution_ids_by_run(run_id))
+    if run_id in active_run_ids:
+        return ResumeRunResponse(
+            run_id=run.id,
+            status=run.status,
+            selected_count=run.selected_count,
+            skipped_count=skipped_count,
+        )
+    if run.status == "completed_with_errors":
+        raise ConflictError("该运行已有错误结果，请使用 retry-errors")
+    if run.status == "completed":
+        raise ConflictError("已完成运行无需恢复")
+    summary = json.loads(run.summary_json or "{}")
+    auto_judge = summary.get("auto_judge", True)
+    TestRunRepository(db).update_status(
+        run,
+        "pending",
+        summary={
+            "phase": "pending",
+            "resuming": True,
+            "skipped": skipped_count,
+            "auto_judge": auto_judge,
+        },
+    )
+    background_tasks.add_task(execute_run, run.id, run.profile, auto_judge, settings)
+    return ResumeRunResponse(
+        run_id=run.id,
+        status=run.status,
+        selected_count=run.selected_count,
+        skipped_count=skipped_count,
+    )
+
+
+@router.post(
+    "/runs/{run_id}/retry-errors",
+    status_code=status.HTTP_202_ACCEPTED,
+    response_model=RetryErrorsResponse,
+    summary="重试运行中的错误样例",
+    description=(
+        "仅用于 `completed_with_errors`。删除该运行已落库的 `execution_status=error` 结果，"
+        "保留成功结果，并只重新执行失败样例。运行中或没有错误结果时返回 409。"
+    ),
+    responses={404: {"model": ErrorResponse}, 409: {"model": ErrorResponse}},
+)
+async def retry_run_errors(
+    background_tasks: BackgroundTasks,
+    run_id: int = Path(description="要重试错误样例的测试运行 ID。", gt=0),
+    db: Session = Depends(get_database),
+    settings: Settings = Depends(get_settings),
+):
+    run_repo = TestRunRepository(db)
+    run = run_repo.get(run_id)
+    if not run:
+        raise NotFoundError("测试运行不存在")
+    if run_id in active_run_ids or run.status != "completed_with_errors":
+        raise ConflictError("只有已结束且包含错误的运行可以重试")
+    result_repo = ResultRepository(db)
+    retry_count = result_repo.delete_errors_by_run(run_id)
+    if not retry_count:
+        db.rollback()
+        raise ConflictError("该运行没有可重试的 error 结果")
+    remaining = result_repo.list_by_run(run_id)
+    run.completed_count = sum(row.execution_status == "completed" for row in remaining)
+    run.error_count = 0
+    summary = json.loads(run.summary_json or "{}")
+    auto_judge = summary.get("auto_judge", True)
+    run_repo.update_status(
+        run,
+        "pending",
+        summary={
+            "phase": "pending",
+            "retry_errors": True,
+            "retry_count": retry_count,
+            "auto_judge": auto_judge,
+        },
+    )
+    background_tasks.add_task(execute_run, run.id, run.profile, auto_judge, settings)
+    logger.info("Run errors requeued run_id=%s retry_count=%s", run_id, retry_count)
+    return RetryErrorsResponse(
+        run_id=run.id,
+        status=run.status,
+        selected_count=run.selected_count,
+        retry_count=retry_count,
+    )
+
+
+@router.post(
+    "/runs/{run_id}/results/{execution_id}/retry",
+    status_code=status.HTTP_202_ACCEPTED,
+    response_model=RetryResultResponse,
+    summary="重试运行中的指定样例",
+    description=(
+        "仅用于 `completed` 或 `completed_with_errors` 运行。删除指定 `execution_id` "
+        "的已有结果，保留其他结果，并使用原运行的 profile 和自动仲裁设置重新排队。"
+        "运行中、非终态或重复提交返回 409；运行或指定结果不存在返回 404。"
+    ),
+    responses={404: {"model": ErrorResponse}, 409: {"model": ErrorResponse}},
+)
+async def retry_run_result(
+    background_tasks: BackgroundTasks,
+    run_id: int = Path(description="测试运行 ID。", gt=0),
+    execution_id: str = Path(description="要重试的样例执行 ID。", min_length=1, max_length=128),
+    db: Session = Depends(get_database),
+    settings: Settings = Depends(get_settings),
+):
+    run_repo = TestRunRepository(db)
+    run = run_repo.get(run_id)
+    if not run:
+        raise NotFoundError("测试运行不存在")
+    if run_id in active_run_ids or run.status not in {"completed", "completed_with_errors"}:
+        raise ConflictError("只有已结束的运行可以重试指定样例")
+    result_repo = ResultRepository(db)
+    if not result_repo.delete_by_run_and_execution(run_id, execution_id):
+        db.rollback()
+        raise NotFoundError("测试结果不存在")
+    remaining = result_repo.list_by_run(run_id)
+    run.completed_count = sum(row.execution_status == "completed" for row in remaining)
+    run.error_count = sum(row.execution_status == "error" for row in remaining)
+    summary = json.loads(run.summary_json or "{}")
+    auto_judge = summary.get("auto_judge", True)
+    run_repo.update_status(
+        run,
+        "pending",
+        summary={
+            "phase": "pending",
+            "retry_execution_id": execution_id,
+            "auto_judge": auto_judge,
+        },
+    )
+    background_tasks.add_task(execute_run, run.id, run.profile, auto_judge, settings)
+    logger.info("Run result requeued run_id=%s execution_id=%s", run_id, execution_id)
+    return RetryResultResponse(
+        run_id=run.id,
+        status=run.status,
+        selected_count=run.selected_count,
+        execution_id=execution_id,
+    )
+
+
+@router.get(
+    "/runs/{run_id}",
+    response_model=RunDetailResponse,
+    summary="查询测试运行状态",
+    description=(
+        "根据运行 ID 查询阶段和实时进度，建议每 1～2 秒轮询。\n\n"
+        "`selected_count` 是筛选后计划执行数；`completed_count` 只计成功产生结果的样例；"
+        "`error_count` 计调用目标模型或裁判模型失败的样例。执行阶段始终满足 "
+        "`completed_count + error_count <= selected_count`。\n\n"
+        "前端应以 `terminal` 为停止轮询的唯一判定；为 true 时后台已结束，"
+        "`poll_after_seconds` 为 0。`processed_count` 和 `progress_percent` 可直接用于进度条，"
+        "`current_execution_id` 用于展示当前样例。\n\n"
+        "`summary.phase` 可为 `probing`、`executing`、`finished` 或 `failed`。"
+        "当 phase 为 `failed` 时，`summary.error` 给出全局失败原因。运行 ID 不存在时返回 404。"
+    ),
+    responses={404: {"model": ErrorResponse, "description": "指定的测试运行不存在。"}},
+)
+def get_run(
+    run_id: int = Path(description="要查询的测试运行 ID，必须为正整数。", gt=0),
+    db: Session = Depends(get_database),
+):
+    run = TestRunRepository(db).get(run_id)
+    if not run:
+        raise NotFoundError("测试运行不存在")
+    return _run_detail(run)
+
+
+@router.patch(
+    "/runs/{run_id}",
+    response_model=RunDetailResponse,
+    summary="取消测试运行",
+    responses={404: {"model": ErrorResponse}, 409: {"model": ErrorResponse}},
+)
+def update_run(
+    payload: UpdateRunRequest,
+    run_id: int = Path(description="测试运行 ID。", gt=0),
+    db: Session = Depends(get_database),
+):
+    repo = TestRunRepository(db)
+    run = repo.get(run_id)
+    if not run:
+        raise NotFoundError("测试运行不存在")
+    if run.status in TERMINAL_STATUSES:
+        raise ConflictError("测试运行已处于终态")
+    repo.update_status(run, payload.status, summary={"phase": "cancelled", "cancelled": True})
+    logger.info("Run cancelled run_id=%s", run_id)
+    return _run_detail(run)
+
+
+@router.delete(
+    "/runs/{run_id}",
+    status_code=status.HTTP_204_NO_CONTENT,
+    summary="删除测试运行",
+    responses={404: {"model": ErrorResponse}, 409: {"model": ErrorResponse}},
+)
+def delete_run(
+    run_id: int = Path(description="测试运行 ID。", gt=0),
+    db: Session = Depends(get_database),
+) -> Response:
+    repo = TestRunRepository(db)
+    run = repo.get(run_id)
+    if not run:
+        raise NotFoundError("测试运行不存在")
+    if run.status not in TERMINAL_STATUSES:
+        raise ConflictError("只能删除已结束或已取消的测试运行")
+    repo.delete(run)
+    logger.info("Run deleted run_id=%s", run_id)
+    return Response(status_code=status.HTTP_204_NO_CONTENT)
+
+
+@router.get(
+    "/runs/{run_id}/results",
+    response_model=list[ResultItem],
+    summary="查询测试结果",
+    description=(
+        "返回已持久化的逐样例结果；运行中调用时可能只返回部分列表。\n\n"
+        "`execution_status=error` 表示该样例调用失败，原因在 `error_message`；"
+        "`verdict` 是独立的仲裁结论。"
+        "`model_input` 是与 `execution_id` 对应的完整测试输入，包含 messages、system 上下文及可选工具定义。"
+        "开启自动裁判时，成功样例的 status 通常为 `pass`、`fail` 或 "
+        "`needs_human_review`，结构化裁判放在 `judge_result`；关闭时 status 为 `completed`、"
+        "`judge_result` 为空对象。未知运行 ID 返回 404。"
+    ),
+)
+def get_run_results(
+    run_id: int = Path(description="要查询结果的测试运行 ID，必须为正整数。", gt=0),
+    db: Session = Depends(get_database),
+    settings: Settings = Depends(get_settings),
+):
+    if not TestRunRepository(db).get(run_id):
+        raise NotFoundError("测试运行不存在")
+    rows = ResultRepository(db).list_by_run(run_id)
+    inputs = {
+        case["execution_id"]: case.get("model_input", {})
+        for case in DatasetGateway(settings).load()
+    }
+    return [
+        ResultItem(
+            execution_id=row.execution_id,
+            case_kind=row.case_kind,
+            interaction_mode=row.interaction_mode,
+            execution_status=row.execution_status,
+            verdict=row.verdict,
+            model_input=inputs.get(row.execution_id, {}),
+            model_response=row.model_response,
+            judge_result=json.loads(row.judge_result_json or "{}"),
+            error_message=row.error_message,
+        )
+        for row in rows
+    ]

app/api/v1/router.py

@@ -0,0 +1,12 @@
+from fastapi import APIRouter, Security
+
+from app.api.security import bearer_scheme
+from app.api.v1.endpoints import auth, health, providers, reports, runs
+
+api_router = APIRouter()
+api_router.include_router(health.router, tags=["system"])
+api_router.include_router(auth.router, tags=["authentication"])
+protected = [Security(bearer_scheme)]
+api_router.include_router(providers.router, tags=["providers"], dependencies=protected)
+api_router.include_router(runs.router, tags=["runs"], dependencies=protected)
+api_router.include_router(reports.router, tags=["reports"], dependencies=protected)