first commit

alembic/env.py

@@ -0,0 +1,44 @@
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+
+from app.core.config import get_settings
+from app.db.base import Base
+from app.db.models import ModelProfile, ProviderConfig, RevokedToken, TestResult, TestRun, User  # noqa: F401
+
+config = context.config
+config.set_main_option("sqlalchemy.url", get_settings().database_url)
+
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+target_metadata = Base.metadata
+
+
+def run_migrations_offline():
+    context.configure(
+        url=config.get_main_option("sqlalchemy.url"),
+        target_metadata=target_metadata,
+        literal_binds=True,
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+    with connectable.connect() as connection:
+        context.configure(connection=connection, target_metadata=target_metadata)
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

@@ -0,0 +1,19 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

alembic/versions/0001_initial.py

@@ -0,0 +1,66 @@
+"""initial schema
+
+Revision ID: 0001
+Revises:
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0001"
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "model_profiles",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("profile_type", sa.String(length=32), nullable=False),
+        sa.Column("model_name", sa.String(length=255), nullable=False),
+        sa.Column("endpoint", sa.String(length=1024), nullable=False),
+        sa.Column("capabilities_json", sa.Text(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+    )
+    op.create_index("ix_model_profiles_profile_type", "model_profiles", ["profile_type"])
+
+    op.create_table(
+        "test_runs",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("run_type", sa.String(length=32), nullable=False),
+        sa.Column("status", sa.String(length=32), nullable=False),
+        sa.Column("profile", sa.String(length=32), nullable=False),
+        sa.Column("selected_count", sa.Integer(), nullable=False),
+        sa.Column("completed_count", sa.Integer(), nullable=False),
+        sa.Column("failed_count", sa.Integer(), nullable=False),
+        sa.Column("summary_json", sa.Text(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+    )
+    op.create_index("ix_test_runs_status", "test_runs", ["status"])
+    op.create_index("ix_test_runs_run_type", "test_runs", ["run_type"])
+
+    op.create_table(
+        "test_results",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("run_id", sa.Integer(), nullable=False),
+        sa.Column("execution_id", sa.String(length=128), nullable=False),
+        sa.Column("case_kind", sa.String(length=32), nullable=False),
+        sa.Column("interaction_mode", sa.String(length=32), nullable=False),
+        sa.Column("status", sa.String(length=32), nullable=False),
+        sa.Column("model_response", sa.Text(), nullable=False),
+        sa.Column("judge_result_json", sa.Text(), nullable=False),
+        sa.Column("error_message", sa.Text(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+    )
+    op.create_index("ix_test_results_run_id", "test_results", ["run_id"])
+    op.create_index("ix_test_results_execution_id", "test_results", ["execution_id"])
+    op.create_index("ix_test_results_case_kind", "test_results", ["case_kind"])
+    op.create_index("ix_test_results_interaction_mode", "test_results", ["interaction_mode"])
+    op.create_index("ix_test_results_status", "test_results", ["status"])
+
+
+def downgrade():
+    op.drop_table("test_results")
+    op.drop_table("test_runs")
+    op.drop_table("model_profiles")

alembic/versions/0002_run_idempotency.py

@@ -0,0 +1,27 @@
+"""persist idempotency keys for run creation
+
+Revision ID: 0002
+Revises: 0001
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = "0002"
+down_revision = "0001"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("test_runs") as batch:
+        batch.add_column(sa.Column("idempotency_key", sa.String(length=128), nullable=True))
+        batch.add_column(sa.Column("request_fingerprint", sa.String(length=128), nullable=False, server_default=""))
+        batch.create_unique_constraint("uq_test_runs_idempotency_key", ["idempotency_key"])
+
+
+def downgrade():
+    with op.batch_alter_table("test_runs") as batch:
+        batch.drop_constraint("uq_test_runs_idempotency_key", type_="unique")
+        batch.drop_column("request_fingerprint")
+        batch.drop_column("idempotency_key")

alembic/versions/0003_result_audit_context.py

@@ -0,0 +1,18 @@
+"""persist dataset audit context with test results
+
+Revision ID: 0003
+Revises: 0002
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0003"
+down_revision = "0002"
+branch_labels = None
+depends_on = None
+
+def upgrade():
+    op.add_column("test_results", sa.Column("audit_context_json", sa.Text(), nullable=False, server_default="{}"))
+
+def downgrade():
+    op.drop_column("test_results", "audit_context_json")

alembic/versions/0004_users.py

@@ -0,0 +1,30 @@
+"""add local authentication users
+
+Revision ID: 0004
+Revises: 0003
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0004"
+down_revision = "0003"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "users",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("username", sa.String(length=64), nullable=False),
+        sa.Column("password_hash", sa.String(length=255), nullable=False),
+        sa.Column("is_active", sa.Boolean(), nullable=False, server_default=sa.true()),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.UniqueConstraint("username"),
+    )
+    op.create_index("ix_users_username", "users", ["username"])
+
+
+def downgrade():
+    op.drop_index("ix_users_username", table_name="users")
+    op.drop_table("users")

alembic/versions/0005_user_management.py

@@ -0,0 +1,30 @@
+"""add local user roles and token revocation
+
+Revision ID: 0005
+Revises: 0004
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0005"
+down_revision = "0004"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("users", sa.Column("is_admin", sa.Boolean(), nullable=False, server_default=sa.false()))
+    op.create_table(
+        "revoked_tokens",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("token_digest", sa.String(length=64), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.UniqueConstraint("token_digest"),
+    )
+    op.create_index("ix_revoked_tokens_token_digest", "revoked_tokens", ["token_digest"])
+
+
+def downgrade():
+    op.drop_index("ix_revoked_tokens_token_digest", table_name="revoked_tokens")
+    op.drop_table("revoked_tokens")
+    op.drop_column("users", "is_admin")

alembic/versions/0006_provider_crud.py

@@ -0,0 +1,38 @@
+"""persist provider configuration
+
+Revision ID: 0006
+Revises: 0005
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0006"
+down_revision = "0005"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "provider_configs",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("profile_type", sa.String(32), nullable=False),
+        sa.Column("model_name", sa.String(255), nullable=False),
+        sa.Column("endpoint", sa.String(1024), nullable=False),
+        sa.Column("chat_path", sa.String(255), nullable=False),
+        sa.Column("models_path", sa.String(255), nullable=False),
+        sa.Column("auth_type", sa.String(32), nullable=False),
+        sa.Column("api_key", sa.Text(), nullable=False),
+        sa.Column("auth_header", sa.String(255), nullable=False),
+        sa.Column("auth_prefix", sa.String(255), nullable=False),
+        sa.Column("verify_ssl", sa.Boolean(), nullable=False),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), nullable=False),
+        sa.UniqueConstraint("profile_type"),
+    )
+    op.create_index("ix_provider_configs_profile_type", "provider_configs", ["profile_type"])
+
+
+def downgrade():
+    op.drop_index("ix_provider_configs_profile_type", table_name="provider_configs")
+    op.drop_table("provider_configs")

alembic/versions/0007_user_token_version.py

@@ -0,0 +1,22 @@
+"""invalidate user tokens after password reset
+
+Revision ID: 0007
+Revises: 0006
+"""
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0007"
+down_revision = "0006"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column(
+        "users", sa.Column("token_version", sa.Integer(), nullable=False, server_default="0")
+    )
+
+
+def downgrade():
+    op.drop_column("users", "token_version")

alembic/versions/0008_refresh_tokens.py

@@ -0,0 +1,43 @@
+"""add rotating refresh tokens
+
+Revision ID: 0008
+Revises: 0007
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+revision = "0008"
+down_revision = "0007"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        "refresh_tokens",
+        sa.Column("id", sa.Integer(), primary_key=True),
+        sa.Column("user_id", sa.Integer(), nullable=False),
+        sa.Column("family_id", sa.String(36), nullable=False),
+        sa.Column("token_digest", sa.String(64), nullable=False),
+        sa.Column("token_version", sa.Integer(), nullable=False),
+        sa.Column("expires_at", sa.Integer(), nullable=False),
+        sa.Column("consumed_at", sa.Integer(), nullable=True),
+        sa.Column("idempotency_key", sa.String(128), nullable=True),
+        sa.Column("replacement_digest", sa.String(64), nullable=True),
+        sa.Column("access_expires_at", sa.Integer(), nullable=True),
+        sa.Column("revoked_at", sa.Integer(), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), nullable=False),
+    )
+    op.create_index("ix_refresh_tokens_user_id", "refresh_tokens", ["user_id"])
+    op.create_index("ix_refresh_tokens_family_id", "refresh_tokens", ["family_id"])
+    op.create_index(
+        "ix_refresh_tokens_token_digest", "refresh_tokens", ["token_digest"], unique=True
+    )
+
+
+def downgrade():
+    op.drop_index("ix_refresh_tokens_token_digest", table_name="refresh_tokens")
+    op.drop_index("ix_refresh_tokens_family_id", table_name="refresh_tokens")
+    op.drop_index("ix_refresh_tokens_user_id", table_name="refresh_tokens")
+    op.drop_table("refresh_tokens")

alembic/versions/0009_result_status_dimensions.py

@@ -0,0 +1,62 @@
+"""separate result execution status and verdict
+
+Revision ID: 0009
+Revises: 0008
+"""
+
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = "0009"
+down_revision = "0008"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("test_runs") as batch:
+        batch.add_column(sa.Column("error_count", sa.Integer(), nullable=False, server_default="0"))
+    op.execute("UPDATE test_runs SET error_count = failed_count")
+    with op.batch_alter_table("test_runs") as batch:
+        batch.drop_column("failed_count")
+    with op.batch_alter_table("test_results") as batch:
+        batch.add_column(
+            sa.Column("execution_status", sa.String(32), nullable=False, server_default="completed")
+        )
+        batch.add_column(sa.Column("verdict", sa.String(32), nullable=True))
+    op.execute(
+        "UPDATE test_results SET "
+        "execution_status = CASE WHEN status = 'error' THEN 'error' ELSE 'completed' END, "
+        "verdict = CASE WHEN status IN "
+        "('pass', 'fail', 'needs_human_review', 'judge_format_error') THEN status ELSE NULL END"
+    )
+    with op.batch_alter_table("test_results") as batch:
+        batch.drop_index("ix_test_results_status")
+        batch.drop_column("status")
+        batch.create_index("ix_test_results_execution_status", ["execution_status"])
+        batch.create_index("ix_test_results_verdict", ["verdict"])
+        batch.create_unique_constraint(
+            "uq_test_results_run_execution", ["run_id", "execution_id"]
+        )
+
+
+def downgrade():
+    with op.batch_alter_table("test_runs") as batch:
+        batch.add_column(sa.Column("failed_count", sa.Integer(), nullable=False, server_default="0"))
+    op.execute("UPDATE test_runs SET failed_count = error_count")
+    with op.batch_alter_table("test_runs") as batch:
+        batch.drop_column("error_count")
+    with op.batch_alter_table("test_results") as batch:
+        batch.add_column(sa.Column("status", sa.String(32), nullable=False, server_default="completed"))
+    op.execute(
+        "UPDATE test_results SET status = "
+        "CASE WHEN execution_status = 'error' THEN 'error' ELSE COALESCE(verdict, 'completed') END"
+    )
+    with op.batch_alter_table("test_results") as batch:
+        batch.drop_constraint("uq_test_results_run_execution", type_="unique")
+        batch.drop_index("ix_test_results_verdict")
+        batch.drop_index("ix_test_results_execution_status")
+        batch.drop_column("verdict")
+        batch.drop_column("execution_status")
+        batch.create_index("ix_test_results_status", ["status"])

alembic/versions/0010_run_capabilities.py

@@ -0,0 +1,23 @@
+"""persist capability probe results per run
+
+Revision ID: 0010
+Revises: 0009
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+revision = "0010"
+down_revision = "0009"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("test_runs") as batch:
+        batch.add_column(sa.Column("capabilities_json", sa.Text(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("test_runs") as batch:
+        batch.drop_column("capabilities_json")