first commit

2026-07-18 21:00:26 +08:00
commit 14722be770
105 changed files with 25004 additions and 0 deletions
@@ -0,0 +1,104 @@
+"""Read-only rerun of Run 7's 15 tool-call cases requiring manual review."""
+
+import argparse
+import asyncio
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+
+from app.core.config import get_settings
+from app.db.session import SessionLocal
+from app.providers.model_provider import ProviderFactory
+from app.services.dataset_service import DatasetGateway
+from app.services.test_execution_service import TestExecutionService
+
+
+EXECUTION_IDS = {
+    "R0049",
+    "R0079",
+    "R0089",
+    "R0099",
+    "R0109",
+    "R0119",
+    "R0129",
+    "R0139",
+    "R0149",
+    "R0159",
+    "R0169",
+    "R0189",
+    "R0199",
+    "R0209",
+    "R0245",
+}
+
+
+def arguments():
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--timeout", type=float, default=180, help="Per request timeout in seconds")
+    parser.add_argument(
+        "--output",
+        type=Path,
+        default=Path("outputs/run_7_manual_review_rerun.json"),
+    )
+    return parser.parse_args()
+
+
+async def main():
+    args = arguments()
+    settings = get_settings()
+    cases = {
+        case["execution_id"]: case
+        for case in DatasetGateway(settings).load()
+        if case["execution_id"] in EXECUTION_IDS
+    }
+    missing = EXECUTION_IDS - cases.keys()
+    if missing:
+        raise SystemExit(f"Dataset missing execution IDs: {', '.join(sorted(missing))}")
+
+    results = []
+    with SessionLocal() as db:
+        factory = ProviderFactory(settings, db)
+        target = factory.create("target")
+        judge = factory.create("judge")
+        service = TestExecutionService(db=db, target_provider=target, judge_provider=judge)
+
+        for index, execution_id in enumerate(sorted(EXECUTION_IDS), 1):
+            case = cases[execution_id]
+            model_input = case["model_input"]
+            print(f"[{index:02d}/15] {execution_id}", flush=True)
+            item = {"execution_id": execution_id}
+            try:
+                reply = await asyncio.wait_for(
+                    target.chat(
+                        model_input["messages"],
+                        tools=[model_input["tool_schema"]],
+                    ),
+                    args.timeout,
+                )
+                model_response = reply.content or (
+                    json.dumps({"tool_calls": reply.tool_calls}, ensure_ascii=False)
+                    if reply.tool_calls
+                    else ""
+                )
+                item.update(
+                    content=reply.content,
+                    tool_calls=reply.tool_calls,
+                    model_response=model_response,
+                    raw_response=reply.raw,
+                    judge_result=await asyncio.wait_for(
+                        service._judge(case, model_response), args.timeout
+                    ),
+                )
+            except Exception as exc:
+                item["error"] = f"{type(exc).__name__}: {exc}"
+            results.append(item)
+
+    args.output.parent.mkdir(parents=True, exist_ok=True)
+    args.output.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
+    print(f"Saved {len(results)} results to {args.output}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())