baozaotumao2025
105 lines
3.2 KiB
Python
105 lines
3.2 KiB
Python
"""Read-only rerun of Run 7's 15 tool-call cases requiring manual review."""
|
|
|
|
import argparse
|
|
import asyncio
|
|
import json
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
|
|
from app.core.config import get_settings
|
|
from app.db.session import SessionLocal
|
|
from app.providers.model_provider import ProviderFactory
|
|
from app.services.dataset_service import DatasetGateway
|
|
from app.services.test_execution_service import TestExecutionService
|
|
|
|
|
|
EXECUTION_IDS = {
|
|
"R0049",
|
|
"R0079",
|
|
"R0089",
|
|
"R0099",
|
|
"R0109",
|
|
"R0119",
|
|
"R0129",
|
|
"R0139",
|
|
"R0149",
|
|
"R0159",
|
|
"R0169",
|
|
"R0189",
|
|
"R0199",
|
|
"R0209",
|
|
"R0245",
|
|
}
|
|
|
|
|
|
def arguments():
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument("--timeout", type=float, default=180, help="Per request timeout in seconds")
|
|
parser.add_argument(
|
|
"--output",
|
|
type=Path,
|
|
default=Path("outputs/run_7_manual_review_rerun.json"),
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
async def main():
|
|
args = arguments()
|
|
settings = get_settings()
|
|
cases = {
|
|
case["execution_id"]: case
|
|
for case in DatasetGateway(settings).load()
|
|
if case["execution_id"] in EXECUTION_IDS
|
|
}
|
|
missing = EXECUTION_IDS - cases.keys()
|
|
if missing:
|
|
raise SystemExit(f"Dataset missing execution IDs: {', '.join(sorted(missing))}")
|
|
|
|
results = []
|
|
with SessionLocal() as db:
|
|
factory = ProviderFactory(settings, db)
|
|
target = factory.create("target")
|
|
judge = factory.create("judge")
|
|
service = TestExecutionService(db=db, target_provider=target, judge_provider=judge)
|
|
|
|
for index, execution_id in enumerate(sorted(EXECUTION_IDS), 1):
|
|
case = cases[execution_id]
|
|
model_input = case["model_input"]
|
|
print(f"[{index:02d}/15] {execution_id}", flush=True)
|
|
item = {"execution_id": execution_id}
|
|
try:
|
|
reply = await asyncio.wait_for(
|
|
target.chat(
|
|
model_input["messages"],
|
|
tools=[model_input["tool_schema"]],
|
|
),
|
|
args.timeout,
|
|
)
|
|
model_response = reply.content or (
|
|
json.dumps({"tool_calls": reply.tool_calls}, ensure_ascii=False)
|
|
if reply.tool_calls
|
|
else ""
|
|
)
|
|
item.update(
|
|
content=reply.content,
|
|
tool_calls=reply.tool_calls,
|
|
model_response=model_response,
|
|
raw_response=reply.raw,
|
|
judge_result=await asyncio.wait_for(
|
|
service._judge(case, model_response), args.timeout
|
|
),
|
|
)
|
|
except Exception as exc:
|
|
item["error"] = f"{type(exc).__name__}: {exc}"
|
|
results.append(item)
|
|
|
|
args.output.parent.mkdir(parents=True, exist_ok=True)
|
|
args.output.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
|
|
print(f"Saved {len(results)} results to {args.output}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(main())
|