first commit
This commit is contained in:
baozaotumao2025
@@ -0,0 +1,285 @@
|
||||
"""Inspect a stuck execution and optionally probe target/judge calls without DB writes."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.db.models import TestResult, TestRun
|
||||
from app.db.session import SessionLocal
|
||||
from app.providers.model_provider import ProviderFactory
|
||||
from app.services.dataset_service import DatasetGateway
|
||||
from app.services.test_execution_service import TestExecutionService
|
||||
|
||||
|
||||
def arguments():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--execution-id", default="R0169")
|
||||
parser.add_argument("--run-id", type=int)
|
||||
parser.add_argument(
|
||||
"--probe",
|
||||
action="store_true",
|
||||
help="Actually call target and judge providers; never writes results",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--compare",
|
||||
action="store_true",
|
||||
help="Probe a cumulative schema ladder and stop at the first failure",
|
||||
)
|
||||
parser.add_argument("--timeout", type=float, default=60, help="Outer timeout per probe")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def retry_budget(timeout: float, retries: int, delay: float, max_delay: float) -> float:
|
||||
return timeout * (retries + 1) + sum(
|
||||
min(delay * 2**attempt, max_delay) for attempt in range(retries)
|
||||
)
|
||||
|
||||
|
||||
def elapsed_seconds(value: datetime) -> int:
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
return max(0, int((datetime.now(timezone.utc) - value).total_seconds()))
|
||||
|
||||
|
||||
def find_run(db, run_id: int | None, execution_id: str):
|
||||
if run_id:
|
||||
return db.get(TestRun, run_id)
|
||||
runs = db.scalars(select(TestRun).order_by(TestRun.id.desc())).all()
|
||||
for run in runs:
|
||||
summary = json.loads(run.summary_json or "{}")
|
||||
if execution_id in {
|
||||
summary.get("current_execution_id"),
|
||||
summary.get("retry_execution_id"),
|
||||
}:
|
||||
return run
|
||||
result = db.scalar(
|
||||
select(TestResult)
|
||||
.where(TestResult.execution_id == execution_id)
|
||||
.order_by(TestResult.id.desc())
|
||||
)
|
||||
return db.get(TestRun, result.run_id) if result else None
|
||||
|
||||
|
||||
def build_request(case):
|
||||
model_input = case["model_input"]
|
||||
messages = []
|
||||
system_parts = [model_input["system"]] if model_input.get("system") else []
|
||||
for attachment in model_input.get("attachments", []):
|
||||
if attachment["type"] == "image":
|
||||
messages.append(
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": attachment["file_content"]},
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
else:
|
||||
system_parts.append("测试资料：" + json.dumps(attachment, ensure_ascii=False))
|
||||
if system_parts:
|
||||
messages.insert(0, {"role": "system", "content": "\n\n".join(system_parts)})
|
||||
messages.extend(model_input.get("messages", []))
|
||||
tools = [model_input["tool_schema"]] if model_input.get("tool_schema") else None
|
||||
return messages, tools
|
||||
|
||||
|
||||
def simple_tools(tools):
|
||||
if not tools:
|
||||
return None
|
||||
function = tools[0]["function"]
|
||||
return [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": function["name"],
|
||||
"description": function.get("description", ""),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}},
|
||||
"required": ["query"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def schema_ladder(tools):
|
||||
simple = simple_tools(tools)
|
||||
if not simple:
|
||||
return []
|
||||
original = tools[0]["function"]["parameters"]
|
||||
variants = [("SIMPLE TOOL", simple)]
|
||||
|
||||
query_bounds = copy.deepcopy(simple)
|
||||
query_bounds[0]["function"]["parameters"]["properties"]["query"] = copy.deepcopy(
|
||||
original["properties"]["query"]
|
||||
)
|
||||
variants.append(("+ QUERY BOUNDS", query_bounds))
|
||||
|
||||
closed = copy.deepcopy(query_bounds)
|
||||
closed[0]["function"]["parameters"]["additionalProperties"] = False
|
||||
variants.append(("+ ADDITIONAL PROPERTIES FALSE", closed))
|
||||
|
||||
time_range = copy.deepcopy(closed)
|
||||
parameters = time_range[0]["function"]["parameters"]
|
||||
parameters["properties"]["time_range"] = copy.deepcopy(
|
||||
original["properties"]["time_range"]
|
||||
)
|
||||
parameters["required"].append("time_range")
|
||||
variants.append(("+ TIME RANGE ENUM", time_range))
|
||||
|
||||
source_enum = copy.deepcopy(time_range)
|
||||
source_schema = copy.deepcopy(original["properties"]["source_types"])
|
||||
source_schema.pop("uniqueItems", None)
|
||||
source_schema.pop("maxItems", None)
|
||||
source_enum[0]["function"]["parameters"]["properties"]["source_types"] = source_schema
|
||||
variants.append(("+ SOURCE TYPES ENUM", source_enum))
|
||||
|
||||
variants.append(("+ ARRAY LIMITS (ORIGINAL)", tools))
|
||||
return variants
|
||||
|
||||
|
||||
async def timed(label, awaitable, timeout):
|
||||
started = time.monotonic()
|
||||
try:
|
||||
value = await asyncio.wait_for(awaitable, timeout)
|
||||
except Exception as exc:
|
||||
print(f"{label}: FAIL {time.monotonic() - started:.2f}s {type(exc).__name__}: {exc}")
|
||||
return None
|
||||
print(f"{label}: OK {time.monotonic() - started:.2f}s")
|
||||
return value
|
||||
|
||||
|
||||
async def main():
|
||||
args = arguments()
|
||||
settings = get_settings()
|
||||
case = next(
|
||||
(
|
||||
item
|
||||
for item in DatasetGateway(settings).load()
|
||||
if item["execution_id"] == args.execution_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not case:
|
||||
raise SystemExit(f"Unknown execution_id: {args.execution_id}")
|
||||
|
||||
with SessionLocal() as db:
|
||||
run = find_run(db, args.run_id, args.execution_id)
|
||||
if not run:
|
||||
raise SystemExit("Run not found; pass --run-id explicitly")
|
||||
summary = json.loads(run.summary_json or "{}")
|
||||
result = db.scalar(
|
||||
select(TestResult).where(
|
||||
TestResult.run_id == run.id,
|
||||
TestResult.execution_id == args.execution_id,
|
||||
)
|
||||
)
|
||||
budget = retry_budget(
|
||||
settings.request_timeout_seconds,
|
||||
settings.request_retry_count,
|
||||
settings.request_retry_delay_seconds,
|
||||
settings.request_retry_max_delay_seconds,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"run_id": run.id,
|
||||
"run_status": run.status,
|
||||
"run_updated_seconds_ago": elapsed_seconds(run.updated_at),
|
||||
"phase": summary.get("phase"),
|
||||
"current_execution_id": summary.get("current_execution_id"),
|
||||
"retry_execution_id": summary.get("retry_execution_id"),
|
||||
"auto_judge": summary.get("auto_judge"),
|
||||
"result": (
|
||||
{
|
||||
"execution_status": result.execution_status,
|
||||
"verdict": result.verdict,
|
||||
"error": result.error_message,
|
||||
}
|
||||
if result
|
||||
else None
|
||||
),
|
||||
"case_mode": case.get("interaction_mode"),
|
||||
"has_tool_schema": bool(case["model_input"].get("tool_schema")),
|
||||
"provider_timeout_seconds": settings.request_timeout_seconds,
|
||||
"provider_retry_count": settings.request_retry_count,
|
||||
"worst_case_seconds_per_provider_call": budget,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
if result:
|
||||
print("DIAGNOSIS: execution already has a stored result; refresh the run detail/UI.")
|
||||
elif summary.get("current_execution_id") == args.execution_id:
|
||||
print("DIAGNOSIS: backend is waiting in target or judge call, or its worker was interrupted.")
|
||||
else:
|
||||
print("DIAGNOSIS: execution is queued or the in-process background task was lost.")
|
||||
if not args.probe and not args.compare:
|
||||
print("NEXT: add --compare for target A/B tests or --probe for target and judge.")
|
||||
return
|
||||
|
||||
factory = ProviderFactory(settings, db)
|
||||
target = factory.create("target")
|
||||
messages, tools = build_request(case)
|
||||
print(f"TARGET: endpoint={target.endpoint} model={target.model}")
|
||||
if args.compare:
|
||||
outcomes = {}
|
||||
for label, variant_tools in [("NO TOOLS", None), *schema_ladder(tools)]:
|
||||
reply = await timed(
|
||||
label, target.chat(messages, tools=variant_tools), args.timeout
|
||||
)
|
||||
outcomes[label] = "ok" if reply is not None else "timeout_or_error"
|
||||
if reply is None:
|
||||
break
|
||||
print("A/B RESULT:", json.dumps(outcomes, ensure_ascii=False))
|
||||
failed = next((label for label, result in outcomes.items() if result != "ok"), None)
|
||||
if failed == "NO TOOLS":
|
||||
print("DIAGNOSIS: prompt/model/server path is slow; tool schema is not the cause.")
|
||||
elif failed:
|
||||
print(f"DIAGNOSIS: first failing schema stage is {failed}.")
|
||||
else:
|
||||
print("DIAGNOSIS: the full original schema works now; suspect intermittent load.")
|
||||
return
|
||||
reply = await timed("TARGET", target.chat(messages, tools=tools), args.timeout)
|
||||
if reply is None:
|
||||
return
|
||||
response = reply.content or json.dumps(
|
||||
{"tool_calls": reply.tool_calls}, ensure_ascii=False
|
||||
)
|
||||
print(
|
||||
"TARGET RESULT:",
|
||||
json.dumps(
|
||||
{
|
||||
"content_length": len(reply.content),
|
||||
"tool_call_count": len(reply.tool_calls),
|
||||
"preview": response[:500],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
)
|
||||
if summary.get("auto_judge"):
|
||||
judge = factory.create("judge")
|
||||
print(f"JUDGE: endpoint={judge.endpoint} model={judge.model}")
|
||||
service = TestExecutionService(db=db, target_provider=target, judge_provider=judge)
|
||||
judged = await timed("JUDGE", service._judge(case, response), args.timeout)
|
||||
if judged is not None:
|
||||
print("JUDGE RESULT:", json.dumps(judged, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,131 @@
|
||||
"""Read-only diagnosis for a run stuck on one execution."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import time
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.db.repositories.repositories import ResultRepository, TestRunRepository
|
||||
from app.db.session import SessionLocal
|
||||
from app.providers.model_provider import ProviderFactory
|
||||
from app.services.dataset_service import DatasetGateway
|
||||
from app.services.test_execution_service import TestExecutionService
|
||||
|
||||
|
||||
def arguments():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--run-id", type=int, default=7)
|
||||
parser.add_argument("--execution-id", default="R0229")
|
||||
parser.add_argument(
|
||||
"--timeout", type=float, default=60, help="Each model probe timeout in seconds"
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
async def timed(label, awaitable, timeout):
|
||||
started = time.monotonic()
|
||||
try:
|
||||
result = await asyncio.wait_for(awaitable, timeout)
|
||||
except Exception as exc:
|
||||
elapsed = time.monotonic() - started
|
||||
print(f"{label}: FAIL after {elapsed:.2f}s: {type(exc).__name__}: {exc}")
|
||||
return None
|
||||
print(f"{label}: OK in {time.monotonic() - started:.2f}s")
|
||||
return result
|
||||
|
||||
|
||||
async def main():
|
||||
args = arguments()
|
||||
settings = get_settings()
|
||||
case = next(
|
||||
(
|
||||
item
|
||||
for item in DatasetGateway(settings).load()
|
||||
if item["execution_id"] == args.execution_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not case:
|
||||
raise SystemExit(f"Unknown execution_id: {args.execution_id}")
|
||||
|
||||
with SessionLocal() as db:
|
||||
run = TestRunRepository(db).get(args.run_id)
|
||||
if not run:
|
||||
raise SystemExit(f"Unknown run_id: {args.run_id}")
|
||||
rows = ResultRepository(db).list_by_run(args.run_id)
|
||||
print(
|
||||
"DATABASE:",
|
||||
json.dumps(
|
||||
{
|
||||
"status": run.status,
|
||||
"selected": run.selected_count,
|
||||
"stored_results": len(rows),
|
||||
"execution_stored": any(
|
||||
row.execution_id == args.execution_id for row in rows
|
||||
),
|
||||
"summary": json.loads(run.summary_json or "{}"),
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
)
|
||||
|
||||
model_input = case["model_input"]
|
||||
messages = []
|
||||
system_parts = [model_input["system"]] if model_input.get("system") else []
|
||||
system_parts.extend(
|
||||
"测试资料：" + json.dumps(item, ensure_ascii=False)
|
||||
for item in model_input.get("attachments", [])
|
||||
if item["type"] != "image"
|
||||
)
|
||||
if system_parts:
|
||||
messages.append({"role": "system", "content": "\n\n".join(system_parts)})
|
||||
messages.extend(model_input["messages"])
|
||||
tools = [model_input["tool_schema"]] if model_input.get("tool_schema") else None
|
||||
|
||||
factory = ProviderFactory(settings, db)
|
||||
target = factory.create("target")
|
||||
judge = factory.create("judge")
|
||||
print(f"TARGET: endpoint={target.endpoint} model={target.model}")
|
||||
target_reply = await timed(
|
||||
"TARGET PROBE", target.chat(messages, tools=tools), args.timeout
|
||||
)
|
||||
if target_reply is None:
|
||||
print("DIAGNOSIS: 目标模型请求链失败或超时。")
|
||||
return
|
||||
model_response = target_reply.content or (
|
||||
json.dumps({"tool_calls": target_reply.tool_calls}, ensure_ascii=False)
|
||||
if target_reply.tool_calls
|
||||
else ""
|
||||
)
|
||||
print(
|
||||
"TARGET OUTPUT:",
|
||||
json.dumps(
|
||||
{
|
||||
"content_length": len(target_reply.content),
|
||||
"tool_call_count": len(target_reply.tool_calls),
|
||||
"preview": model_response[:500],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
)
|
||||
|
||||
print(f"JUDGE: endpoint={judge.endpoint} model={judge.model}")
|
||||
service = TestExecutionService(
|
||||
db=db, target_provider=target, judge_provider=judge
|
||||
)
|
||||
judge_result = await timed(
|
||||
"JUDGE PROBE", service._judge(case, model_response), args.timeout
|
||||
)
|
||||
if judge_result is None:
|
||||
print("DIAGNOSIS: 目标模型正常，仲裁模型请求链失败或超时。")
|
||||
return
|
||||
print("JUDGE OUTPUT:", json.dumps(judge_result, ensure_ascii=False))
|
||||
print(
|
||||
"DIAGNOSIS: 两个模型直连均正常；若运行仍卡住，"
|
||||
"优先检查后端进程重启、后台任务丢失或间歇性上游故障。"
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
@@ -0,0 +1,172 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Export one completed safety-test run as a readable Markdown report."""
|
||||
|
||||
import argparse
|
||||
import getpass
|
||||
import json
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
from urllib.error import HTTPError, URLError
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
def request_json(
|
||||
base_url: str,
|
||||
path: str,
|
||||
*,
|
||||
token: str | None = None,
|
||||
payload: dict[str, Any] | None = None,
|
||||
) -> Any:
|
||||
headers = {"Accept": "application/json"}
|
||||
data = None
|
||||
if payload is not None:
|
||||
headers["Content-Type"] = "application/json"
|
||||
data = json.dumps(payload).encode()
|
||||
if token:
|
||||
headers["Authorization"] = f"Bearer {token}"
|
||||
request = Request(base_url.rstrip("/") + path, data=data, headers=headers)
|
||||
try:
|
||||
with urlopen(request, timeout=30) as response:
|
||||
return json.load(response)
|
||||
except HTTPError as exc:
|
||||
body = exc.read().decode("utf-8", errors="replace")
|
||||
try:
|
||||
message = json.loads(body).get("error", {}).get("message", body)
|
||||
except json.JSONDecodeError:
|
||||
message = body
|
||||
raise RuntimeError(f"HTTP {exc.code}: {message}") from exc
|
||||
except URLError as exc:
|
||||
raise RuntimeError(f"无法连接服务：{exc.reason}") from exc
|
||||
|
||||
|
||||
def block(value: Any) -> str:
|
||||
text = value if isinstance(value, str) else json.dumps(value, ensure_ascii=False, indent=2)
|
||||
longest = max((len(part) for part in re.findall(r"`+", text)), default=0)
|
||||
fence = "`" * max(3, longest + 1)
|
||||
return f"{fence}\n{text}\n{fence}"
|
||||
|
||||
|
||||
def render_input(model_input: dict[str, Any]) -> list[str]:
|
||||
sections: list[str] = []
|
||||
if system := model_input.get("system"):
|
||||
sections.extend(["#### System", "", block(system), ""])
|
||||
for message in model_input.get("messages", []):
|
||||
role = str(message.get("role", "unknown")).capitalize()
|
||||
sections.extend([f"#### {role}", "", block(message.get("content", "")), ""])
|
||||
extras = {
|
||||
key: value
|
||||
for key, value in model_input.items()
|
||||
if key not in {"system", "messages"} and value is not None and value != "" and value is not False
|
||||
}
|
||||
if extras:
|
||||
sections.extend(["#### 其他输入信息", "", block(extras), ""])
|
||||
return sections or ["无", ""]
|
||||
|
||||
|
||||
def is_completed_pass(result: dict[str, Any]) -> bool:
|
||||
return result.get("execution_status") == "completed" and result.get("verdict") == "pass"
|
||||
|
||||
|
||||
def render_markdown(run: dict[str, Any], results: list[dict[str, Any]]) -> str:
|
||||
lines = [
|
||||
f"# Run {run['run_id']} 安全测试详细报告",
|
||||
"",
|
||||
"## 运行概况",
|
||||
"",
|
||||
f"- 状态：{run.get('status', 'unknown')}",
|
||||
f"- 阶段：{run.get('phase', 'unknown')}",
|
||||
f"- 样例总数：{run.get('selected_count', len(results))}",
|
||||
f"- 已完成：{run.get('completed_count', 0)}",
|
||||
f"- 执行错误：{run.get('error_count', 0)}",
|
||||
f"- 开始时间：{run.get('started_at', '')}",
|
||||
f"- 完成时间：{run.get('finished_at', '')}",
|
||||
"",
|
||||
f"- 本报告 Pass 样例：{len(results)}",
|
||||
"",
|
||||
"## Pass 样例详情",
|
||||
"",
|
||||
]
|
||||
for index, result in enumerate(results, 1):
|
||||
judge = result.get("judge_result") or {}
|
||||
lines.extend(
|
||||
[
|
||||
f"### {index}. {result.get('execution_id', 'unknown')}",
|
||||
"",
|
||||
f"- 测试类型：{result.get('case_kind', 'unknown')}",
|
||||
f"- 交互模式：{result.get('interaction_mode', 'unknown')}",
|
||||
f"- 执行状态：{result.get('execution_status', 'unknown')}",
|
||||
f"- 仲裁结论：{result.get('verdict') or '未仲裁'}",
|
||||
"",
|
||||
"#### 模型输入",
|
||||
"",
|
||||
*render_input(result.get("model_input") or {}),
|
||||
"#### 模型输出",
|
||||
"",
|
||||
block(result.get("model_response", "")),
|
||||
"",
|
||||
"#### 仲裁结果",
|
||||
"",
|
||||
]
|
||||
)
|
||||
if judge:
|
||||
lines.extend(
|
||||
[
|
||||
f"- 结论：{judge.get('verdict', result.get('verdict', 'unknown'))}",
|
||||
f"- 评分：{judge.get('score', '无')}",
|
||||
f"- 理由：{judge.get('reason', '无')}",
|
||||
]
|
||||
)
|
||||
if raw := judge.get("raw_response"):
|
||||
lines.extend(["", "##### 仲裁原始输出", "", block(raw)])
|
||||
else:
|
||||
lines.append("未启用自动仲裁或没有仲裁结果。")
|
||||
if error := result.get("error_message"):
|
||||
lines.extend(["", "#### 执行错误", "", block(error)])
|
||||
lines.extend(["", "---", ""])
|
||||
return "\n".join(lines).rstrip() + "\n"
|
||||
|
||||
|
||||
def parse_args() -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(description="将已完成的 run 导出为 Markdown 报告")
|
||||
parser.add_argument("--run-id", type=int, default=7, help="要导出的 run ID（默认：7）")
|
||||
parser.add_argument("--base-url", default="http://127.0.0.1:8000", help="API 服务地址")
|
||||
parser.add_argument("--username", help="登录用户名；省略时交互输入")
|
||||
parser.add_argument("--output", type=Path, help="输出文件路径")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def main() -> int:
|
||||
args = parse_args()
|
||||
if args.run_id < 1:
|
||||
print("错误：--run-id 必须是正整数", file=sys.stderr)
|
||||
return 2
|
||||
username = args.username or input("Username: ").strip()
|
||||
password = getpass.getpass("Password: ")
|
||||
try:
|
||||
login = request_json(
|
||||
args.base_url,
|
||||
"/api/v1/auth/login",
|
||||
payload={"username": username, "password": password},
|
||||
)
|
||||
token = login["access_token"]
|
||||
run = request_json(args.base_url, f"/api/v1/runs/{args.run_id}", token=token)
|
||||
if not run.get("terminal"):
|
||||
raise RuntimeError(f"Run {args.run_id} 尚未结束，当前状态：{run.get('status')}")
|
||||
results = request_json(
|
||||
args.base_url, f"/api/v1/runs/{args.run_id}/results", token=token
|
||||
)
|
||||
results = [result for result in results if is_completed_pass(result)]
|
||||
output = args.output or Path("outputs") / f"run_{args.run_id}_pass_results.md"
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text(render_markdown(run, results), encoding="utf-8")
|
||||
except (RuntimeError, KeyError, OSError) as exc:
|
||||
print(f"导出失败：{exc}", file=sys.stderr)
|
||||
return 1
|
||||
print(f"导出完成：{output}（共 {len(results)} 个测试样例）")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Executable
+4
@@ -0,0 +1,4 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
mkdir -p data
|
||||
uv run python -m app.init_db
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
uv run alembic upgrade head
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Read-only rerun of Run 7's 15 tool-call cases requiring manual review."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.db.session import SessionLocal
|
||||
from app.providers.model_provider import ProviderFactory
|
||||
from app.services.dataset_service import DatasetGateway
|
||||
from app.services.test_execution_service import TestExecutionService
|
||||
|
||||
|
||||
EXECUTION_IDS = {
|
||||
"R0049",
|
||||
"R0079",
|
||||
"R0089",
|
||||
"R0099",
|
||||
"R0109",
|
||||
"R0119",
|
||||
"R0129",
|
||||
"R0139",
|
||||
"R0149",
|
||||
"R0159",
|
||||
"R0169",
|
||||
"R0189",
|
||||
"R0199",
|
||||
"R0209",
|
||||
"R0245",
|
||||
}
|
||||
|
||||
|
||||
def arguments():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--timeout", type=float, default=180, help="Per request timeout in seconds")
|
||||
parser.add_argument(
|
||||
"--output",
|
||||
type=Path,
|
||||
default=Path("outputs/run_7_manual_review_rerun.json"),
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
async def main():
|
||||
args = arguments()
|
||||
settings = get_settings()
|
||||
cases = {
|
||||
case["execution_id"]: case
|
||||
for case in DatasetGateway(settings).load()
|
||||
if case["execution_id"] in EXECUTION_IDS
|
||||
}
|
||||
missing = EXECUTION_IDS - cases.keys()
|
||||
if missing:
|
||||
raise SystemExit(f"Dataset missing execution IDs: {', '.join(sorted(missing))}")
|
||||
|
||||
results = []
|
||||
with SessionLocal() as db:
|
||||
factory = ProviderFactory(settings, db)
|
||||
target = factory.create("target")
|
||||
judge = factory.create("judge")
|
||||
service = TestExecutionService(db=db, target_provider=target, judge_provider=judge)
|
||||
|
||||
for index, execution_id in enumerate(sorted(EXECUTION_IDS), 1):
|
||||
case = cases[execution_id]
|
||||
model_input = case["model_input"]
|
||||
print(f"[{index:02d}/15] {execution_id}", flush=True)
|
||||
item = {"execution_id": execution_id}
|
||||
try:
|
||||
reply = await asyncio.wait_for(
|
||||
target.chat(
|
||||
model_input["messages"],
|
||||
tools=[model_input["tool_schema"]],
|
||||
),
|
||||
args.timeout,
|
||||
)
|
||||
model_response = reply.content or (
|
||||
json.dumps({"tool_calls": reply.tool_calls}, ensure_ascii=False)
|
||||
if reply.tool_calls
|
||||
else ""
|
||||
)
|
||||
item.update(
|
||||
content=reply.content,
|
||||
tool_calls=reply.tool_calls,
|
||||
model_response=model_response,
|
||||
raw_response=reply.raw,
|
||||
judge_result=await asyncio.wait_for(
|
||||
service._judge(case, model_response), args.timeout
|
||||
),
|
||||
)
|
||||
except Exception as exc:
|
||||
item["error"] = f"{type(exc).__name__}: {exc}"
|
||||
results.append(item)
|
||||
|
||||
args.output.parent.mkdir(parents=True, exist_ok=True)
|
||||
args.output.write_text(json.dumps(results, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
print(f"Saved {len(results)} results to {args.output}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Executable
+3
@@ -0,0 +1,3 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
uv run alembic downgrade -1
|
||||
Reference in New Issue
Block a user