first commit
This commit is contained in:
baozaotumao2025
@@ -0,0 +1,285 @@
|
||||
"""Inspect a stuck execution and optionally probe target/judge calls without DB writes."""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import copy
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from app.core.config import get_settings
|
||||
from app.db.models import TestResult, TestRun
|
||||
from app.db.session import SessionLocal
|
||||
from app.providers.model_provider import ProviderFactory
|
||||
from app.services.dataset_service import DatasetGateway
|
||||
from app.services.test_execution_service import TestExecutionService
|
||||
|
||||
|
||||
def arguments():
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--execution-id", default="R0169")
|
||||
parser.add_argument("--run-id", type=int)
|
||||
parser.add_argument(
|
||||
"--probe",
|
||||
action="store_true",
|
||||
help="Actually call target and judge providers; never writes results",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--compare",
|
||||
action="store_true",
|
||||
help="Probe a cumulative schema ladder and stop at the first failure",
|
||||
)
|
||||
parser.add_argument("--timeout", type=float, default=60, help="Outer timeout per probe")
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
def retry_budget(timeout: float, retries: int, delay: float, max_delay: float) -> float:
|
||||
return timeout * (retries + 1) + sum(
|
||||
min(delay * 2**attempt, max_delay) for attempt in range(retries)
|
||||
)
|
||||
|
||||
|
||||
def elapsed_seconds(value: datetime) -> int:
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=timezone.utc)
|
||||
return max(0, int((datetime.now(timezone.utc) - value).total_seconds()))
|
||||
|
||||
|
||||
def find_run(db, run_id: int | None, execution_id: str):
|
||||
if run_id:
|
||||
return db.get(TestRun, run_id)
|
||||
runs = db.scalars(select(TestRun).order_by(TestRun.id.desc())).all()
|
||||
for run in runs:
|
||||
summary = json.loads(run.summary_json or "{}")
|
||||
if execution_id in {
|
||||
summary.get("current_execution_id"),
|
||||
summary.get("retry_execution_id"),
|
||||
}:
|
||||
return run
|
||||
result = db.scalar(
|
||||
select(TestResult)
|
||||
.where(TestResult.execution_id == execution_id)
|
||||
.order_by(TestResult.id.desc())
|
||||
)
|
||||
return db.get(TestRun, result.run_id) if result else None
|
||||
|
||||
|
||||
def build_request(case):
|
||||
model_input = case["model_input"]
|
||||
messages = []
|
||||
system_parts = [model_input["system"]] if model_input.get("system") else []
|
||||
for attachment in model_input.get("attachments", []):
|
||||
if attachment["type"] == "image":
|
||||
messages.append(
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "image_url",
|
||||
"image_url": {"url": attachment["file_content"]},
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
else:
|
||||
system_parts.append("测试资料：" + json.dumps(attachment, ensure_ascii=False))
|
||||
if system_parts:
|
||||
messages.insert(0, {"role": "system", "content": "\n\n".join(system_parts)})
|
||||
messages.extend(model_input.get("messages", []))
|
||||
tools = [model_input["tool_schema"]] if model_input.get("tool_schema") else None
|
||||
return messages, tools
|
||||
|
||||
|
||||
def simple_tools(tools):
|
||||
if not tools:
|
||||
return None
|
||||
function = tools[0]["function"]
|
||||
return [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": function["name"],
|
||||
"description": function.get("description", ""),
|
||||
"parameters": {
|
||||
"type": "object",
|
||||
"properties": {"query": {"type": "string"}},
|
||||
"required": ["query"],
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def schema_ladder(tools):
|
||||
simple = simple_tools(tools)
|
||||
if not simple:
|
||||
return []
|
||||
original = tools[0]["function"]["parameters"]
|
||||
variants = [("SIMPLE TOOL", simple)]
|
||||
|
||||
query_bounds = copy.deepcopy(simple)
|
||||
query_bounds[0]["function"]["parameters"]["properties"]["query"] = copy.deepcopy(
|
||||
original["properties"]["query"]
|
||||
)
|
||||
variants.append(("+ QUERY BOUNDS", query_bounds))
|
||||
|
||||
closed = copy.deepcopy(query_bounds)
|
||||
closed[0]["function"]["parameters"]["additionalProperties"] = False
|
||||
variants.append(("+ ADDITIONAL PROPERTIES FALSE", closed))
|
||||
|
||||
time_range = copy.deepcopy(closed)
|
||||
parameters = time_range[0]["function"]["parameters"]
|
||||
parameters["properties"]["time_range"] = copy.deepcopy(
|
||||
original["properties"]["time_range"]
|
||||
)
|
||||
parameters["required"].append("time_range")
|
||||
variants.append(("+ TIME RANGE ENUM", time_range))
|
||||
|
||||
source_enum = copy.deepcopy(time_range)
|
||||
source_schema = copy.deepcopy(original["properties"]["source_types"])
|
||||
source_schema.pop("uniqueItems", None)
|
||||
source_schema.pop("maxItems", None)
|
||||
source_enum[0]["function"]["parameters"]["properties"]["source_types"] = source_schema
|
||||
variants.append(("+ SOURCE TYPES ENUM", source_enum))
|
||||
|
||||
variants.append(("+ ARRAY LIMITS (ORIGINAL)", tools))
|
||||
return variants
|
||||
|
||||
|
||||
async def timed(label, awaitable, timeout):
|
||||
started = time.monotonic()
|
||||
try:
|
||||
value = await asyncio.wait_for(awaitable, timeout)
|
||||
except Exception as exc:
|
||||
print(f"{label}: FAIL {time.monotonic() - started:.2f}s {type(exc).__name__}: {exc}")
|
||||
return None
|
||||
print(f"{label}: OK {time.monotonic() - started:.2f}s")
|
||||
return value
|
||||
|
||||
|
||||
async def main():
|
||||
args = arguments()
|
||||
settings = get_settings()
|
||||
case = next(
|
||||
(
|
||||
item
|
||||
for item in DatasetGateway(settings).load()
|
||||
if item["execution_id"] == args.execution_id
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not case:
|
||||
raise SystemExit(f"Unknown execution_id: {args.execution_id}")
|
||||
|
||||
with SessionLocal() as db:
|
||||
run = find_run(db, args.run_id, args.execution_id)
|
||||
if not run:
|
||||
raise SystemExit("Run not found; pass --run-id explicitly")
|
||||
summary = json.loads(run.summary_json or "{}")
|
||||
result = db.scalar(
|
||||
select(TestResult).where(
|
||||
TestResult.run_id == run.id,
|
||||
TestResult.execution_id == args.execution_id,
|
||||
)
|
||||
)
|
||||
budget = retry_budget(
|
||||
settings.request_timeout_seconds,
|
||||
settings.request_retry_count,
|
||||
settings.request_retry_delay_seconds,
|
||||
settings.request_retry_max_delay_seconds,
|
||||
)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"run_id": run.id,
|
||||
"run_status": run.status,
|
||||
"run_updated_seconds_ago": elapsed_seconds(run.updated_at),
|
||||
"phase": summary.get("phase"),
|
||||
"current_execution_id": summary.get("current_execution_id"),
|
||||
"retry_execution_id": summary.get("retry_execution_id"),
|
||||
"auto_judge": summary.get("auto_judge"),
|
||||
"result": (
|
||||
{
|
||||
"execution_status": result.execution_status,
|
||||
"verdict": result.verdict,
|
||||
"error": result.error_message,
|
||||
}
|
||||
if result
|
||||
else None
|
||||
),
|
||||
"case_mode": case.get("interaction_mode"),
|
||||
"has_tool_schema": bool(case["model_input"].get("tool_schema")),
|
||||
"provider_timeout_seconds": settings.request_timeout_seconds,
|
||||
"provider_retry_count": settings.request_retry_count,
|
||||
"worst_case_seconds_per_provider_call": budget,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
)
|
||||
)
|
||||
if result:
|
||||
print("DIAGNOSIS: execution already has a stored result; refresh the run detail/UI.")
|
||||
elif summary.get("current_execution_id") == args.execution_id:
|
||||
print("DIAGNOSIS: backend is waiting in target or judge call, or its worker was interrupted.")
|
||||
else:
|
||||
print("DIAGNOSIS: execution is queued or the in-process background task was lost.")
|
||||
if not args.probe and not args.compare:
|
||||
print("NEXT: add --compare for target A/B tests or --probe for target and judge.")
|
||||
return
|
||||
|
||||
factory = ProviderFactory(settings, db)
|
||||
target = factory.create("target")
|
||||
messages, tools = build_request(case)
|
||||
print(f"TARGET: endpoint={target.endpoint} model={target.model}")
|
||||
if args.compare:
|
||||
outcomes = {}
|
||||
for label, variant_tools in [("NO TOOLS", None), *schema_ladder(tools)]:
|
||||
reply = await timed(
|
||||
label, target.chat(messages, tools=variant_tools), args.timeout
|
||||
)
|
||||
outcomes[label] = "ok" if reply is not None else "timeout_or_error"
|
||||
if reply is None:
|
||||
break
|
||||
print("A/B RESULT:", json.dumps(outcomes, ensure_ascii=False))
|
||||
failed = next((label for label, result in outcomes.items() if result != "ok"), None)
|
||||
if failed == "NO TOOLS":
|
||||
print("DIAGNOSIS: prompt/model/server path is slow; tool schema is not the cause.")
|
||||
elif failed:
|
||||
print(f"DIAGNOSIS: first failing schema stage is {failed}.")
|
||||
else:
|
||||
print("DIAGNOSIS: the full original schema works now; suspect intermittent load.")
|
||||
return
|
||||
reply = await timed("TARGET", target.chat(messages, tools=tools), args.timeout)
|
||||
if reply is None:
|
||||
return
|
||||
response = reply.content or json.dumps(
|
||||
{"tool_calls": reply.tool_calls}, ensure_ascii=False
|
||||
)
|
||||
print(
|
||||
"TARGET RESULT:",
|
||||
json.dumps(
|
||||
{
|
||||
"content_length": len(reply.content),
|
||||
"tool_call_count": len(reply.tool_calls),
|
||||
"preview": response[:500],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
)
|
||||
if summary.get("auto_judge"):
|
||||
judge = factory.create("judge")
|
||||
print(f"JUDGE: endpoint={judge.endpoint} model={judge.model}")
|
||||
service = TestExecutionService(db=db, target_provider=target, judge_provider=judge)
|
||||
judged = await timed("JUDGE", service._judge(case, response), args.timeout)
|
||||
if judged is not None:
|
||||
print("JUDGE RESULT:", json.dumps(judged, ensure_ascii=False))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user