diff --git a/.DS_Store b/.DS_Store index a4b6d84..da671af 100644 Binary files a/.DS_Store and b/.DS_Store differ diff --git a/.coverage b/.coverage index 1692fd1..897994e 100644 Binary files a/.coverage and b/.coverage differ diff --git a/README.md b/README.md index fecd64c..d9411ce 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,14 @@ 逐条结果用 `execution_status`(`completed|error`)表示执行状态,用可空 `verdict` 表示仲裁结论;Run 的 `error_count` 不包含 `verdict=fail`。`resume` 只补跑未落库样例;`retry-errors` 只重跑所有 `execution_status=error`;`results/{execution_id}/retry` 可在终态 Run 中只重跑指定结果。 报告是运行与逐条结果的实时派生视图:由创建 run 隐式产生,不单独 POST 或 PATCH;删除终态 run 时报告随源数据一起消失。 +终态 Run 可导出包含模型输入、输出和裁判结果的 Markdown 详细报告;默认导出全部样例: + +```bash +uv run python scripts/export_run.py --run-id 42 +``` + +使用 `--execution-status completed|error` 按执行状态过滤,使用 `--verdict pass|fail|needs_human_review|judge_format_error|none` 按评价结果过滤;两者可组合。默认输出为 `outputs/run__results.md`。 + ## 快速启动 ```bash diff --git a/docs/tutorial/getting-started.md b/docs/tutorial/getting-started.md index bff9029..aee117d 100644 --- a/docs/tutorial/getting-started.md +++ b/docs/tutorial/getting-started.md @@ -105,4 +105,12 @@ curl -X POST http://127.0.0.1:8000/api/v1/runs \ 报告不单独保存:`POST /runs` 创建运行后即可查询实时报告,结果变化时报告自动重算,因此没有报告 POST/PATCH。删除终态 run 会同时删除其结果,之后对应报告返回 404。 +运行结束后可将详细结果导出为 Markdown: + +```bash +uv run python scripts/export_run.py --run-id 42 +``` + +默认导出全部样例到 `outputs/run_42_results.md`。仅导出执行错误时添加 `--execution-status error`;仅导出评价失败时添加 `--verdict fail`;两个参数可组合。脚本会交互请求用户名和密码,还可用 `--base-url`、`--username` 和 `--output` 指定服务地址、用户名和输出路径。 + 取消运行使用 `PATCH /api/v1/runs/{run_id}` 和 `{"status":"cancelled"}`。取消是协作式的:正在进行的单次模型请求结束后停止后续样例。终态运行可用 `DELETE /api/v1/runs/{run_id}` 删除,运行中删除返回 409。 diff --git a/docs/tutorial/operations.md b/docs/tutorial/operations.md index e71e229..330324d 100644 --- a/docs/tutorial/operations.md +++ b/docs/tutorial/operations.md @@ -126,6 +126,33 @@ curl -X POST http://127.0.0.1:8000/api/v1/runs/42/results/R0049/retry 报告不是省略题目结果的替代品:逐条输入、原始模型回复、裁判结果和错误信息在 `GET /api/v1/runs/{run_id}/results` 返回。报告同时返回当前已执行题目的 `test_result` 和发布准入结论。 +### 导出 Markdown 详细报告 + +`scripts/export_run.py` 通过 API 登录并导出一个终态 Run。默认包含全部逐条结果: + +```bash +uv run python scripts/export_run.py \ + --run-id 42 \ + --base-url http://127.0.0.1:8000 \ + --username gly +``` + +默认输出为 `outputs/run_42_results.md`;用 `--output ` 可更改路径。密码始终交互输入,不会出现在命令行历史中。 + +可用以下参数筛选: + +- `--execution-status completed|error`:按执行状态导出。 +- `--verdict pass|fail|needs_human_review|judge_format_error|none`:按评价结果导出,`none` 表示未评价。 + +两个参数可组合,例如只导出已执行且评价失败的样例: + +```bash +uv run python scripts/export_run.py \ + --run-id 42 \ + --execution-status completed \ + --verdict fail +``` + `data/admission_gate_config.json` 只配置可由当前运行验证的覆盖门槛和指标阈值。覆盖门槛为至少 451 个执行 实例、328 个风险题、123 个对照题、31 个标准方向,以及单轮、多轮、工具、多模态、RF 和 OC 的最低数量。 运行必须完成且结果数等于选中题数;任一覆盖项不足时返回 `NOT_EVALUATED`,并在 `coverage` 中返回 diff --git a/scripts/export_run.py b/scripts/export_run.py index 9798ee1..751ae4a 100644 --- a/scripts/export_run.py +++ b/scripts/export_run.py @@ -65,8 +65,12 @@ def render_input(model_input: dict[str, Any]) -> list[str]: return sections or ["无", ""] -def is_completed_pass(result: dict[str, Any]) -> bool: - return result.get("execution_status") == "completed" and result.get("verdict") == "pass" +def matches_result( + result: dict[str, Any], execution_status: str | None, verdict: str | None +) -> bool: + return (execution_status is None or result.get("execution_status") == execution_status) and ( + verdict is None or result.get("verdict") == verdict + ) def render_markdown(run: dict[str, Any], results: list[dict[str, Any]]) -> str: @@ -83,9 +87,9 @@ def render_markdown(run: dict[str, Any], results: list[dict[str, Any]]) -> str: f"- 开始时间:{run.get('started_at', '')}", f"- 完成时间:{run.get('finished_at', '')}", "", - f"- 本报告 Pass 样例:{len(results)}", + f"- 本报告导出样例:{len(results)}", "", - "## Pass 样例详情", + "## 测试样例详情", "", ] for index, result in enumerate(results, 1): @@ -133,6 +137,16 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--run-id", type=int, default=7, help="要导出的 run ID(默认:7)") parser.add_argument("--base-url", default="http://127.0.0.1:8000", help="API 服务地址") parser.add_argument("--username", help="登录用户名;省略时交互输入") + parser.add_argument( + "--execution-status", + choices=("completed", "error"), + help="只导出指定执行状态;默认导出全部", + ) + parser.add_argument( + "--verdict", + choices=("pass", "fail", "needs_human_review", "judge_format_error", "none"), + help="只导出指定评价结果;none 表示未评价;默认导出全部", + ) parser.add_argument("--output", type=Path, help="输出文件路径") return parser.parse_args() @@ -157,8 +171,18 @@ def main() -> int: results = request_json( args.base_url, f"/api/v1/runs/{args.run_id}/results", token=token ) - results = [result for result in results if is_completed_pass(result)] - output = args.output or Path("outputs") / f"run_{args.run_id}_pass_results.md" + verdict = None if args.verdict is None else args.verdict + results = [ + result + for result in results + if matches_result( + result, + args.execution_status, + None if verdict == "none" else verdict, + ) + and (verdict != "none" or result.get("verdict") is None) + ] + output = args.output or Path("outputs") / f"run_{args.run_id}_results.md" output.parent.mkdir(parents=True, exist_ok=True) output.write_text(render_markdown(run, results), encoding="utf-8") except (RuntimeError, KeyError, OSError) as exc: diff --git a/tests/test_export_run.py b/tests/test_export_run.py index 196726a..f0c41ea 100644 --- a/tests/test_export_run.py +++ b/tests/test_export_run.py @@ -2,7 +2,9 @@ from pathlib import Path from runpy import run_path -render_markdown = run_path(Path(__file__).parents[1] / "scripts/export_run.py")["render_markdown"] +exports = run_path(Path(__file__).parents[1] / "scripts/export_run.py") +matches_result = exports["matches_result"] +render_markdown = exports["render_markdown"] def test_render_markdown_contains_input_output_and_judgement(): @@ -24,3 +26,12 @@ def test_render_markdown_contains_input_output_and_judgement(): ) assert all(value in text for value in ["测试输入", "模型输出", "fail", "0.9", "仲裁理由"]) + + +def test_result_filters_default_to_all_and_combine(): + passed = {"execution_status": "completed", "verdict": "pass"} + + assert matches_result(passed, None, None) + assert matches_result(passed, "completed", "pass") + assert not matches_result(passed, "error", None) + assert not matches_result(passed, None, "fail")