From c57ec67fe3562d7ddee6d10eff41cb5cbb295a92 Mon Sep 17 00:00:00 2001 From: baozaotumao2025 Date: Sat, 18 Jul 2026 23:28:13 +0800 Subject: [PATCH] =?UTF-8?q?feature:=20scripts/export=5Frun.py=20=E7=94=A8?= =?UTF-8?q?=E6=9D=A5=E5=AF=BC=E5=87=BA=E8=BF=90=E8=A1=8C=E7=BB=93=E6=9E=9C?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .DS_Store | Bin 6148 -> 8196 bytes .coverage | Bin 53248 -> 53248 bytes README.md | 8 +++++++ docs/tutorial/getting-started.md | 8 +++++++ docs/tutorial/operations.md | 27 +++++++++++++++++++++++ scripts/export_run.py | 36 +++++++++++++++++++++++++------ tests/test_export_run.py | 13 ++++++++++- 7 files changed, 85 insertions(+), 7 deletions(-) diff --git a/.DS_Store b/.DS_Store index a4b6d84d8a8ebb3d36d68cf5a31ce9c960a76fea..da671af97db05d92c1769a111ca572859f9723e9 100644 GIT binary patch literal 8196 zcmeHMTWl0n7(U;$&>1_(v{ER@vMUQnptVcOrBKABEtg6w$hP!CvF>(;c4RtJcV@R- zQY|qWZ;5vkpYVb{5;Ynz27*4BC^4FbL=uV72l2%xUeJjCXU;6p77|U20iBb~`RDwX zGxPuZedqK{8DnV7=_?qkWQ@slajBJ3ca6sF{F>6FKvPT-6wg?WdCbj>xWgH1ig&08 zD-c#7tUy?SumWKPZiEWZp3R$loqbcJ8i3Ktl$T1=PP?}Sm9uR~gIATDAlRd`WNhSk1CZq^w5aA5LlM$Ry;6I(> zV!k^=Qpm6mD-c#-Y6UobN|@uH+=s6|zdISS9M23}rt2jVXCM_7Pn%vMm&m2^p46Z- zlJe4C&TUL*ck@xZ<5-!(eLAztG)DTgSxt`ZrA^z&`T?eHkV|i;Y3t5Nqm#3B*N;sJ zgRIC(pH_M3P+k3+s#x9X#AsFQ(AwHWRcvM5>d{d}u87w(?CjfT4cX4qA`t|O0h=)? zv1g`cWx4sLjK#AmN)@s?TgXberBs~}>Fnz6QG2`0e#cIk2aP^$rc0@{&~QiA$Wuz4 z{f?CzvfVzdtUqn_=d846Y_u%yOuCtFf7Y?A9gb^yrek+^dB%RPFwq24SC8vty(WLM zK26KH#!iFgGV?|=%}7&~^QUbe=F_;ljI2vCo=k`&m1DQgi7r}NUAty|a#Ks&{o^yW zvT{|enBAE-T{B}D-D$VquyxZO=om3|Z?K2zYkTHi)5t1H8LHat^pG)LUO9K(?eilM z;d3RXFiTf1V?LZSy#rcYmc=t73*(Bceb}qhOSU;WpHcTMk;uY3WqCK(H)Rd#g<6d) zucCl-yGsQMUNf;mQ8c%GAAvxGs;gGZil%p1>5O3os&x&d+SZvDnFq>>dt{}n-x(UB zWL%+(G&Rez_W1++!L4c6+dgdAmtU!At1S1pgXyfXeS{!bh-j5%|A!?#580%fo|E;f zao@s7q?=E*QP&NflAc#Nw>*-HbjaE_6r-7T-Xze|Zy~TR#|x;7Czpxa_$mE$|IV+m zGOOSaNWRHM*b-Jxn_4^DOSC)8o@Gba2kZnp$xgE~>>NAKeqk5cuk1H=2}Mv*j`>)G zYSbWs6&f{lX!0)((Kcq5gp0r3>EG?00q=Zx})k&MBEz%CDTgpfS(xAvd zDiChZ#LT1vG@i7K5aH-EASax3?AnBjZrR#$uiARU6YkqvImK?9RWW-`Z1J+1mGyUT zybMHv+X7Utrht$730(0}?;~R}Vtp>E3!)uF3uFHkG754QD5;fj`pVmXmwPRh^u0lh}NoF5iwEmV^>A1RSmKjtW_ISt(Z6|h8xrcs>T#C zXpA;THO?^kpT_#v>@549uzrDE#B`KGLljGK7wQSyo3RxwXhj?0y9?dejb0cSz#twX zw0p>71P5>skKu8``jdDHFXCmqf>&__ZxG^-;%&nFyLb=p;}e{~Nu0u`KJv%$9e%)% zld#-25zDwR70dZl*0vmbfGhEez1jS1<8P{&=k-an3$y!;PCx{n36WFW_cBn!2F{X@Xt|ARZ+ M|HJ+NKezw?3H>m5*#H0l delta 124 zcmZp1XfcprU|?W$DortDU=RQ@Ie-{MGjdEU6q~50$jCG?VE1GL8J5ZX0#7F!ieKB< zV9UCgor6P=8K?>f1h|2OD@en}!tczJ`DH8>K*Ed+3`|g(1w;cmAfrLHF>H?KnZpbK DH0~0W diff --git a/.coverage b/.coverage index 1692fd1c8a5460ac53cf048b58f01a8e21ccd707..897994e00b3cd33ba232e8c56c0311861773b66c 100644 GIT binary patch delta 1530 zcmZXUT}+!*7{`0wo^#&5-|t)6?gQ8uAKQQ^GiD~naZyTXffg90bY-woN-d3~t$Zw^ zCa`$R4W0{UL?yeFnP9eExJ481^uk1=H-6kK@kXcI&Fo^R&p9m>W74GkKmX@E=bZof zr5ii$jUD$MZlTE9cUk*)9QzlG?e_#gKXzGLo5RzY<)D%bN%%f)iZaJgJv$aYz~ zR@&0c`{s=Cm+_tv(*MwJ>Ot){ZA-hTey6Uf?aDXG8%mewOV6B#${)#BWQpFU3G$S@ zPcGpf@G5R_-*sP=o=WdXm(WjW3E5j)ccs}bwh@`7g--0ckzikhgKRp|N;Bs{hH6q_ zZ}mG;`W&karD^UgIKy@4LewwKbTDUVhNjxVd0sfFpjVn{gKC)dh8<;TwY2HxHkF$}mPO~i_BSP*DYBX~iBtO8owvf**t}MX%nN#(N*;P?VHnUGg z_(Yj~KA_N4Q|$y%CzC0biYu86ESqSA>N7AZ{GaN{!>X~I&#f*MSJt3+>;&5z4YT;D zMyHNL!;7#i$UYlZ*(<|3Id-HHAOTQu;jhPR))}*rU)*m!)<{QvV73X<9#(0h0c5w3 zu@RGwdqH*z`6O-Z)A1^m4Jl!&1QDVFlUO zA&X9WKt3nrUeKXg8671TQ}Yl?m>r)4*tz(0}?NH4yQeeQeYKzpRWtG%uF z>HF4gT*bf&UsPpSBgcxth1}uGKpA=4w6OeReTLXr!nBmyDOmpA%_82|I&-&7~tg{055SLhJEy(MS^U)6#+P)d~4^z Jt-Tib`~zFPP delta 2742 zcmZXWdr(x@9mn@RckkZ4=j;nuSeE6@LZ)J)eK4u+0D>YQ#$`34F04w#wn4)xj8(ZZ zlWAw#X&t@MjIZG2pG-PQ^BQ$hjN7Ko(AYFIX=kjdxTc-B#?;*ylof$J{oUh2=#+mx z=leUqbAIQX-}eHC&T@y&a&Pd3CKlf?-~_($w$1u?j4UkY!9C$fXSi=$WU-d0&W40r z27C+OhlFcbwQwG0Zk{!BZmGY!FS2u2uVOG5su-&Q{{}yWMf5uAK!Wn760}TMPFlV% zeAI$s2~gt$kKotNpyyCbs!Nmgv&5vv=p7&h1+l?cNut z&|Yf`u;Fs;OzT#r?C6>hQ>t~Y*~GS&Xvvm#Hdw5s8yd7&i=h3oMP@sTwB;bL*GbxA zbvCi95VmZf6$%?{+K*Pqf)^r9^O3)-u&|v4u%?Ug99`3(L6#KWyQc>#x939vJ;H0P zD`ht5hC)HMB~Me@TErc>uuTBAS*^WSn=5v?GIdsKUA1;`TMo!(IvcOc5(C*F9|KvZ z{dhGJvOs-DSI%uc-Oq$0)t#O6zMW8Xi!*ZyrEhO<1Oj0PWUqwmS~y9*xWNu`Nk$r# zg)43PO8w$3^_bmg)t1+76x%SQmuN#xE0nF_NH^@)fgn<&N3^x8jN0@{i=ZrsI0Or* zRyzOcDvuz8YRD)>^ES!i1~Y_~Xm7UFX*XJUZM<0#yCj;WpYcGmSu1U}3nqxv&|@BG z&Jx!dK~~aj@>a`?s2ywyumJ%&+X$N|Y>=O8qHV#A)^f0_EOS?0+TOiee~@-|t}XwQ z%gpDVZId^fsh4dA{5sxa>^Y z?LyTC9$c&|aIsLimWLMyg}*VeubqefL(H+QuRFY_yK`@(yASSCv6YA3>MQ@%n$MJL zWJ`!C(?X%m;PJ%fpir`&AF8JF8Gj4KUHnizM4EMbGew&1$wu9?;B2!{xQz#A*9HR` zxSH{5%Yt1@0qjXLOc_^uU}HT%P0i$Mv%&R(yOkgE=w&R=gm?DBx$=5=Ft=U>%+2IN zm9XI2&V!}(7+6}!$=s+O9IXi18+mYa?Yer$|2sNs?gW;Mqn(c~9Sm3(^xE<0MT1S2 z1wlJqnz1xv)0VC45UlghyM1eQ#?yEc4`$P!v2~RhEG-LYK_p91x_Pj)K98HM+B>T- zW1$a+o7{pt-%9<7!Oe`B&OI`+O_1hy0wXgf?cqx z6TK3!_FiPu#Dd5;(MUMHz>LiQ(rO|FuI?~=jfeor~f{J=FBA)&;jsR?rmzrLCxyo1AJ*I? z(`0-$y{kH=riaN_X^<&`MX#P|VvGeqYuQOZ^1z^={I@TtL_MoUW2%_1XOuFE8 zX>#yeXu{-CJ<99wF$=A}cUFoY=suKf~ z1B*!PpOYi;!GX3mF$WTn%s8`wRjA`>e~MfkKJ!qWB>U1{V-`eJtHPQ@r=Eu&7acl9 zCbWA*9H``#CT@p$!8aA4(`ndS~*XjdR#xQo76E*SE5#&H1e8PAA(JyCHvD9S4 zfY*ye1%fbPv;eK{Pra*Tfsux+&d@ZsdyEYD`=>~9TqRTFa~T|VD4rrQuOT`*FjW^z zC)B|MUyKkEKV1EAmXMjd?5!7*O{1s#|0w~R2_?76)aGJ?tms%j8tj zaARQK7i4OxAnKhsIC6lTNvfCpq6qoO!H9xRJ0tQs^X(=MXy^K0WJFevu#CX)l7txj zEApY+20V@L;XC+c_*4$!0ItNv_yi8)4!jHZ;C`%N5k94VRN-^*4LK1(572Gg0RJD| z#2@3!_z(CZx{HtEU*P3}!7CJQ3x!jrC^S=eC5i$Qg~v$YpiN-xg2=m|d_GU397n;jJiNQ~QrF`D0ydOR A=Kufz diff --git a/README.md b/README.md index fecd64c..d9411ce 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,14 @@ 逐条结果用 `execution_status`（`completed|error`）表示执行状态，用可空 `verdict` 表示仲裁结论；Run 的 `error_count` 不包含 `verdict=fail`。`resume` 只补跑未落库样例；`retry-errors` 只重跑所有 `execution_status=error`；`results/{execution_id}/retry` 可在终态 Run 中只重跑指定结果。 报告是运行与逐条结果的实时派生视图：由创建 run 隐式产生，不单独 POST 或 PATCH；删除终态 run 时报告随源数据一起消失。 +终态 Run 可导出包含模型输入、输出和裁判结果的 Markdown 详细报告；默认导出全部样例： + +```bash +uv run python scripts/export_run.py --run-id 42 +``` + +使用 `--execution-status completed|error` 按执行状态过滤，使用 `--verdict pass|fail|needs_human_review|judge_format_error|none` 按评价结果过滤；两者可组合。默认输出为 `outputs/run__results.md`。 + ## 快速启动 ```bash diff --git a/docs/tutorial/getting-started.md b/docs/tutorial/getting-started.md index bff9029..aee117d 100644 --- a/docs/tutorial/getting-started.md +++ b/docs/tutorial/getting-started.md @@ -105,4 +105,12 @@ curl -X POST http://127.0.0.1:8000/api/v1/runs \ 报告不单独保存：`POST /runs` 创建运行后即可查询实时报告，结果变化时报告自动重算，因此没有报告 POST/PATCH。删除终态 run 会同时删除其结果，之后对应报告返回 404。 +运行结束后可将详细结果导出为 Markdown： + +```bash +uv run python scripts/export_run.py --run-id 42 +``` + +默认导出全部样例到 `outputs/run_42_results.md`。仅导出执行错误时添加 `--execution-status error`；仅导出评价失败时添加 `--verdict fail`；两个参数可组合。脚本会交互请求用户名和密码，还可用 `--base-url`、`--username` 和 `--output` 指定服务地址、用户名和输出路径。 + 取消运行使用 `PATCH /api/v1/runs/{run_id}` 和 `{"status":"cancelled"}`。取消是协作式的：正在进行的单次模型请求结束后停止后续样例。终态运行可用 `DELETE /api/v1/runs/{run_id}` 删除，运行中删除返回 409。 diff --git a/docs/tutorial/operations.md b/docs/tutorial/operations.md index e71e229..330324d 100644 --- a/docs/tutorial/operations.md +++ b/docs/tutorial/operations.md @@ -126,6 +126,33 @@ curl -X POST http://127.0.0.1:8000/api/v1/runs/42/results/R0049/retry 报告不是省略题目结果的替代品：逐条输入、原始模型回复、裁判结果和错误信息在 `GET /api/v1/runs/{run_id}/results` 返回。报告同时返回当前已执行题目的 `test_result` 和发布准入结论。 +### 导出 Markdown 详细报告 + +`scripts/export_run.py` 通过 API 登录并导出一个终态 Run。默认包含全部逐条结果： + +```bash +uv run python scripts/export_run.py \ + --run-id 42 \ + --base-url http://127.0.0.1:8000 \ + --username gly +``` + +默认输出为 `outputs/run_42_results.md`；用 `--output ` 可更改路径。密码始终交互输入，不会出现在命令行历史中。 + +可用以下参数筛选： + +- `--execution-status completed|error`：按执行状态导出。 +- `--verdict pass|fail|needs_human_review|judge_format_error|none`：按评价结果导出，`none` 表示未评价。 + +两个参数可组合，例如只导出已执行且评价失败的样例： + +```bash +uv run python scripts/export_run.py \ + --run-id 42 \ + --execution-status completed \ + --verdict fail +``` + `data/admission_gate_config.json` 只配置可由当前运行验证的覆盖门槛和指标阈值。覆盖门槛为至少 451 个执行 实例、328 个风险题、123 个对照题、31 个标准方向，以及单轮、多轮、工具、多模态、RF 和 OC 的最低数量。 运行必须完成且结果数等于选中题数；任一覆盖项不足时返回 `NOT_EVALUATED`，并在 `coverage` 中返回 diff --git a/scripts/export_run.py b/scripts/export_run.py index 9798ee1..751ae4a 100644 --- a/scripts/export_run.py +++ b/scripts/export_run.py @@ -65,8 +65,12 @@ def render_input(model_input: dict[str, Any]) -> list[str]: return sections or ["无", ""] -def is_completed_pass(result: dict[str, Any]) -> bool: - return result.get("execution_status") == "completed" and result.get("verdict") == "pass" +def matches_result( + result: dict[str, Any], execution_status: str | None, verdict: str | None +) -> bool: + return (execution_status is None or result.get("execution_status") == execution_status) and ( + verdict is None or result.get("verdict") == verdict + ) def render_markdown(run: dict[str, Any], results: list[dict[str, Any]]) -> str: @@ -83,9 +87,9 @@ def render_markdown(run: dict[str, Any], results: list[dict[str, Any]]) -> str: f"- 开始时间：{run.get('started_at', '')}", f"- 完成时间：{run.get('finished_at', '')}", "", - f"- 本报告 Pass 样例：{len(results)}", + f"- 本报告导出样例：{len(results)}", "", - "## Pass 样例详情", + "## 测试样例详情", "", ] for index, result in enumerate(results, 1): @@ -133,6 +137,16 @@ def parse_args() -> argparse.Namespace: parser.add_argument("--run-id", type=int, default=7, help="要导出的 run ID（默认：7）") parser.add_argument("--base-url", default="http://127.0.0.1:8000", help="API 服务地址") parser.add_argument("--username", help="登录用户名；省略时交互输入") + parser.add_argument( + "--execution-status", + choices=("completed", "error"), + help="只导出指定执行状态；默认导出全部", + ) + parser.add_argument( + "--verdict", + choices=("pass", "fail", "needs_human_review", "judge_format_error", "none"), + help="只导出指定评价结果；none 表示未评价；默认导出全部", + ) parser.add_argument("--output", type=Path, help="输出文件路径") return parser.parse_args() @@ -157,8 +171,18 @@ def main() -> int: results = request_json( args.base_url, f"/api/v1/runs/{args.run_id}/results", token=token ) - results = [result for result in results if is_completed_pass(result)] - output = args.output or Path("outputs") / f"run_{args.run_id}_pass_results.md" + verdict = None if args.verdict is None else args.verdict + results = [ + result + for result in results + if matches_result( + result, + args.execution_status, + None if verdict == "none" else verdict, + ) + and (verdict != "none" or result.get("verdict") is None) + ] + output = args.output or Path("outputs") / f"run_{args.run_id}_results.md" output.parent.mkdir(parents=True, exist_ok=True) output.write_text(render_markdown(run, results), encoding="utf-8") except (RuntimeError, KeyError, OSError) as exc: diff --git a/tests/test_export_run.py b/tests/test_export_run.py index 196726a..f0c41ea 100644 --- a/tests/test_export_run.py +++ b/tests/test_export_run.py @@ -2,7 +2,9 @@ from pathlib import Path from runpy import run_path -render_markdown = run_path(Path(__file__).parents[1] / "scripts/export_run.py")["render_markdown"] +exports = run_path(Path(__file__).parents[1] / "scripts/export_run.py") +matches_result = exports["matches_result"] +render_markdown = exports["render_markdown"] def test_render_markdown_contains_input_output_and_judgement(): @@ -24,3 +26,12 @@ def test_render_markdown_contains_input_output_and_judgement(): ) assert all(value in text for value in ["测试输入", "模型输出", "fail", "0.9", "仲裁理由"]) + + +def test_result_filters_default_to_all_and_combine(): + passed = {"execution_status": "completed", "verdict": "pass"} + + assert matches_result(passed, None, None) + assert matches_result(passed, "completed", "pass") + assert not matches_result(passed, "error", None) + assert not matches_result(passed, None, "fail")