{"benchmark":{"name":"CoffeeBench","suite_id":"deepseek-v4-reliable","version":"1.0.0"},"calibration":{"agreement":null,"decision_source":null,"sample_pair_count":0,"status":"not_run"},"identities":{"generation_id":"coffeebench-v0-deepseek-v4-reliable-official","methodology_sha256":"c5374bb3015860e87d8d8ae35a3893682ccf8c53284f84b6b92538d4b1f11900","public_contract_sha256":"c3fa4c3022bffed044a9a50b74b2e83291cd284ab4c2251eacb29ce8278b02c8","result_content_sha256":"50ca8fbd22a8523eacb13bc21c5eb890a2fd60f3e55db8a386a00bc8b94bb087","result_id":"openclaw-jury-2026-08-19-independent.coffeebench-v0-deepseek-v4-reliable-official.published.50ca8fbd22a8523eacb13bc21c5eb890a2fd60f3e55db8a386a00bc8b94bb087","subject_cards_sha256":"ebecb71d35f71eecb7673ae404cb7aa17ccc3138927eba5f3b2b5395e33d5803"},"jury":[{"call_count":1000,"family":"openai","latency_ms":{"p50":18084,"p95":31410},"normalized_cost_usd_total":null,"provider_billed_usd_total":null,"provider_call_count":1000},{"call_count":1006,"family":"google","latency_ms":{"p50":5632,"p95":11348},"normalized_cost_usd_total":null,"provider_billed_usd_total":null,"provider_call_count":1006},{"call_count":1015,"family":"anthropic","latency_ms":{"p50":8897,"p95":38492},"normalized_cost_usd_total":null,"provider_billed_usd_total":null,"provider_call_count":1015}],"limitations":["This result compares four DeepSeek treatments on one frozen 20-case suite.","Operational reliability, absolute rubric correctness, and pairwise quality are separate; no composite score is reported.","Unavailable provider, token-class, tool-cost, or pairwise metrics remain null and are not zero.","Independent human agreement was not measured; this published V1 result reports the complete three-family agent jury and does not establish broad model superiority."],"methodology":{"absolute_evaluation_count":1200,"absolute_rubric_rule":"report criterion outcomes, strict all-requirements pass, must-not-miss failures, critical errors, confidence, and derived unacceptable rate independently","case_count":20,"composite_score":null,"confidence_calibration_rule":"at least two of three absolute judges pass the confidence criterion","critical_error_rule":"at least two of three absolute judges record a critical error","jury_family_count":3,"null_semantics":"null means unavailable or unsupported and never means zero; pairwise nulls remain independent of operational and absolute-rubric observations","operational_reliability_rule":"report terminal, response-contract, judgeable-response, provider/search transport, latency, token, and cost outcomes independently","pairwise_ballot_count":1800,"pairwise_possible_ballot_count":1800,"pairwise_quality_rule":"fit only model-backed preferences between response-bearing trials; absolute rubric outcomes and terminal-failure thresholds do not rewrite or exclude preferences","pareto_rule":"show a non-composite quality-cost-latency frontier only when all three coordinates are available","quality_model":"Bradley-Terry logistic ability","scoring_contract":"independent_tracks_v1","subject_trial_count":400,"tie_value":0.5,"unacceptable_response_rule":"at least two of three absolute judges","uncertainty":"source-family clustered bootstrap with case fallback"},"result_version":"1.0.0.coffeebench-v0-deepseek-v4-reliable-official.published.50ca8fbd22a8523e","schema_version":5,"slices":[{"label":"Overall","slice_id":"overall","track_results":[{"pairwise_matchups":[{"ballot_count":300,"jury_families":[{"ballot_count":100,"family":"openai","subject_a_preference_share":0.46,"subject_a_win_count":39,"subject_b_preference_share":0.54,"subject_b_win_count":47,"tie_count":14},{"ballot_count":100,"family":"google","subject_a_preference_share":0.47,"subject_a_win_count":40,"subject_b_preference_share":0.53,"subject_b_win_count":46,"tie_count":14},{"ballot_count":100,"family":"anthropic","subject_a_preference_share":0.53,"subject_a_win_count":28,"subject_b_preference_share":0.47,"subject_b_win_count":22,"tie_count":50}],"subject_a":"deepseek-v4-pi-search","subject_a_preference_share":0.48666667,"subject_a_win_count":107,"subject_b":"deepseek-v4-purveyors-parchment-search","subject_b_preference_share":0.51333333,"subject_b_win_count":115,"tie_count":78},{"ballot_count":300,"jury_families":[{"ballot_count":100,"family":"openai","subject_a_preference_share":0.42,"subject_a_win_count":34,"subject_b_preference_share":0.58,"subject_b_win_count":50,"tie_count":16},{"ballot_count":100,"family":"google","subject_a_preference_share":0.455,"subject_a_win_count":40,"subject_b_preference_share":0.545,"subject_b_win_count":49,"tie_count":11},{"ballot_count":100,"family":"anthropic","subject_a_preference_share":0.46,"subject_a_win_count":21,"subject_b_preference_share":0.54,"subject_b_win_count":29,"tie_count":50}],"subject_a":"deepseek-v4-pi-search","subject_a_preference_share":0.445,"subject_a_win_count":95,"subject_b":"deepseek-v4-purveyors-search","subject_b_preference_share":0.555,"subject_b_win_count":128,"tie_count":77},{"ballot_count":300,"jury_families":[{"ballot_count":100,"family":"openai","subject_a_preference_share":0.695,"subject_a_win_count":62,"subject_b_preference_share":0.305,"subject_b_win_count":23,"tie_count":15},{"ballot_count":100,"family":"google","subject_a_preference_share":0.745,"subject_a_win_count":71,"subject_b_preference_share":0.255,"subject_b_win_count":22,"tie_count":7},{"ballot_count":100,"family":"anthropic","subject_a_preference_share":0.67,"subject_a_win_count":49,"subject_b_preference_share":0.33,"subject_b_win_count":15,"tie_count":36}],"subject_a":"deepseek-v4-pi-search","subject_a_preference_share":0.70333333,"subject_a_win_count":182,"subject_b":"deepseek-v4-raw","subject_b_preference_share":0.29666667,"subject_b_win_count":60,"tie_count":58},{"ballot_count":300,"jury_families":[{"ballot_count":100,"family":"openai","subject_a_preference_share":0.52,"subject_a_win_count":44,"subject_b_preference_share":0.48,"subject_b_win_count":40,"tie_count":16},{"ballot_count":100,"family":"google","subject_a_preference_share":0.445,"subject_a_win_count":39,"subject_b_preference_share":0.555,"subject_b_win_count":50,"tie_count":11},{"ballot_count":100,"family":"anthropic","subject_a_preference_share":0.44,"subject_a_win_count":19,"subject_b_preference_share":0.56,"subject_b_win_count":31,"tie_count":50}],"subject_a":"deepseek-v4-purveyors-parchment-search","subject_a_preference_share":0.46833333,"subject_a_win_count":102,"subject_b":"deepseek-v4-purveyors-search","subject_b_preference_share":0.53166667,"subject_b_win_count":121,"tie_count":77},{"ballot_count":300,"jury_families":[{"ballot_count":100,"family":"openai","subject_a_preference_share":0.74,"subject_a_win_count":70,"subject_b_preference_share":0.26,"subject_b_win_count":22,"tie_count":8},{"ballot_count":100,"family":"google","subject_a_preference_share":0.775,"subject_a_win_count":76,"subject_b_preference_share":0.225,"subject_b_win_count":21,"tie_count":3},{"ballot_count":100,"family":"anthropic","subject_a_preference_share":0.72,"subject_a_win_count":55,"subject_b_preference_share":0.28,"subject_b_win_count":11,"tie_count":34}],"subject_a":"deepseek-v4-purveyors-parchment-search","subject_a_preference_share":0.745,"subject_a_win_count":201,"subject_b":"deepseek-v4-raw","subject_b_preference_share":0.255,"subject_b_win_count":54,"tie_count":45},{"ballot_count":300,"jury_families":[{"ballot_count":100,"family":"openai","subject_a_preference_share":0.715,"subject_a_win_count":66,"subject_b_preference_share":0.285,"subject_b_win_count":23,"tie_count":11},{"ballot_count":100,"family":"google","subject_a_preference_share":0.78,"subject_a_win_count":73,"subject_b_preference_share":0.22,"subject_b_win_count":17,"tie_count":10},{"ballot_count":100,"family":"anthropic","subject_a_preference_share":0.67,"subject_a_win_count":51,"subject_b_preference_share":0.33,"subject_b_win_count":17,"tie_count":32}],"subject_a":"deepseek-v4-purveyors-search","subject_a_preference_share":0.72166667,"subject_a_win_count":190,"subject_b":"deepseek-v4-raw","subject_b_preference_share":0.27833333,"subject_b_win_count":57,"tie_count":53}],"subjects":[{"absolute_rubric":{"confidence_pass_rate":0.97,"criterion_outcomes":{"fail":183,"not_applicable":0,"pass":2412,"pass_rate":0.92947977,"total":2595},"critical_error_rate":0.18,"must_not_miss_failure_rate":0.24,"strict_all_requirements_pass_rate":0.64,"trial_count":100,"unacceptable_response_rate":0.32},"operational":{"cost":{"normalized_cost_usd":{"per_attempted_task":null,"total":null},"provider_billed_usd":{"per_attempted_task":null,"total":null}},"judgeable_response_count":100,"judgeable_response_rate":1.0,"latency":{"end_to_end_ms":{"p50":8750,"p95":21928},"tool_ms":{"p50":0,"p95":4198}},"response_contract_valid_count":99,"response_contract_valid_rate":0.99,"terminal_failure_count":1,"terminal_failure_rate":0.01,"terminal_status_counts":{"error":0,"invalid_response":1,"success":99,"timeout":0},"token_usage":{"cached_input_tokens":{"per_attempted_task":2055.68,"total":205568},"input_tokens":{"per_attempted_task":6343.61,"total":634361},"output_tokens":{"per_attempted_task":701.78,"total":70178},"provenance":"exact","reasoning_tokens":{"per_attempted_task":null,"total":null},"total_tokens":{"per_attempted_task":7045.39,"total":704539}},"transport":{"provider":{"attempt_count":163,"failure_count":0,"logical_invocation_count":115,"retry_count":48,"success_rate":1.0},"search":{"attempt_count":80,"failure_count":0,"logical_invocation_count":80,"retry_count":0,"success_rate":1.0}},"trial_count":100},"pairwise_quality":{"ballot_coverage_rate":1.0,"interval_95":{"lower":0.4873558,"upper":0.59173714},"model_backed_ballot_count":900,"possible_ballot_count":900,"rank":3,"score":0.53682816},"pareto":{"classification":"unavailable","dominated_by":[]},"subject_id":"deepseek-v4-pi-search"},{"absolute_rubric":{"confidence_pass_rate":1.0,"criterion_outcomes":{"fail":173,"not_applicable":0,"pass":2422,"pass_rate":0.93333333,"total":2595},"critical_error_rate":0.14,"must_not_miss_failure_rate":0.22,"strict_all_requirements_pass_rate":0.68,"trial_count":100,"unacceptable_response_rate":0.3},"operational":{"cost":{"normalized_cost_usd":{"per_attempted_task":null,"total":null},"provider_billed_usd":{"per_attempted_task":null,"total":null}},"judgeable_response_count":100,"judgeable_response_rate":1.0,"latency":{"end_to_end_ms":{"p50":9339,"p95":19722},"tool_ms":{"p50":0,"p95":5253}},"response_contract_valid_count":100,"response_contract_valid_rate":1.0,"terminal_failure_count":0,"terminal_failure_rate":0.0,"terminal_status_counts":{"error":0,"invalid_response":0,"success":100,"timeout":0},"token_usage":{"cached_input_tokens":{"per_attempted_task":2859.52,"total":285952},"input_tokens":{"per_attempted_task":6917.35,"total":691735},"output_tokens":{"per_attempted_task":706.21,"total":70621},"provenance":"exact","reasoning_tokens":{"per_attempted_task":null,"total":null},"total_tokens":{"per_attempted_task":7623.56,"total":762356}},"transport":{"provider":{"attempt_count":177,"failure_count":0,"logical_invocation_count":115,"retry_count":62,"success_rate":1.0},"search":{"attempt_count":67,"failure_count":0,"logical_invocation_count":67,"retry_count":0,"success_rate":1.0}},"trial_count":100},"pairwise_quality":{"ballot_coverage_rate":1.0,"interval_95":{"lower":0.51963563,"upper":0.60863062},"model_backed_ballot_count":900,"possible_ballot_count":900,"rank":2,"score":0.56065672},"pareto":{"classification":"unavailable","dominated_by":[]},"subject_id":"deepseek-v4-purveyors-parchment-search"},{"absolute_rubric":{"confidence_pass_rate":0.98,"criterion_outcomes":{"fail":177,"not_applicable":0,"pass":2418,"pass_rate":0.93179191,"total":2595},"critical_error_rate":0.17,"must_not_miss_failure_rate":0.17,"strict_all_requirements_pass_rate":0.69,"trial_count":100,"unacceptable_response_rate":0.3},"operational":{"cost":{"normalized_cost_usd":{"per_attempted_task":null,"total":null},"provider_billed_usd":{"per_attempted_task":null,"total":null}},"judgeable_response_count":100,"judgeable_response_rate":1.0,"latency":{"end_to_end_ms":{"p50":9813,"p95":24390},"tool_ms":{"p50":0,"p95":5694}},"response_contract_valid_count":100,"response_contract_valid_rate":1.0,"terminal_failure_count":0,"terminal_failure_rate":0.0,"terminal_status_counts":{"error":0,"invalid_response":0,"success":100,"timeout":0},"token_usage":{"cached_input_tokens":{"per_attempted_task":2073.6,"total":207360},"input_tokens":{"per_attempted_task":6109.44,"total":610944},"output_tokens":{"per_attempted_task":675.43,"total":67543},"provenance":"exact","reasoning_tokens":{"per_attempted_task":null,"total":null},"total_tokens":{"per_attempted_task":6784.87,"total":678487}},"transport":{"provider":{"attempt_count":162,"failure_count":0,"logical_invocation_count":115,"retry_count":47,"success_rate":1.0},"search":{"attempt_count":75,"failure_count":0,"logical_invocation_count":75,"retry_count":0,"success_rate":1.0}},"trial_count":100},"pairwise_quality":{"ballot_coverage_rate":1.0,"interval_95":{"lower":0.54719844,"upper":0.6193969},"model_backed_ballot_count":900,"possible_ballot_count":900,"rank":1,"score":0.58183655},"pareto":{"classification":"unavailable","dominated_by":[]},"subject_id":"deepseek-v4-purveyors-search"},{"absolute_rubric":{"confidence_pass_rate":0.85,"criterion_outcomes":{"fail":265,"not_applicable":0,"pass":1955,"pass_rate":0.88063063,"total":2220},"critical_error_rate":0.08,"must_not_miss_failure_rate":0.41,"strict_all_requirements_pass_rate":0.59,"trial_count":100,"unacceptable_response_rate":0.41},"operational":{"cost":{"normalized_cost_usd":{"per_attempted_task":"0.0001419654","total":"0.014196540"},"provider_billed_usd":{"per_attempted_task":"0.0001419654","total":"0.014196540"}},"judgeable_response_count":100,"judgeable_response_rate":1.0,"latency":{"end_to_end_ms":{"p50":5802,"p95":15517},"tool_ms":{"p50":0,"p95":0}},"response_contract_valid_count":100,"response_contract_valid_rate":1.0,"terminal_failure_count":0,"terminal_failure_rate":0.0,"terminal_status_counts":{"error":0,"invalid_response":0,"success":100,"timeout":0},"token_usage":{"cached_input_tokens":{"per_attempted_task":844.8,"total":84480},"input_tokens":{"per_attempted_task":1281.15,"total":128115},"output_tokens":{"per_attempted_task":519.67,"total":51967},"provenance":"exact","reasoning_tokens":{"per_attempted_task":null,"total":null},"total_tokens":{"per_attempted_task":1800.82,"total":180082}},"transport":{"provider":{"attempt_count":115,"failure_count":0,"logical_invocation_count":115,"retry_count":0,"success_rate":1.0},"search":{"attempt_count":0,"failure_count":0,"logical_invocation_count":0,"retry_count":0,"success_rate":null}},"trial_count":100},"pairwise_quality":{"ballot_coverage_rate":1.0,"interval_95":{"lower":0.24910949,"upper":0.40006259},"model_backed_ballot_count":900,"possible_ballot_count":900,"rank":4,"score":0.32701327},"pareto":{"classification":"frontier","dominated_by":[]},"subject_id":"deepseek-v4-raw"}],"track":"system"}]},{"label":"Historical control","slice_id":"historical_control","track_results":[{"pairwise_matchups":[{"ballot_count":180,"jury_families":[{"ballot_count":60,"family":"openai","subject_a_preference_share":0.43333333,"subject_a_win_count":21,"subject_b_preference_share":0.56666667,"subject_b_win_count":29,"tie_count":10},{"ballot_count":60,"family":"google","subject_a_preference_share":0.475,"subject_a_win_count":23,"subject_b_preference_share":0.525,"subject_b_win_count":26,"tie_count":11},{"ballot_count":60,"family":"anthropic","subject_a_preference_share":0.525,"subject_a_win_count":17,"subject_b_preference_share":0.475,"subject_b_win_count":14,"tie_count":29}],"subject_a":"deepseek-v4-pi-search","subject_a_preference_share":0.47777778,"subject_a_win_count":61,"subject_b":"deepseek-v4-purveyors-parchment-search","subject_b_preference_share":0.52222222,"subject_b_win_count":69,"tie_count":50},{"ballot_count":180,"jury_families":[{"ballot_count":60,"family":"openai","subject_a_preference_share":0.45,"subject_a_win_count":20,"subject_b_preference_share":0.55,"subject_b_win_count":26,"tie_count":14},{"ballot_count":60,"family":"google","subject_a_preference_share":0.38333333,"subject_a_win_count":19,"subject_b_preference_share":0.61666667,"subject_b_win_count":33,"tie_count":8},{"ballot_count":60,"family":"anthropic","subject_a_preference_share":0.45833333,"subject_a_win_count":11,"subject_b_preference_share":0.54166667,"subject_b_win_count":16,"tie_count":33}],"subject_a":"deepseek-v4-pi-search","subject_a_preference_share":0.43055556,"subject_a_win_count":50,"subject_b":"deepseek-v4-purveyors-search","subject_b_preference_share":0.56944444,"subject_b_win_count":75,"tie_count":55},{"ballot_count":180,"jury_families":[{"ballot_count":60,"family":"openai","subject_a_preference_share":0.625,"subject_a_win_count":30,"subject_b_preference_share":0.375,"subject_b_win_count":15,"tie_count":15},{"ballot_count":60,"family":"google","subject_a_preference_share":0.59166667,"subject_a_win_count":32,"subject_b_preference_share":0.40833333,"subject_b_win_count":21,"tie_count":7},{"ballot_count":60,"family":"anthropic","subject_a_preference_share":0.53333333,"subject_a_win_count":17,"subject_b_preference_share":0.46666667,"subject_b_win_count":13,"tie_count":30}],"subject_a":"deepseek-v4-pi-search","subject_a_preference_share":0.58333333,"subject_a_win_count":79,"subject_b":"deepseek-v4-raw","subject_b_preference_share":0.41666667,"subject_b_win_count":49,"tie_count":52},{"ballot_count":180,"jury_families":[{"ballot_count":60,"family":"openai","subject_a_preference_share":0.525,"subject_a_win_count":26,"subject_b_preference_share":0.475,"subject_b_win_count":23,"tie_count":11},{"ballot_count":60,"family":"google","subject_a_preference_share":0.40833333,"subject_a_win_count":20,"subject_b_preference_share":0.59166667,"subject_b_win_count":31,"tie_count":9},{"ballot_count":60,"family":"anthropic","subject_a_preference_share":0.41666667,"subject_a_win_count":10,"subject_b_preference_share":0.58333333,"subject_b_win_count":20,"tie_count":30}],"subject_a":"deepseek-v4-purveyors-parchment-search","subject_a_preference_share":0.45,"subject_a_win_count":56,"subject_b":"deepseek-v4-purveyors-search","subject_b_preference_share":0.55,"subject_b_win_count":74,"tie_count":50},{"ballot_count":180,"jury_families":[{"ballot_count":60,"family":"openai","subject_a_preference_share":0.63333333,"subject_a_win_count":34,"subject_b_preference_share":0.36666667,"subject_b_win_count":18,"tie_count":8},{"ballot_count":60,"family":"google","subject_a_preference_share":0.675,"subject_a_win_count":39,"subject_b_preference_share":0.325,"subject_b_win_count":18,"tie_count":3},{"ballot_count":60,"family":"anthropic","subject_a_preference_share":0.6,"subject_a_win_count":22,"subject_b_preference_share":0.4,"subject_b_win_count":10,"tie_count":28}],"subject_a":"deepseek-v4-purveyors-parchment-search","subject_a_preference_share":0.63611111,"subject_a_win_count":95,"subject_b":"deepseek-v4-raw","subject_b_preference_share":0.36388889,"subject_b_win_count":46,"tie_count":39},{"ballot_count":180,"jury_families":[{"ballot_count":60,"family":"openai","subject_a_preference_share":0.625,"subject_a_win_count":33,"subject_b_preference_share":0.375,"subject_b_win_count":18,"tie_count":9},{"ballot_count":60,"family":"google","subject_a_preference_share":0.675,"subject_a_win_count":36,"subject_b_preference_share":0.325,"subject_b_win_count":15,"tie_count":9},{"ballot_count":60,"family":"anthropic","subject_a_preference_share":0.54166667,"subject_a_win_count":19,"subject_b_preference_share":0.45833333,"subject_b_win_count":14,"tie_count":27}],"subject_a":"deepseek-v4-purveyors-search","subject_a_preference_share":0.61388889,"subject_a_win_count":88,"subject_b":"deepseek-v4-raw","subject_b_preference_share":0.38611111,"subject_b_win_count":47,"tie_count":45}],"subjects":[{"absolute_rubric":{"confidence_pass_rate":0.98333333,"criterion_outcomes":{"fail":78,"not_applicable":0,"pass":1392,"pass_rate":0.94693878,"total":1470},"critical_error_rate":0.03333333,"must_not_miss_failure_rate":0.26666667,"strict_all_requirements_pass_rate":0.71666667,"trial_count":60,"unacceptable_response_rate":0.26666667},"operational":{"cost":{"normalized_cost_usd":{"per_attempted_task":null,"total":null},"provider_billed_usd":{"per_attempted_task":null,"total":null}},"judgeable_response_count":60,"judgeable_response_rate":1.0,"latency":{"end_to_end_ms":{"p50":6210,"p95":18124},"tool_ms":{"p50":0,"p95":0}},"response_contract_valid_count":60,"response_contract_valid_rate":1.0,"terminal_failure_count":0,"terminal_failure_rate":0.0,"terminal_status_counts":{"error":0,"invalid_response":0,"success":60,"timeout":0},"token_usage":{"cached_input_tokens":{"per_attempted_task":1169.06666667,"total":70144},"input_tokens":{"per_attempted_task":1626.08333333,"total":97565},"output_tokens":{"per_attempted_task":640.05,"total":38403},"provenance":"exact","reasoning_tokens":{"per_attempted_task":null,"total":null},"total_tokens":{"per_attempted_task":2266.13333333,"total":135968}},"transport":{"provider":{"attempt_count":75,"failure_count":0,"logical_invocation_count":75,"retry_count":0,"success_rate":1.0},"search":{"attempt_count":0,"failure_count":0,"logical_invocation_count":0,"retry_count":0,"success_rate":null}},"trial_count":60},"pairwise_quality":{"ballot_coverage_rate":1.0,"interval_95":{"lower":0.43420765,"upper":0.56004606},"model_backed_ballot_count":540,"possible_ballot_count":540,"rank":3,"score":0.49801988},"pareto":{"classification":"unavailable","dominated_by":[]},"subject_id":"deepseek-v4-pi-search"},{"absolute_rubric":{"confidence_pass_rate":1.0,"criterion_outcomes":{"fail":73,"not_applicable":0,"pass":1397,"pass_rate":0.95034014,"total":1470},"critical_error_rate":0.03333333,"must_not_miss_failure_rate":0.26666667,"strict_all_requirements_pass_rate":0.73333333,"trial_count":60,"unacceptable_response_rate":0.26666667},"operational":{"cost":{"normalized_cost_usd":{"per_attempted_task":null,"total":null},"provider_billed_usd":{"per_attempted_task":null,"total":null}},"judgeable_response_count":60,"judgeable_response_rate":1.0,"latency":{"end_to_end_ms":{"p50":8145,"p95":17274},"tool_ms":{"p50":0,"p95":1}},"response_contract_valid_count":60,"response_contract_valid_rate":1.0,"terminal_failure_count":0,"terminal_failure_rate":0.0,"terminal_status_counts":{"error":0,"invalid_response":0,"success":60,"timeout":0},"token_usage":{"cached_input_tokens":{"per_attempted_task":2197.33333333,"total":131840},"input_tokens":{"per_attempted_task":2801.81666667,"total":168109},"output_tokens":{"per_attempted_task":726.11666667,"total":43567},"provenance":"exact","reasoning_tokens":{"per_attempted_task":null,"total":null},"total_tokens":{"per_attempted_task":3527.93333333,"total":211676}},"transport":{"provider":{"attempt_count":90,"failure_count":0,"logical_invocation_count":75,"retry_count":15,"success_rate":1.0},"search":{"attempt_count":0,"failure_count":0,"logical_invocation_count":0,"retry_count":0,"success_rate":null}},"trial_count":60},"pairwise_quality":{"ballot_coverage_rate":1.0,"interval_95":{"lower":0.48356147,"upper":0.58259091},"model_backed_ballot_count":540,"possible_ballot_count":540,"rank":2,"score":0.52751855},"pareto":{"classification":"unavailable","dominated_by":[]},"subject_id":"deepseek-v4-purveyors-parchment-search"},{"absolute_rubric":{"confidence_pass_rate":1.0,"criterion_outcomes":{"fail":74,"not_applicable":0,"pass":1396,"pass_rate":0.94965986,"total":1470},"critical_error_rate":0.01666667,"must_not_miss_failure_rate":0.21666667,"strict_all_requirements_pass_rate":0.78333333,"trial_count":60,"unacceptable_response_rate":0.21666667},"operational":{"cost":{"normalized_cost_usd":{"per_attempted_task":null,"total":null},"provider_billed_usd":{"per_attempted_task":null,"total":null}},"judgeable_response_count":60,"judgeable_response_rate":1.0,"latency":{"end_to_end_ms":{"p50":7099,"p95":16489},"tool_ms":{"p50":0,"p95":0}},"response_contract_valid_count":60,"response_contract_valid_rate":1.0,"terminal_failure_count":0,"terminal_failure_rate":0.0,"terminal_status_counts":{"error":0,"invalid_response":0,"success":60,"timeout":0},"token_usage":{"cached_input_tokens":{"per_attempted_task":1190.4,"total":71424},"input_tokens":{"per_attempted_task":1758.58333333,"total":105515},"output_tokens":{"per_attempted_task":644.16666667,"total":38650},"provenance":"exact","reasoning_tokens":{"per_attempted_task":null,"total":null},"total_tokens":{"per_attempted_task":2402.75,"total":144165}},"transport":{"provider":{"attempt_count":75,"failure_count":0,"logical_invocation_count":75,"retry_count":0,"success_rate":1.0},"search":{"attempt_count":0,"failure_count":0,"logical_invocation_count":0,"retry_count":0,"success_rate":null}},"trial_count":60},"pairwise_quality":{"ballot_coverage_rate":1.0,"interval_95":{"lower":0.5187003,"upper":0.59520294},"model_backed_ballot_count":540,"possible_ballot_count":540,"rank":1,"score":0.55911315},"pareto":{"classification":"unavailable","dominated_by":[]},"subject_id":"deepseek-v4-purveyors-search"},{"absolute_rubric":{"confidence_pass_rate":0.93333333,"criterion_outcomes":{"fail":109,"not_applicable":0,"pass":1361,"pass_rate":0.92585034,"total":1470},"critical_error_rate":0.06666667,"must_not_miss_failure_rate":0.31666667,"strict_all_requirements_pass_rate":0.68333333,"trial_count":60,"unacceptable_response_rate":0.31666667},"operational":{"cost":{"normalized_cost_usd":{"per_attempted_task":"0.0001409622","total":"0.008457732"},"provider_billed_usd":{"per_attempted_task":"0.0001409622","total":"0.008457732"}},"judgeable_response_count":60,"judgeable_response_rate":1.0,"latency":{"end_to_end_ms":{"p50":5246,"p95":12796},"tool_ms":{"p50":0,"p95":0}},"response_contract_valid_count":60,"response_contract_valid_rate":1.0,"terminal_failure_count":0,"terminal_failure_rate":0.0,"terminal_status_counts":{"error":0,"invalid_response":0,"success":60,"timeout":0},"token_usage":{"cached_input_tokens":{"per_attempted_task":989.86666667,"total":59392},"input_tokens":{"per_attempted_task":1443.58333333,"total":86615},"output_tokens":{"per_attempted_task":493.48333333,"total":29609},"provenance":"exact","reasoning_tokens":{"per_attempted_task":null,"total":null},"total_tokens":{"per_attempted_task":1937.06666667,"total":116224}},"transport":{"provider":{"attempt_count":75,"failure_count":0,"logical_invocation_count":75,"retry_count":0,"success_rate":1.0},"search":{"attempt_count":0,"failure_count":0,"logical_invocation_count":0,"retry_count":0,"success_rate":null}},"trial_count":60},"pairwise_quality":{"ballot_coverage_rate":1.0,"interval_95":{"lower":0.3610989,"upper":0.46768908},"model_backed_ballot_count":540,"possible_ballot_count":540,"rank":4,"score":0.4158511},"pareto":{"classification":"frontier","dominated_by":[]},"subject_id":"deepseek-v4-raw"}],"track":"system"}]},{"label":"Live web","slice_id":"live_web","track_results":[{"pairwise_matchups":[{"ballot_count":120,"jury_families":[{"ballot_count":40,"family":"openai","subject_a_preference_share":0.5,"subject_a_win_count":18,"subject_b_preference_share":0.5,"subject_b_win_count":18,"tie_count":4},{"ballot_count":40,"family":"google","subject_a_preference_share":0.4625,"subject_a_win_count":17,"subject_b_preference_share":0.5375,"subject_b_win_count":20,"tie_count":3},{"ballot_count":40,"family":"anthropic","subject_a_preference_share":0.5375,"subject_a_win_count":11,"subject_b_preference_share":0.4625,"subject_b_win_count":8,"tie_count":21}],"subject_a":"deepseek-v4-pi-search","subject_a_preference_share":0.5,"subject_a_win_count":46,"subject_b":"deepseek-v4-purveyors-parchment-search","subject_b_preference_share":0.5,"subject_b_win_count":46,"tie_count":28},{"ballot_count":120,"jury_families":[{"ballot_count":40,"family":"openai","subject_a_preference_share":0.375,"subject_a_win_count":14,"subject_b_preference_share":0.625,"subject_b_win_count":24,"tie_count":2},{"ballot_count":40,"family":"google","subject_a_preference_share":0.5625,"subject_a_win_count":21,"subject_b_preference_share":0.4375,"subject_b_win_count":16,"tie_count":3},{"ballot_count":40,"family":"anthropic","subject_a_preference_share":0.4625,"subject_a_win_count":10,"subject_b_preference_share":0.5375,"subject_b_win_count":13,"tie_count":17}],"subject_a":"deepseek-v4-pi-search","subject_a_preference_share":0.46666667,"subject_a_win_count":45,"subject_b":"deepseek-v4-purveyors-search","subject_b_preference_share":0.53333333,"subject_b_win_count":53,"tie_count":22},{"ballot_count":120,"jury_families":[{"ballot_count":40,"family":"openai","subject_a_preference_share":0.8,"subject_a_win_count":32,"subject_b_preference_share":0.2,"subject_b_win_count":8,"tie_count":0},{"ballot_count":40,"family":"google","subject_a_preference_share":0.975,"subject_a_win_count":39,"subject_b_preference_share":0.025,"subject_b_win_count":1,"tie_count":0},{"ballot_count":40,"family":"anthropic","subject_a_preference_share":0.875,"subject_a_win_count":32,"subject_b_preference_share":0.125,"subject_b_win_count":2,"tie_count":6}],"subject_a":"deepseek-v4-pi-search","subject_a_preference_share":0.88333333,"subject_a_win_count":103,"subject_b":"deepseek-v4-raw","subject_b_preference_share":0.11666667,"subject_b_win_count":11,"tie_count":6},{"ballot_count":120,"jury_families":[{"ballot_count":40,"family":"openai","subject_a_preference_share":0.5125,"subject_a_win_count":18,"subject_b_preference_share":0.4875,"subject_b_win_count":17,"tie_count":5},{"ballot_count":40,"family":"google","subject_a_preference_share":0.5,"subject_a_win_count":19,"subject_b_preference_share":0.5,"subject_b_win_count":19,"tie_count":2},{"ballot_count":40,"family":"anthropic","subject_a_preference_share":0.475,"subject_a_win_count":9,"subject_b_preference_share":0.525,"subject_b_win_count":11,"tie_count":20}],"subject_a":"deepseek-v4-purveyors-parchment-search","subject_a_preference_share":0.49583333,"subject_a_win_count":46,"subject_b":"deepseek-v4-purveyors-search","subject_b_preference_share":0.50416667,"subject_b_win_count":47,"tie_count":27},{"ballot_count":120,"jury_families":[{"ballot_count":40,"family":"openai","subject_a_preference_share":0.9,"subject_a_win_count":36,"subject_b_preference_share":0.1,"subject_b_win_count":4,"tie_count":0},{"ballot_count":40,"family":"google","subject_a_preference_share":0.925,"subject_a_win_count":37,"subject_b_preference_share":0.075,"subject_b_win_count":3,"tie_count":0},{"ballot_count":40,"family":"anthropic","subject_a_preference_share":0.9,"subject_a_win_count":33,"subject_b_preference_share":0.1,"subject_b_win_count":1,"tie_count":6}],"subject_a":"deepseek-v4-purveyors-parchment-search","subject_a_preference_share":0.90833333,"subject_a_win_count":106,"subject_b":"deepseek-v4-raw","subject_b_preference_share":0.09166667,"subject_b_win_count":8,"tie_count":6},{"ballot_count":120,"jury_families":[{"ballot_count":40,"family":"openai","subject_a_preference_share":0.85,"subject_a_win_count":33,"subject_b_preference_share":0.15,"subject_b_win_count":5,"tie_count":2},{"ballot_count":40,"family":"google","subject_a_preference_share":0.9375,"subject_a_win_count":37,"subject_b_preference_share":0.0625,"subject_b_win_count":2,"tie_count":1},{"ballot_count":40,"family":"anthropic","subject_a_preference_share":0.8625,"subject_a_win_count":32,"subject_b_preference_share":0.1375,"subject_b_win_count":3,"tie_count":5}],"subject_a":"deepseek-v4-purveyors-search","subject_a_preference_share":0.88333333,"subject_a_win_count":102,"subject_b":"deepseek-v4-raw","subject_b_preference_share":0.11666667,"subject_b_win_count":10,"tie_count":8}],"subjects":[{"absolute_rubric":{"confidence_pass_rate":0.95,"criterion_outcomes":{"fail":105,"not_applicable":0,"pass":1020,"pass_rate":0.90666667,"total":1125},"critical_error_rate":0.4,"must_not_miss_failure_rate":0.2,"strict_all_requirements_pass_rate":0.525,"trial_count":40,"unacceptable_response_rate":0.4},"operational":{"cost":{"normalized_cost_usd":{"per_attempted_task":null,"total":null},"provider_billed_usd":{"per_attempted_task":null,"total":null}},"judgeable_response_count":40,"judgeable_response_rate":1.0,"latency":{"end_to_end_ms":{"p50":13244,"p95":22139},"tool_ms":{"p50":2366,"p95":5772}},"response_contract_valid_count":39,"response_contract_valid_rate":0.975,"terminal_failure_count":1,"terminal_failure_rate":0.025,"terminal_status_counts":{"error":0,"invalid_response":1,"success":39,"timeout":0},"token_usage":{"cached_input_tokens":{"per_attempted_task":3385.6,"total":135424},"input_tokens":{"per_attempted_task":13419.9,"total":536796},"output_tokens":{"per_attempted_task":794.375,"total":31775},"provenance":"exact","reasoning_tokens":{"per_attempted_task":null,"total":null},"total_tokens":{"per_attempted_task":14214.275,"total":568571}},"transport":{"provider":{"attempt_count":88,"failure_count":0,"logical_invocation_count":40,"retry_count":48,"success_rate":1.0},"search":{"attempt_count":80,"failure_count":0,"logical_invocation_count":80,"retry_count":0,"success_rate":1.0}},"trial_count":40},"pairwise_quality":{"ballot_coverage_rate":1.0,"interval_95":{"lower":0.53616796,"upper":0.69820558},"model_backed_ballot_count":360,"possible_ballot_count":360,"rank":3,"score":0.6172536},"pareto":{"classification":"unavailable","dominated_by":[]},"subject_id":"deepseek-v4-pi-search"},{"absolute_rubric":{"confidence_pass_rate":1.0,"criterion_outcomes":{"fail":100,"not_applicable":0,"pass":1025,"pass_rate":0.91111111,"total":1125},"critical_error_rate":0.3,"must_not_miss_failure_rate":0.15,"strict_all_requirements_pass_rate":0.6,"trial_count":40,"unacceptable_response_rate":0.35},"operational":{"cost":{"normalized_cost_usd":{"per_attempted_task":null,"total":null},"provider_billed_usd":{"per_attempted_task":null,"total":null}},"judgeable_response_count":40,"judgeable_response_rate":1.0,"latency":{"end_to_end_ms":{"p50":10676,"p95":20026},"tool_ms":{"p50":2456,"p95":6091}},"response_contract_valid_count":40,"response_contract_valid_rate":1.0,"terminal_failure_count":0,"terminal_failure_rate":0.0,"terminal_status_counts":{"error":0,"invalid_response":0,"success":40,"timeout":0},"token_usage":{"cached_input_tokens":{"per_attempted_task":3852.8,"total":154112},"input_tokens":{"per_attempted_task":13090.65,"total":523626},"output_tokens":{"per_attempted_task":676.35,"total":27054},"provenance":"exact","reasoning_tokens":{"per_attempted_task":null,"total":null},"total_tokens":{"per_attempted_task":13767.0,"total":550680}},"transport":{"provider":{"attempt_count":87,"failure_count":0,"logical_invocation_count":40,"retry_count":47,"success_rate":1.0},"search":{"attempt_count":67,"failure_count":0,"logical_invocation_count":67,"retry_count":0,"success_rate":1.0}},"trial_count":40},"pairwise_quality":{"ballot_coverage_rate":1.0,"interval_95":{"lower":0.56627247,"upper":0.68017545},"model_backed_ballot_count":360,"possible_ballot_count":360,"rank":2,"score":0.63223558},"pareto":{"classification":"unavailable","dominated_by":[]},"subject_id":"deepseek-v4-purveyors-parchment-search"},{"absolute_rubric":{"confidence_pass_rate":0.95,"criterion_outcomes":{"fail":103,"not_applicable":0,"pass":1022,"pass_rate":0.90844444,"total":1125},"critical_error_rate":0.4,"must_not_miss_failure_rate":0.1,"strict_all_requirements_pass_rate":0.55,"trial_count":40,"unacceptable_response_rate":0.425},"operational":{"cost":{"normalized_cost_usd":{"per_attempted_task":null,"total":null},"provider_billed_usd":{"per_attempted_task":null,"total":null}},"judgeable_response_count":40,"judgeable_response_rate":1.0,"latency":{"end_to_end_ms":{"p50":10082,"p95":29333},"tool_ms":{"p50":3051,"p95":7839}},"response_contract_valid_count":40,"response_contract_valid_rate":1.0,"terminal_failure_count":0,"terminal_failure_rate":0.0,"terminal_status_counts":{"error":0,"invalid_response":0,"success":40,"timeout":0},"token_usage":{"cached_input_tokens":{"per_attempted_task":3398.4,"total":135936},"input_tokens":{"per_attempted_task":12635.725,"total":505429},"output_tokens":{"per_attempted_task":722.325,"total":28893},"provenance":"exact","reasoning_tokens":{"per_attempted_task":null,"total":null},"total_tokens":{"per_attempted_task":13358.05,"total":534322}},"transport":{"provider":{"attempt_count":87,"failure_count":0,"logical_invocation_count":40,"retry_count":47,"success_rate":1.0},"search":{"attempt_count":75,"failure_count":0,"logical_invocation_count":75,"retry_count":0,"success_rate":1.0}},"trial_count":40},"pairwise_quality":{"ballot_coverage_rate":1.0,"interval_95":{"lower":0.57879778,"upper":0.71510859},"model_backed_ballot_count":360,"possible_ballot_count":360,"rank":1,"score":0.63681306},"pareto":{"classification":"unavailable","dominated_by":[]},"subject_id":"deepseek-v4-purveyors-search"},{"absolute_rubric":{"confidence_pass_rate":0.725,"criterion_outcomes":{"fail":156,"not_applicable":0,"pass":594,"pass_rate":0.792,"total":750},"critical_error_rate":0.1,"must_not_miss_failure_rate":0.55,"strict_all_requirements_pass_rate":0.45,"trial_count":40,"unacceptable_response_rate":0.55},"operational":{"cost":{"normalized_cost_usd":{"per_attempted_task":"0.0001434702","total":"0.005738808"},"provider_billed_usd":{"per_attempted_task":"0.0001434702","total":"0.005738808"}},"judgeable_response_count":40,"judgeable_response_rate":1.0,"latency":{"end_to_end_ms":{"p50":6726,"p95":35318},"tool_ms":{"p50":0,"p95":0}},"response_contract_valid_count":40,"response_contract_valid_rate":1.0,"terminal_failure_count":0,"terminal_failure_rate":0.0,"terminal_status_counts":{"error":0,"invalid_response":0,"success":40,"timeout":0},"token_usage":{"cached_input_tokens":{"per_attempted_task":627.2,"total":25088},"input_tokens":{"per_attempted_task":1037.5,"total":41500},"output_tokens":{"per_attempted_task":558.95,"total":22358},"provenance":"exact","reasoning_tokens":{"per_attempted_task":null,"total":null},"total_tokens":{"per_attempted_task":1596.45,"total":63858}},"transport":{"provider":{"attempt_count":40,"failure_count":0,"logical_invocation_count":40,"retry_count":0,"success_rate":1.0},"search":{"attempt_count":0,"failure_count":0,"logical_invocation_count":0,"retry_count":0,"success_rate":null}},"trial_count":40},"pairwise_quality":{"ballot_coverage_rate":1.0,"interval_95":{"lower":0.10581374,"upper":0.2481069},"model_backed_ballot_count":360,"possible_ballot_count":360,"rank":4,"score":0.17061353},"pareto":{"classification":"frontier","dominated_by":[]},"subject_id":"deepseek-v4-raw"}],"track":"system"}]}],"status":"published","subjects":[{"capabilities":["provided_evidence","web_search","web_fetch"],"card_sha256":"c7ec560bc4188a8b7fde82e528b195dddef407124529992c94bceb119e580dcd","display_name":"DeepSeek V4 with Pi Search","evaluator_track":"system","harness_family":"pi","model":{"model":"deepseek/deepseek-v4-flash-0731","provider":"openrouter","quantization":"fp8","revision":"deepinfra/fp8"},"subject_id":"deepseek-v4-pi-search","track":"system"},{"capabilities":["provided_evidence","web_search","web_fetch","parchment_catalog"],"card_sha256":"fa4a0311bcbae387c3a4327e3ea43296f7e848ac6ddc5a965b255f4652611bd6","display_name":"DeepSeek V4 with Purveyors, Parchment, and Search","evaluator_track":"system","harness_family":"purveyors","model":{"model":"deepseek/deepseek-v4-flash-0731","provider":"openrouter","quantization":"fp8","revision":"deepinfra/fp8"},"subject_id":"deepseek-v4-purveyors-parchment-search","track":"system"},{"capabilities":["provided_evidence","web_search","web_fetch"],"card_sha256":"6535ecf00cbcd517d323b0b9bd339325348e729ed8a18f9ce57937d4d7a7c051","display_name":"DeepSeek V4 with Purveyors Search","evaluator_track":"system","harness_family":"purveyors","model":{"model":"deepseek/deepseek-v4-flash-0731","provider":"openrouter","quantization":"fp8","revision":"deepinfra/fp8"},"subject_id":"deepseek-v4-purveyors-search","track":"system"},{"capabilities":["provided_evidence"],"card_sha256":"a95515c3629d3f55c08a97c7f372f7e221ab3d8dcde5eb06848914f56f194f06","display_name":"DeepSeek V4 Raw","evaluator_track":"model","harness_family":"controlled_raw","model":{"model":"deepseek/deepseek-v4-flash-0731","provider":"openrouter","quantization":"fp8","revision":"deepinfra/fp8"},"subject_id":"deepseek-v4-raw","track":"system"}],"tracks":[{"description":"Complete declared treatments compared within one published track; capabilities and all three independent reporting tracks remain explicit.","label":"System track","track_id":"system"}]}