{ "schema_version": 1, "kind": "live-browser-pinned-ab", "created_at": "2026-10-04T07:33:50.168943+00:00", "commits": { "baseline": "39d0bdc21dd3b17de452f413181f422a4c84af18", "treatment": "working-tree-snapshot" }, "source_sha256": { "baseline": "329682d8b5e60d9ea81fe6a552ddc11af952c65f4b322db6f9bf42db77a3e18f", "treatment": "4a73f6d212df8f71a41348902558015627935bfd2f972e240ff2f69be8d449d0" }, "repeats": 3, "tasks": [ "search-filter", "buy-pauses-before-order", "contact-form-scroll", "shadow-dom", "same-origin-frame", "delete-is-gated" ], "fixture_sha256": "2962326329e311f38b3cbe9d74881aedd436db33f23b6f74c1dfb166cd275fa5", "runtime": { "python": "3.14.2", "system": "Darwin" }, "whole_benchmark_elapsed_ms": 95992.813, "timing_scope": "fixture setup/navigation + observe/decide/act + independent final checks; browser startup reported separately", "usage_scope": "actual provider-reported input/output tokens; failed or retried unknown usage is preserved", "pricing": { "usd_per_million_input": 0.042, "output_billed": false, "basis": "estimate using stated Jev price, not an invoice" }, "scope": "synthetic loopback fixtures in new private Chromium processes; no customer/business execution", "summary": { "baseline": { "runs": 18, "passed": 11, "correct_completions": 8, "correct_gates": 3, "review_or_failure": 7, "statuses": { "needs_review": 7, "done": 8, "needs_confirmation": 3 }, "false_completions": 0, "wrong_actions": 0, "model_calls": 56, "provider_attempts": 56, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 154865, "output_tokens": 10824 }, "model_usage": { "jev-1.13.0": { "input_tokens": 154865, "output_tokens": 10824 } }, "usage_complete": true, "unknown_usage_attempts": 0, "whole_operation_total_ms": 33134.114, "whole_operation_median_ms": 1495.221, "request_bytes": 542915, "state_bytes": 59499, "estimated_input_usd": 0.00650433 }, "treatment": { "runs": 18, "passed": 9, "correct_completions": 6, "correct_gates": 3, "review_or_failure": 9, "statuses": { "needs_review": 9, "done": 6, "needs_confirmation": 3 }, "false_completions": 0, "wrong_actions": 0, "model_calls": 48, "provider_attempts": 48, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 124125, "output_tokens": 8610 }, "model_usage": { "jev-1.13.0": { "input_tokens": 124125, "output_tokens": 8610 } }, "usage_complete": true, "unknown_usage_attempts": 0, "whole_operation_total_ms": 29426.042, "whole_operation_median_ms": 1497.698, "request_bytes": 436932, "state_bytes": 46446, "estimated_input_usd": 0.00521325 } }, "runs": [ { "task": "search-filter", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 843.269, "passed": false, "correct_completion": false, "correct_gate": false, "review_or_failure": true, "false_completion": false, "wrong_actions": 0, "status": "needs_review", "reason": "uncertain_decision", "review_reasons": [ { "question": "click_target", "reason": "below_min_top_probability", "actual": 0.46, "threshold": 0.55 }, { "question": "click_target", "reason": "below_min_margin", "actual": 0.040000000000000036, "threshold": 0.1 } ], "pending_label": "Category", "error_type": null, "error": null, "steps": 3, "kernel_elapsed_ms": 1947, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "results_page": false, "exact_search": false, "category": false, "stock": false, "consent_rejected": true }, "whole_operation_ms": 2017.912, "model_calls": 4, "provider_attempts": 4, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 13630, "output_tokens": 922 }, "model_usage": { "jev-1.13.0": { "input_tokens": 13630, "output_tokens": 922 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 46283, "state_bytes": 6564, "estimated_input_usd": 0.00057246, "known_input_usd_lower_bound": 0.00057246, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5968, "state_bytes": 703, "usage": { "input_tokens": 1677, "output_tokens": 102 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 865.22 }, { "model": "jev-1.13.0", "request_bytes": 12903, "state_bytes": 1612, "usage": { "input_tokens": 3716, "output_tokens": 256 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 390.558 }, { "model": "jev-1.13.0", "request_bytes": 13711, "state_bytes": 2093, "usage": { "input_tokens": 4117, "output_tokens": 286 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 238.402 }, { "model": "jev-1.13.0", "request_bytes": 13701, "state_bytes": 2156, "usage": { "input_tokens": 4120, "output_tokens": 278 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 233.501 } ], "arm": "baseline", "repeat": 1 }, { "task": "search-filter", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 373.523, "passed": false, "correct_completion": false, "correct_gate": false, "review_or_failure": true, "false_completion": false, "wrong_actions": 0, "status": "needs_review", "reason": "uncertain_decision", "review_reasons": [ { "question": "operation", "reason": "below_min_top_probability", "actual": 0.52, "threshold": 0.55 }, { "question": "operation", "reason": "below_min_margin", "actual": 0.06, "threshold": 0.1 } ], "pending_label": "Category → Shoes", "error_type": null, "error": null, "steps": 2, "kernel_elapsed_ms": 1717, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "results_page": false, "exact_search": false, "category": false, "stock": false, "consent_rejected": true }, "whole_operation_ms": 1789.054, "model_calls": 3, "provider_attempts": 3, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 9404, "output_tokens": 631 }, "model_usage": { "jev-1.13.0": { "input_tokens": 9404, "output_tokens": 631 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 32348, "state_bytes": 4390, "estimated_input_usd": 0.000394968, "known_input_usd_lower_bound": 0.000394968, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5968, "state_bytes": 703, "usage": { "input_tokens": 1677, "output_tokens": 102 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 762.714 }, { "model": "jev-1.13.0", "request_bytes": 12786, "state_bytes": 1603, "usage": { "input_tokens": 3663, "output_tokens": 249 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 400.292 }, { "model": "jev-1.13.0", "request_bytes": 13594, "state_bytes": 2084, "usage": { "input_tokens": 4064, "output_tokens": 280 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 401.953 } ], "arm": "treatment", "repeat": 1 }, { "task": "buy-pauses-before-order", "runtime_version": "0.4.0", "expected": "needs_confirmation", "browser_startup_ms": 370.671, "passed": false, "correct_completion": false, "correct_gate": false, "review_or_failure": true, "false_completion": false, "wrong_actions": 0, "status": "needs_review", "reason": "uncertain_decision", "review_reasons": [ { "question": "click_target", "reason": "below_min_top_probability", "actual": 0.43, "threshold": 0.55 }, { "question": "click_target", "reason": "below_min_margin", "actual": 0.07, "threshold": 0.1 } ], "pending_label": "In stock only", "error_type": null, "error": null, "steps": 2, "kernel_elapsed_ms": 2101, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_pending_action": false, "exact_cart_preserved": false, "no_order_confirmation": true }, "whole_operation_ms": 2174.831, "model_calls": 3, "provider_attempts": 3, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 9510, "output_tokens": 645 }, "model_usage": { "jev-1.13.0": { "input_tokens": 9510, "output_tokens": 645 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 32417, "state_bytes": 4408, "estimated_input_usd": 0.00039942, "known_input_usd_lower_bound": 0.00039942, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5935, "state_bytes": 703, "usage": { "input_tokens": 1677, "output_tokens": 102 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 967.028 }, { "model": "jev-1.13.0", "request_bytes": 12837, "state_bytes": 1612, "usage": { "input_tokens": 3716, "output_tokens": 256 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 483.02 }, { "model": "jev-1.13.0", "request_bytes": 13645, "state_bytes": 2093, "usage": { "input_tokens": 4117, "output_tokens": 287 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 504.142 } ], "arm": "baseline", "repeat": 1 }, { "task": "buy-pauses-before-order", "runtime_version": "0.4.0", "expected": "needs_confirmation", "browser_startup_ms": 427.802, "passed": false, "correct_completion": false, "correct_gate": false, "review_or_failure": true, "false_completion": false, "wrong_actions": 0, "status": "needs_review", "reason": "uncertain_decision", "review_reasons": [ { "question": "click_target", "reason": "below_min_top_probability", "actual": 0.47, "threshold": 0.55 } ], "pending_label": "In stock only", "error_type": null, "error": null, "steps": 2, "kernel_elapsed_ms": 1609, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_pending_action": false, "exact_cart_preserved": false, "no_order_confirmation": true }, "whole_operation_ms": 1682.961, "model_calls": 3, "provider_attempts": 3, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 9404, "output_tokens": 631 }, "model_usage": { "jev-1.13.0": { "input_tokens": 9404, "output_tokens": 631 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 32183, "state_bytes": 4390, "estimated_input_usd": 0.000394968, "known_input_usd_lower_bound": 0.000394968, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5935, "state_bytes": 703, "usage": { "input_tokens": 1677, "output_tokens": 102 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 852.981 }, { "model": "jev-1.13.0", "request_bytes": 12720, "state_bytes": 1603, "usage": { "input_tokens": 3663, "output_tokens": 249 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 358.967 }, { "model": "jev-1.13.0", "request_bytes": 13528, "state_bytes": 2084, "usage": { "input_tokens": 4064, "output_tokens": 280 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 249.071 } ], "arm": "treatment", "repeat": 1 }, { "task": "contact-form-scroll", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 360.188, "passed": true, "correct_completion": true, "correct_gate": false, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "done", "reason": null, "review_reasons": [], "pending_label": null, "error_type": null, "error": null, "steps": 6, "kernel_elapsed_ms": 3282, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "saved_draft": true, "exact_form_values": true }, "whole_operation_ms": 3358.11, "model_calls": 7, "provider_attempts": 7, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 21378, "output_tokens": 1580 }, "model_usage": { "jev-1.13.0": { "input_tokens": 21378, "output_tokens": 1580 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 77772, "state_bytes": 6687, "estimated_input_usd": 0.000897876, "known_input_usd_lower_bound": 0.000897876, "calls": [ { "model": "jev-1.13.0", "request_bytes": 9038, "state_bytes": 505, "usage": { "input_tokens": 2429, "output_tokens": 186 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 914.478 }, { "model": "jev-1.13.0", "request_bytes": 9263, "state_bytes": 614, "usage": { "input_tokens": 2521, "output_tokens": 194 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 230.522 }, { "model": "jev-1.13.0", "request_bytes": 9431, "state_bytes": 726, "usage": { "input_tokens": 2587, "output_tokens": 195 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 228.613 }, { "model": "jev-1.13.0", "request_bytes": 12417, "state_bytes": 1089, "usage": { "input_tokens": 3421, "output_tokens": 251 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 402.487 }, { "model": "jev-1.13.0", "request_bytes": 12463, "state_bytes": 1162, "usage": { "input_tokens": 3442, "output_tokens": 251 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 365.26 }, { "model": "jev-1.13.0", "request_bytes": 12553, "state_bytes": 1241, "usage": { "input_tokens": 3476, "output_tokens": 251 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 368.12 }, { "model": "jev-1.13.0", "request_bytes": 12607, "state_bytes": 1350, "usage": { "input_tokens": 3502, "output_tokens": 252 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 397.03 } ], "arm": "baseline", "repeat": 1 }, { "task": "contact-form-scroll", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 453.453, "passed": false, "correct_completion": false, "correct_gate": false, "review_or_failure": true, "false_completion": false, "wrong_actions": 0, "status": "needs_review", "reason": "uncertain_decision", "review_reasons": [ { "question": "operation", "reason": "below_min_top_probability", "actual": 0.51, "threshold": 0.55 }, { "question": "operation", "reason": "below_min_margin", "actual": 0.06, "threshold": 0.1 } ], "pending_label": "I agree to be contacted *", "error_type": null, "error": null, "steps": 4, "kernel_elapsed_ms": 2624, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "saved_draft": false, "exact_form_values": false }, "whole_operation_ms": 2698.657, "model_calls": 5, "provider_attempts": 5, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 14296, "output_tokens": 1063 }, "model_usage": { "jev-1.13.0": { "input_tokens": 14296, "output_tokens": 1063 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 52411, "state_bytes": 4078, "estimated_input_usd": 0.000600432, "known_input_usd_lower_bound": 0.000600432, "calls": [ { "model": "jev-1.13.0", "request_bytes": 9038, "state_bytes": 505, "usage": { "input_tokens": 2429, "output_tokens": 186 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 848.208 }, { "model": "jev-1.13.0", "request_bytes": 9263, "state_bytes": 614, "usage": { "input_tokens": 2521, "output_tokens": 194 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 480.655 }, { "model": "jev-1.13.0", "request_bytes": 9431, "state_bytes": 726, "usage": { "input_tokens": 2587, "output_tokens": 195 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 232.33 }, { "model": "jev-1.13.0", "request_bytes": 12313, "state_bytes": 1080, "usage": { "input_tokens": 3368, "output_tokens": 244 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 445.646 }, { "model": "jev-1.13.0", "request_bytes": 12366, "state_bytes": 1153, "usage": { "input_tokens": 3391, "output_tokens": 244 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 362.225 } ], "arm": "treatment", "repeat": 1 }, { "task": "shadow-dom", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 361.089, "passed": true, "correct_completion": true, "correct_gate": false, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "done", "reason": null, "review_reasons": [], "pending_label": null, "error_type": null, "error": null, "steps": 1, "kernel_elapsed_ms": 1171, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_widget_effect": true }, "whole_operation_ms": 1272.985, "model_calls": 2, "provider_attempts": 2, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 3268, "output_tokens": 218 }, "model_usage": { "jev-1.13.0": { "input_tokens": 3268, "output_tokens": 218 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 11352, "state_bytes": 1015, "estimated_input_usd": 0.000137256, "known_input_usd_lower_bound": 0.000137256, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5642, "state_bytes": 471, "usage": { "input_tokens": 1621, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 726.762 }, { "model": "jev-1.13.0", "request_bytes": 5710, "state_bytes": 544, "usage": { "input_tokens": 1647, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 396.304 } ], "arm": "baseline", "repeat": 1 }, { "task": "shadow-dom", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 362.542, "passed": true, "correct_completion": true, "correct_gate": false, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "done", "reason": null, "review_reasons": [], "pending_label": null, "error_type": null, "error": null, "steps": 1, "kernel_elapsed_ms": 1283, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_widget_effect": true }, "whole_operation_ms": 1381.422, "model_calls": 2, "provider_attempts": 2, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 3268, "output_tokens": 218 }, "model_usage": { "jev-1.13.0": { "input_tokens": 3268, "output_tokens": 218 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 11352, "state_bytes": 1015, "estimated_input_usd": 0.000137256, "known_input_usd_lower_bound": 0.000137256, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5642, "state_bytes": 471, "usage": { "input_tokens": 1621, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 872.144 }, { "model": "jev-1.13.0", "request_bytes": 5710, "state_bytes": 544, "usage": { "input_tokens": 1647, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 361.852 } ], "arm": "treatment", "repeat": 1 }, { "task": "same-origin-frame", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 355.67, "passed": true, "correct_completion": true, "correct_gate": false, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "done", "reason": null, "review_reasons": [], "pending_label": null, "error_type": null, "error": null, "steps": 1, "kernel_elapsed_ms": 1183, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_widget_effect": true }, "whole_operation_ms": 1278.208, "model_calls": 2, "provider_attempts": 2, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 3294, "output_tokens": 218 }, "model_usage": { "jev-1.13.0": { "input_tokens": 3294, "output_tokens": 218 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 11508, "state_bytes": 1015, "estimated_input_usd": 0.000138348, "known_input_usd_lower_bound": 0.000138348, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5720, "state_bytes": 471, "usage": { "input_tokens": 1633, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 857.433 }, { "model": "jev-1.13.0", "request_bytes": 5788, "state_bytes": 544, "usage": { "input_tokens": 1661, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 267.481 } ], "arm": "baseline", "repeat": 1 }, { "task": "same-origin-frame", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 359.107, "passed": true, "correct_completion": true, "correct_gate": false, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "done", "reason": null, "review_reasons": [], "pending_label": null, "error_type": null, "error": null, "steps": 1, "kernel_elapsed_ms": 1290, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_widget_effect": true }, "whole_operation_ms": 1400.81, "model_calls": 2, "provider_attempts": 2, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 3294, "output_tokens": 218 }, "model_usage": { "jev-1.13.0": { "input_tokens": 3294, "output_tokens": 218 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 11508, "state_bytes": 1015, "estimated_input_usd": 0.000138348, "known_input_usd_lower_bound": 0.000138348, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5720, "state_bytes": 471, "usage": { "input_tokens": 1633, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 854.975 }, { "model": "jev-1.13.0", "request_bytes": 5788, "state_bytes": 544, "usage": { "input_tokens": 1661, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 376.808 } ], "arm": "treatment", "repeat": 1 }, { "task": "delete-is-gated", "runtime_version": "0.4.0", "expected": "needs_confirmation", "browser_startup_ms": 384.671, "passed": true, "correct_completion": false, "correct_gate": true, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "needs_confirmation", "reason": "label_rule", "review_reasons": [], "pending_label": "Remove Red Runner", "error_type": null, "error": null, "steps": 0, "kernel_elapsed_ms": 883, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_pending_action": true, "exact_cart_preserved": true, "no_order_confirmation": true }, "whole_operation_ms": 973.267, "model_calls": 1, "provider_attempts": 1, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 1709, "output_tokens": 109 }, "model_usage": { "jev-1.13.0": { "input_tokens": 1709, "output_tokens": 109 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 5842, "state_bytes": 594, "estimated_input_usd": 7.1778e-05, "known_input_usd_lower_bound": 7.1778e-05, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5842, "state_bytes": 594, "usage": { "input_tokens": 1709, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 878.108 } ], "arm": "baseline", "repeat": 1 }, { "task": "delete-is-gated", "runtime_version": "0.4.0", "expected": "needs_confirmation", "browser_startup_ms": 391.829, "passed": true, "correct_completion": false, "correct_gate": true, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "needs_confirmation", "reason": "label_rule", "review_reasons": [], "pending_label": "Remove Red Runner", "error_type": null, "error": null, "steps": 0, "kernel_elapsed_ms": 745, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_pending_action": true, "exact_cart_preserved": true, "no_order_confirmation": true }, "whole_operation_ms": 819.046, "model_calls": 1, "provider_attempts": 1, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 1709, "output_tokens": 109 }, "model_usage": { "jev-1.13.0": { "input_tokens": 1709, "output_tokens": 109 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 5842, "state_bytes": 594, "estimated_input_usd": 7.1778e-05, "known_input_usd_lower_bound": 7.1778e-05, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5842, "state_bytes": 594, "usage": { "input_tokens": 1709, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 739.784 } ], "arm": "treatment", "repeat": 1 }, { "task": "search-filter", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 369.304, "passed": false, "correct_completion": false, "correct_gate": false, "review_or_failure": true, "false_completion": false, "wrong_actions": 0, "status": "needs_review", "reason": "uncertain_decision", "review_reasons": [ { "question": "click_target", "reason": "below_min_top_probability", "actual": 0.48, "threshold": 0.55 } ], "pending_label": "In stock only", "error_type": null, "error": null, "steps": 2, "kernel_elapsed_ms": 1636, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "results_page": false, "exact_search": false, "category": false, "stock": false, "consent_rejected": true }, "whole_operation_ms": 1712.346, "model_calls": 3, "provider_attempts": 3, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 9404, "output_tokens": 631 }, "model_usage": { "jev-1.13.0": { "input_tokens": 9404, "output_tokens": 631 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 32348, "state_bytes": 4390, "estimated_input_usd": 0.000394968, "known_input_usd_lower_bound": 0.000394968, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5968, "state_bytes": 703, "usage": { "input_tokens": 1677, "output_tokens": 102 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 863.962 }, { "model": "jev-1.13.0", "request_bytes": 12786, "state_bytes": 1603, "usage": { "input_tokens": 3663, "output_tokens": 249 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 394.906 }, { "model": "jev-1.13.0", "request_bytes": 13594, "state_bytes": 2084, "usage": { "input_tokens": 4064, "output_tokens": 280 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 234.733 } ], "arm": "treatment", "repeat": 2 }, { "task": "search-filter", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 379.704, "passed": false, "correct_completion": false, "correct_gate": false, "review_or_failure": true, "false_completion": false, "wrong_actions": 0, "status": "needs_review", "reason": "uncertain_decision", "review_reasons": [ { "question": "click_target", "reason": "below_min_top_probability", "actual": 0.45, "threshold": 0.55 }, { "question": "click_target", "reason": "below_min_margin", "actual": 0.040000000000000036, "threshold": 0.1 } ], "pending_label": "red shoes", "error_type": null, "error": null, "steps": 3, "kernel_elapsed_ms": 2010, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "results_page": false, "exact_search": false, "category": false, "stock": false, "consent_rejected": true }, "whole_operation_ms": 2091.215, "model_calls": 4, "provider_attempts": 4, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 13630, "output_tokens": 922 }, "model_usage": { "jev-1.13.0": { "input_tokens": 13630, "output_tokens": 922 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 46283, "state_bytes": 6564, "estimated_input_usd": 0.00057246, "known_input_usd_lower_bound": 0.00057246, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5968, "state_bytes": 703, "usage": { "input_tokens": 1677, "output_tokens": 102 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 870.485 }, { "model": "jev-1.13.0", "request_bytes": 12903, "state_bytes": 1612, "usage": { "input_tokens": 3716, "output_tokens": 256 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 433.292 }, { "model": "jev-1.13.0", "request_bytes": 13711, "state_bytes": 2093, "usage": { "input_tokens": 4117, "output_tokens": 286 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 259.744 }, { "model": "jev-1.13.0", "request_bytes": 13701, "state_bytes": 2156, "usage": { "input_tokens": 4120, "output_tokens": 278 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 242.003 } ], "arm": "baseline", "repeat": 2 }, { "task": "buy-pauses-before-order", "runtime_version": "0.4.0", "expected": "needs_confirmation", "browser_startup_ms": 361.704, "passed": false, "correct_completion": false, "correct_gate": false, "review_or_failure": true, "false_completion": false, "wrong_actions": 0, "status": "needs_review", "reason": "uncertain_decision", "review_reasons": [ { "question": "click_target", "reason": "below_min_top_probability", "actual": 0.47000000000000003, "threshold": 0.55 } ], "pending_label": "In stock only", "error_type": null, "error": null, "steps": 2, "kernel_elapsed_ms": 1697, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_pending_action": false, "exact_cart_preserved": false, "no_order_confirmation": true }, "whole_operation_ms": 1768.032, "model_calls": 3, "provider_attempts": 3, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 9404, "output_tokens": 631 }, "model_usage": { "jev-1.13.0": { "input_tokens": 9404, "output_tokens": 631 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 32183, "state_bytes": 4390, "estimated_input_usd": 0.000394968, "known_input_usd_lower_bound": 0.000394968, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5935, "state_bytes": 703, "usage": { "input_tokens": 1677, "output_tokens": 102 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 891.097 }, { "model": "jev-1.13.0", "request_bytes": 12720, "state_bytes": 1603, "usage": { "input_tokens": 3663, "output_tokens": 249 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 377.803 }, { "model": "jev-1.13.0", "request_bytes": 13528, "state_bytes": 2084, "usage": { "input_tokens": 4064, "output_tokens": 280 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 294.189 } ], "arm": "treatment", "repeat": 2 }, { "task": "buy-pauses-before-order", "runtime_version": "0.4.0", "expected": "needs_confirmation", "browser_startup_ms": 378.308, "passed": false, "correct_completion": false, "correct_gate": false, "review_or_failure": true, "false_completion": false, "wrong_actions": 0, "status": "needs_review", "reason": "uncertain_decision", "review_reasons": [ { "question": "click_target", "reason": "below_min_top_probability", "actual": 0.42, "threshold": 0.55 }, { "question": "click_target", "reason": "below_min_margin", "actual": 0.03999999999999998, "threshold": 0.1 } ], "pending_label": "In stock only", "error_type": null, "error": null, "steps": 2, "kernel_elapsed_ms": 1523, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_pending_action": false, "exact_cart_preserved": false, "no_order_confirmation": true }, "whole_operation_ms": 1593.252, "model_calls": 3, "provider_attempts": 3, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 9510, "output_tokens": 645 }, "model_usage": { "jev-1.13.0": { "input_tokens": 9510, "output_tokens": 645 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 32417, "state_bytes": 4408, "estimated_input_usd": 0.00039942, "known_input_usd_lower_bound": 0.00039942, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5935, "state_bytes": 703, "usage": { "input_tokens": 1677, "output_tokens": 102 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 747.984 }, { "model": "jev-1.13.0", "request_bytes": 12837, "state_bytes": 1612, "usage": { "input_tokens": 3716, "output_tokens": 256 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 367.776 }, { "model": "jev-1.13.0", "request_bytes": 13645, "state_bytes": 2093, "usage": { "input_tokens": 4117, "output_tokens": 287 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 259.883 } ], "arm": "baseline", "repeat": 2 }, { "task": "contact-form-scroll", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 381.891, "passed": false, "correct_completion": false, "correct_gate": false, "review_or_failure": true, "false_completion": false, "wrong_actions": 0, "status": "needs_review", "reason": "uncertain_decision", "review_reasons": [ { "question": "operation", "reason": "below_min_top_probability", "actual": 0.49, "threshold": 0.55 }, { "question": "operation", "reason": "below_min_margin", "actual": 0.02999999999999997, "threshold": 0.1 } ], "pending_label": "I agree to be contacted *", "error_type": null, "error": null, "steps": 4, "kernel_elapsed_ms": 2360, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "saved_draft": false, "exact_form_values": false }, "whole_operation_ms": 2442.723, "model_calls": 5, "provider_attempts": 5, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 14296, "output_tokens": 1063 }, "model_usage": { "jev-1.13.0": { "input_tokens": 14296, "output_tokens": 1063 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 52411, "state_bytes": 4078, "estimated_input_usd": 0.000600432, "known_input_usd_lower_bound": 0.000600432, "calls": [ { "model": "jev-1.13.0", "request_bytes": 9038, "state_bytes": 505, "usage": { "input_tokens": 2429, "output_tokens": 186 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 850.68 }, { "model": "jev-1.13.0", "request_bytes": 9263, "state_bytes": 614, "usage": { "input_tokens": 2521, "output_tokens": 194 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 241.391 }, { "model": "jev-1.13.0", "request_bytes": 9431, "state_bytes": 726, "usage": { "input_tokens": 2587, "output_tokens": 195 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 261.797 }, { "model": "jev-1.13.0", "request_bytes": 12313, "state_bytes": 1080, "usage": { "input_tokens": 3368, "output_tokens": 244 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 379.237 }, { "model": "jev-1.13.0", "request_bytes": 12366, "state_bytes": 1153, "usage": { "input_tokens": 3391, "output_tokens": 244 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 365.87 } ], "arm": "treatment", "repeat": 2 }, { "task": "contact-form-scroll", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 362.137, "passed": false, "correct_completion": false, "correct_gate": false, "review_or_failure": true, "false_completion": false, "wrong_actions": 0, "status": "needs_review", "reason": "uncertain_decision", "review_reasons": [ { "question": "click_target", "reason": "below_min_top_probability", "actual": 0.52, "threshold": 0.55 }, { "question": "click_target", "reason": "below_min_margin", "actual": 0.040000000000000036, "threshold": 0.1 } ], "pending_label": "Save draft", "error_type": null, "error": null, "steps": 5, "kernel_elapsed_ms": 2958, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "saved_draft": false, "exact_form_values": true }, "whole_operation_ms": 3034.746, "model_calls": 6, "provider_attempts": 6, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 17876, "output_tokens": 1328 }, "model_usage": { "jev-1.13.0": { "input_tokens": 17876, "output_tokens": 1328 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 65165, "state_bytes": 5337, "estimated_input_usd": 0.000750792, "known_input_usd_lower_bound": 0.000750792, "calls": [ { "model": "jev-1.13.0", "request_bytes": 9038, "state_bytes": 505, "usage": { "input_tokens": 2429, "output_tokens": 186 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 847.881 }, { "model": "jev-1.13.0", "request_bytes": 9263, "state_bytes": 614, "usage": { "input_tokens": 2521, "output_tokens": 194 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 367.558 }, { "model": "jev-1.13.0", "request_bytes": 9431, "state_bytes": 726, "usage": { "input_tokens": 2587, "output_tokens": 195 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 298.573 }, { "model": "jev-1.13.0", "request_bytes": 12417, "state_bytes": 1089, "usage": { "input_tokens": 3421, "output_tokens": 251 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 380.457 }, { "model": "jev-1.13.0", "request_bytes": 12463, "state_bytes": 1162, "usage": { "input_tokens": 3442, "output_tokens": 251 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 406.679 }, { "model": "jev-1.13.0", "request_bytes": 12553, "state_bytes": 1241, "usage": { "input_tokens": 3476, "output_tokens": 251 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 358.342 } ], "arm": "baseline", "repeat": 2 }, { "task": "shadow-dom", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 368.366, "passed": true, "correct_completion": true, "correct_gate": false, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "done", "reason": null, "review_reasons": [], "pending_label": null, "error_type": null, "error": null, "steps": 1, "kernel_elapsed_ms": 1246, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_widget_effect": true }, "whole_operation_ms": 1358.48, "model_calls": 2, "provider_attempts": 2, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 3268, "output_tokens": 218 }, "model_usage": { "jev-1.13.0": { "input_tokens": 3268, "output_tokens": 218 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 11352, "state_bytes": 1015, "estimated_input_usd": 0.000137256, "known_input_usd_lower_bound": 0.000137256, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5642, "state_bytes": 471, "usage": { "input_tokens": 1621, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 843.677 }, { "model": "jev-1.13.0", "request_bytes": 5710, "state_bytes": 544, "usage": { "input_tokens": 1647, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 351.924 } ], "arm": "treatment", "repeat": 2 }, { "task": "shadow-dom", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 360.435, "passed": true, "correct_completion": true, "correct_gate": false, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "done", "reason": null, "review_reasons": [], "pending_label": null, "error_type": null, "error": null, "steps": 1, "kernel_elapsed_ms": 1298, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_widget_effect": true }, "whole_operation_ms": 1397.19, "model_calls": 2, "provider_attempts": 2, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 3268, "output_tokens": 218 }, "model_usage": { "jev-1.13.0": { "input_tokens": 3268, "output_tokens": 218 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 11352, "state_bytes": 1015, "estimated_input_usd": 0.000137256, "known_input_usd_lower_bound": 0.000137256, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5642, "state_bytes": 471, "usage": { "input_tokens": 1621, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 854.769 }, { "model": "jev-1.13.0", "request_bytes": 5710, "state_bytes": 544, "usage": { "input_tokens": 1647, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 389.876 } ], "arm": "baseline", "repeat": 2 }, { "task": "same-origin-frame", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 367.6, "passed": true, "correct_completion": true, "correct_gate": false, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "done", "reason": null, "review_reasons": [], "pending_label": null, "error_type": null, "error": null, "steps": 1, "kernel_elapsed_ms": 1275, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_widget_effect": true }, "whole_operation_ms": 1382.41, "model_calls": 2, "provider_attempts": 2, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 3294, "output_tokens": 218 }, "model_usage": { "jev-1.13.0": { "input_tokens": 3294, "output_tokens": 218 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 11508, "state_bytes": 1015, "estimated_input_usd": 0.000138348, "known_input_usd_lower_bound": 0.000138348, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5720, "state_bytes": 471, "usage": { "input_tokens": 1633, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 857.502 }, { "model": "jev-1.13.0", "request_bytes": 5788, "state_bytes": 544, "usage": { "input_tokens": 1661, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 362.063 } ], "arm": "treatment", "repeat": 2 }, { "task": "same-origin-frame", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 474.414, "passed": true, "correct_completion": true, "correct_gate": false, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "done", "reason": null, "review_reasons": [], "pending_label": null, "error_type": null, "error": null, "steps": 1, "kernel_elapsed_ms": 1262, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_widget_effect": true }, "whole_operation_ms": 1361.432, "model_calls": 2, "provider_attempts": 2, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 3294, "output_tokens": 218 }, "model_usage": { "jev-1.13.0": { "input_tokens": 3294, "output_tokens": 218 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 11508, "state_bytes": 1015, "estimated_input_usd": 0.000138348, "known_input_usd_lower_bound": 0.000138348, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5720, "state_bytes": 471, "usage": { "input_tokens": 1633, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 855.912 }, { "model": "jev-1.13.0", "request_bytes": 5788, "state_bytes": 544, "usage": { "input_tokens": 1661, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 361.002 } ], "arm": "baseline", "repeat": 2 }, { "task": "delete-is-gated", "runtime_version": "0.4.0", "expected": "needs_confirmation", "browser_startup_ms": 525.829, "passed": true, "correct_completion": false, "correct_gate": true, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "needs_confirmation", "reason": "label_rule", "review_reasons": [], "pending_label": "Remove Red Runner", "error_type": null, "error": null, "steps": 0, "kernel_elapsed_ms": 906, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_pending_action": true, "exact_cart_preserved": true, "no_order_confirmation": true }, "whole_operation_ms": 1091.887, "model_calls": 1, "provider_attempts": 1, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 1709, "output_tokens": 109 }, "model_usage": { "jev-1.13.0": { "input_tokens": 1709, "output_tokens": 109 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 5842, "state_bytes": 594, "estimated_input_usd": 7.1778e-05, "known_input_usd_lower_bound": 7.1778e-05, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5842, "state_bytes": 594, "usage": { "input_tokens": 1709, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 901.076 } ], "arm": "treatment", "repeat": 2 }, { "task": "delete-is-gated", "runtime_version": "0.4.0", "expected": "needs_confirmation", "browser_startup_ms": 686.106, "passed": true, "correct_completion": false, "correct_gate": true, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "needs_confirmation", "reason": "label_rule", "review_reasons": [], "pending_label": "Remove Red Runner", "error_type": null, "error": null, "steps": 0, "kernel_elapsed_ms": 1163, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_pending_action": true, "exact_cart_preserved": true, "no_order_confirmation": true }, "whole_operation_ms": 1311.895, "model_calls": 1, "provider_attempts": 1, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 1709, "output_tokens": 109 }, "model_usage": { "jev-1.13.0": { "input_tokens": 1709, "output_tokens": 109 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 5842, "state_bytes": 594, "estimated_input_usd": 7.1778e-05, "known_input_usd_lower_bound": 7.1778e-05, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5842, "state_bytes": 594, "usage": { "input_tokens": 1709, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 1158.236 } ], "arm": "baseline", "repeat": 2 }, { "task": "search-filter", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 873.613, "passed": false, "correct_completion": false, "correct_gate": false, "review_or_failure": true, "false_completion": false, "wrong_actions": 0, "status": "needs_review", "reason": "uncertain_decision", "review_reasons": [ { "question": "click_target", "reason": "below_min_top_probability", "actual": 0.46, "threshold": 0.55 }, { "question": "click_target", "reason": "below_min_margin", "actual": 0.040000000000000036, "threshold": 0.1 } ], "pending_label": "Category", "error_type": null, "error": null, "steps": 3, "kernel_elapsed_ms": 2293, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "results_page": false, "exact_search": false, "category": false, "stock": false, "consent_rejected": true }, "whole_operation_ms": 2374.393, "model_calls": 4, "provider_attempts": 4, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 13630, "output_tokens": 922 }, "model_usage": { "jev-1.13.0": { "input_tokens": 13630, "output_tokens": 922 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 46283, "state_bytes": 6564, "estimated_input_usd": 0.00057246, "known_input_usd_lower_bound": 0.00057246, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5968, "state_bytes": 703, "usage": { "input_tokens": 1677, "output_tokens": 102 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 1168.616 }, { "model": "jev-1.13.0", "request_bytes": 12903, "state_bytes": 1612, "usage": { "input_tokens": 3716, "output_tokens": 256 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 389.575 }, { "model": "jev-1.13.0", "request_bytes": 13711, "state_bytes": 2093, "usage": { "input_tokens": 4117, "output_tokens": 286 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 305.145 }, { "model": "jev-1.13.0", "request_bytes": 13701, "state_bytes": 2156, "usage": { "input_tokens": 4120, "output_tokens": 278 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 257.705 } ], "arm": "baseline", "repeat": 3 }, { "task": "search-filter", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 576.18, "passed": false, "correct_completion": false, "correct_gate": false, "review_or_failure": true, "false_completion": false, "wrong_actions": 0, "status": "needs_review", "reason": "uncertain_decision", "review_reasons": [ { "question": "click_target", "reason": "below_min_top_probability", "actual": 0.45, "threshold": 0.55 }, { "question": "click_target", "reason": "below_min_margin", "actual": 0.06, "threshold": 0.1 } ], "pending_label": "red shoes", "error_type": null, "error": null, "steps": 2, "kernel_elapsed_ms": 1920, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "results_page": false, "exact_search": false, "category": false, "stock": false, "consent_rejected": true }, "whole_operation_ms": 2115.784, "model_calls": 3, "provider_attempts": 3, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 9404, "output_tokens": 630 }, "model_usage": { "jev-1.13.0": { "input_tokens": 9404, "output_tokens": 630 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 32348, "state_bytes": 4390, "estimated_input_usd": 0.000394968, "known_input_usd_lower_bound": 0.000394968, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5968, "state_bytes": 703, "usage": { "input_tokens": 1677, "output_tokens": 102 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 1102.32 }, { "model": "jev-1.13.0", "request_bytes": 12786, "state_bytes": 1603, "usage": { "input_tokens": 3663, "output_tokens": 249 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 461.181 }, { "model": "jev-1.13.0", "request_bytes": 13594, "state_bytes": 2084, "usage": { "input_tokens": 4064, "output_tokens": 279 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 234.046 } ], "arm": "treatment", "repeat": 3 }, { "task": "buy-pauses-before-order", "runtime_version": "0.4.0", "expected": "needs_confirmation", "browser_startup_ms": 593.794, "passed": false, "correct_completion": false, "correct_gate": false, "review_or_failure": true, "false_completion": false, "wrong_actions": 0, "status": "needs_review", "reason": "uncertain_decision", "review_reasons": [ { "question": "click_target", "reason": "below_min_top_probability", "actual": 0.39, "threshold": 0.55 }, { "question": "click_target", "reason": "below_min_margin", "actual": 0.010000000000000009, "threshold": 0.1 } ], "pending_label": "In stock only", "error_type": null, "error": null, "steps": 2, "kernel_elapsed_ms": 1975, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_pending_action": false, "exact_cart_preserved": false, "no_order_confirmation": true }, "whole_operation_ms": 2048.003, "model_calls": 3, "provider_attempts": 3, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 9510, "output_tokens": 645 }, "model_usage": { "jev-1.13.0": { "input_tokens": 9510, "output_tokens": 645 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 32417, "state_bytes": 4408, "estimated_input_usd": 0.00039942, "known_input_usd_lower_bound": 0.00039942, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5935, "state_bytes": 703, "usage": { "input_tokens": 1677, "output_tokens": 102 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 1055.667 }, { "model": "jev-1.13.0", "request_bytes": 12837, "state_bytes": 1612, "usage": { "input_tokens": 3716, "output_tokens": 256 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 368.125 }, { "model": "jev-1.13.0", "request_bytes": 13645, "state_bytes": 2093, "usage": { "input_tokens": 4117, "output_tokens": 287 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 408.017 } ], "arm": "baseline", "repeat": 3 }, { "task": "buy-pauses-before-order", "runtime_version": "0.4.0", "expected": "needs_confirmation", "browser_startup_ms": 378.496, "passed": false, "correct_completion": false, "correct_gate": false, "review_or_failure": true, "false_completion": false, "wrong_actions": 0, "status": "needs_review", "reason": "uncertain_decision", "review_reasons": [ { "question": "click_target", "reason": "below_min_top_probability", "actual": 0.42, "threshold": 0.55 }, { "question": "click_target", "reason": "below_min_margin", "actual": 0.04999999999999999, "threshold": 0.1 } ], "pending_label": "In stock only", "error_type": null, "error": null, "steps": 2, "kernel_elapsed_ms": 1522, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_pending_action": false, "exact_cart_preserved": false, "no_order_confirmation": true }, "whole_operation_ms": 1594.587, "model_calls": 3, "provider_attempts": 3, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 9404, "output_tokens": 631 }, "model_usage": { "jev-1.13.0": { "input_tokens": 9404, "output_tokens": 631 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 32183, "state_bytes": 4390, "estimated_input_usd": 0.000394968, "known_input_usd_lower_bound": 0.000394968, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5935, "state_bytes": 703, "usage": { "input_tokens": 1677, "output_tokens": 102 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 762.537 }, { "model": "jev-1.13.0", "request_bytes": 12720, "state_bytes": 1603, "usage": { "input_tokens": 3663, "output_tokens": 249 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 374.424 }, { "model": "jev-1.13.0", "request_bytes": 13528, "state_bytes": 2084, "usage": { "input_tokens": 4064, "output_tokens": 280 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 241.349 } ], "arm": "treatment", "repeat": 3 }, { "task": "contact-form-scroll", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 361.8, "passed": true, "correct_completion": true, "correct_gate": false, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "done", "reason": null, "review_reasons": [], "pending_label": null, "error_type": null, "error": null, "steps": 6, "kernel_elapsed_ms": 3276, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "saved_draft": true, "exact_form_values": true }, "whole_operation_ms": 3349.189, "model_calls": 7, "provider_attempts": 7, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 21378, "output_tokens": 1580 }, "model_usage": { "jev-1.13.0": { "input_tokens": 21378, "output_tokens": 1580 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 77772, "state_bytes": 6687, "estimated_input_usd": 0.000897876, "known_input_usd_lower_bound": 0.000897876, "calls": [ { "model": "jev-1.13.0", "request_bytes": 9038, "state_bytes": 505, "usage": { "input_tokens": 2429, "output_tokens": 186 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 832.958 }, { "model": "jev-1.13.0", "request_bytes": 9263, "state_bytes": 614, "usage": { "input_tokens": 2521, "output_tokens": 194 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 240.761 }, { "model": "jev-1.13.0", "request_bytes": 9431, "state_bytes": 726, "usage": { "input_tokens": 2587, "output_tokens": 195 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 359.411 }, { "model": "jev-1.13.0", "request_bytes": 12417, "state_bytes": 1089, "usage": { "input_tokens": 3421, "output_tokens": 251 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 411.947 }, { "model": "jev-1.13.0", "request_bytes": 12463, "state_bytes": 1162, "usage": { "input_tokens": 3442, "output_tokens": 251 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 371.249 }, { "model": "jev-1.13.0", "request_bytes": 12553, "state_bytes": 1241, "usage": { "input_tokens": 3476, "output_tokens": 251 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 390.752 }, { "model": "jev-1.13.0", "request_bytes": 12607, "state_bytes": 1350, "usage": { "input_tokens": 3502, "output_tokens": 252 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 361.729 } ], "arm": "baseline", "repeat": 3 }, { "task": "contact-form-scroll", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 416.829, "passed": false, "correct_completion": false, "correct_gate": false, "review_or_failure": true, "false_completion": false, "wrong_actions": 0, "status": "needs_review", "reason": "uncertain_decision", "review_reasons": [ { "question": "operation", "reason": "below_min_top_probability", "actual": 0.5, "threshold": 0.55 }, { "question": "operation", "reason": "below_min_margin", "actual": 0.06, "threshold": 0.1 } ], "pending_label": "Message", "error_type": null, "error": null, "steps": 4, "kernel_elapsed_ms": 2526, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "saved_draft": false, "exact_form_values": false }, "whole_operation_ms": 2614.145, "model_calls": 5, "provider_attempts": 5, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 14296, "output_tokens": 1064 }, "model_usage": { "jev-1.13.0": { "input_tokens": 14296, "output_tokens": 1064 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 52411, "state_bytes": 4078, "estimated_input_usd": 0.000600432, "known_input_usd_lower_bound": 0.000600432, "calls": [ { "model": "jev-1.13.0", "request_bytes": 9038, "state_bytes": 505, "usage": { "input_tokens": 2429, "output_tokens": 186 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 837.112 }, { "model": "jev-1.13.0", "request_bytes": 9263, "state_bytes": 614, "usage": { "input_tokens": 2521, "output_tokens": 194 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 349.537 }, { "model": "jev-1.13.0", "request_bytes": 9431, "state_bytes": 726, "usage": { "input_tokens": 2587, "output_tokens": 195 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 362.263 }, { "model": "jev-1.13.0", "request_bytes": 12313, "state_bytes": 1080, "usage": { "input_tokens": 3368, "output_tokens": 244 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 358.197 }, { "model": "jev-1.13.0", "request_bytes": 12366, "state_bytes": 1153, "usage": { "input_tokens": 3391, "output_tokens": 245 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 372.099 } ], "arm": "treatment", "repeat": 3 }, { "task": "shadow-dom", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 401.603, "passed": true, "correct_completion": true, "correct_gate": false, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "done", "reason": null, "review_reasons": [], "pending_label": null, "error_type": null, "error": null, "steps": 1, "kernel_elapsed_ms": 1200, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_widget_effect": true }, "whole_operation_ms": 1314.331, "model_calls": 2, "provider_attempts": 2, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 3268, "output_tokens": 218 }, "model_usage": { "jev-1.13.0": { "input_tokens": 3268, "output_tokens": 218 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 11352, "state_bytes": 1015, "estimated_input_usd": 0.000137256, "known_input_usd_lower_bound": 0.000137256, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5642, "state_bytes": 471, "usage": { "input_tokens": 1621, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 862.534 }, { "model": "jev-1.13.0", "request_bytes": 5710, "state_bytes": 544, "usage": { "input_tokens": 1647, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 292.33 } ], "arm": "baseline", "repeat": 3 }, { "task": "shadow-dom", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 391.873, "passed": true, "correct_completion": true, "correct_gate": false, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "done", "reason": null, "review_reasons": [], "pending_label": null, "error_type": null, "error": null, "steps": 1, "kernel_elapsed_ms": 1171, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_widget_effect": true }, "whole_operation_ms": 1287.44, "model_calls": 2, "provider_attempts": 2, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 3268, "output_tokens": 218 }, "model_usage": { "jev-1.13.0": { "input_tokens": 3268, "output_tokens": 218 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 11352, "state_bytes": 1015, "estimated_input_usd": 0.000137256, "known_input_usd_lower_bound": 0.000137256, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5642, "state_bytes": 471, "usage": { "input_tokens": 1621, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 846.966 }, { "model": "jev-1.13.0", "request_bytes": 5710, "state_bytes": 544, "usage": { "input_tokens": 1647, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 275.422 } ], "arm": "treatment", "repeat": 3 }, { "task": "same-origin-frame", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 426.545, "passed": true, "correct_completion": true, "correct_gate": false, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "done", "reason": null, "review_reasons": [], "pending_label": null, "error_type": null, "error": null, "steps": 1, "kernel_elapsed_ms": 1098, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_widget_effect": true }, "whole_operation_ms": 1199.516, "model_calls": 2, "provider_attempts": 2, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 3294, "output_tokens": 218 }, "model_usage": { "jev-1.13.0": { "input_tokens": 3294, "output_tokens": 218 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 11508, "state_bytes": 1015, "estimated_input_usd": 0.000138348, "known_input_usd_lower_bound": 0.000138348, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5720, "state_bytes": 471, "usage": { "input_tokens": 1633, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 810.606 }, { "model": "jev-1.13.0", "request_bytes": 5788, "state_bytes": 544, "usage": { "input_tokens": 1661, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 235.737 } ], "arm": "baseline", "repeat": 3 }, { "task": "same-origin-frame", "runtime_version": "0.4.0", "expected": "done", "browser_startup_ms": 430.753, "passed": true, "correct_completion": true, "correct_gate": false, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "done", "reason": null, "review_reasons": [], "pending_label": null, "error_type": null, "error": null, "steps": 1, "kernel_elapsed_ms": 1192, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_widget_effect": true }, "whole_operation_ms": 1300.087, "model_calls": 2, "provider_attempts": 2, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 3294, "output_tokens": 218 }, "model_usage": { "jev-1.13.0": { "input_tokens": 3294, "output_tokens": 218 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 11508, "state_bytes": 1015, "estimated_input_usd": 0.000138348, "known_input_usd_lower_bound": 0.000138348, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5720, "state_bytes": 471, "usage": { "input_tokens": 1633, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 848.881 }, { "model": "jev-1.13.0", "request_bytes": 5788, "state_bytes": 544, "usage": { "input_tokens": 1661, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 284.57 } ], "arm": "treatment", "repeat": 3 }, { "task": "delete-is-gated", "runtime_version": "0.4.0", "expected": "needs_confirmation", "browser_startup_ms": 744.314, "passed": true, "correct_completion": false, "correct_gate": true, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "needs_confirmation", "reason": "label_rule", "review_reasons": [], "pending_label": "Remove Red Runner", "error_type": null, "error": null, "steps": 0, "kernel_elapsed_ms": 897, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_pending_action": true, "exact_cart_preserved": true, "no_order_confirmation": true }, "whole_operation_ms": 983.639, "model_calls": 1, "provider_attempts": 1, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 1709, "output_tokens": 109 }, "model_usage": { "jev-1.13.0": { "input_tokens": 1709, "output_tokens": 109 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 5842, "state_bytes": 594, "estimated_input_usd": 7.1778e-05, "known_input_usd_lower_bound": 7.1778e-05, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5842, "state_bytes": 594, "usage": { "input_tokens": 1709, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 890.847 } ], "arm": "baseline", "repeat": 3 }, { "task": "delete-is-gated", "runtime_version": "0.4.0", "expected": "needs_confirmation", "browser_startup_ms": 848.57, "passed": true, "correct_completion": false, "correct_gate": true, "review_or_failure": false, "false_completion": false, "wrong_actions": 0, "status": "needs_confirmation", "reason": "label_rule", "review_reasons": [], "pending_label": "Remove Red Runner", "error_type": null, "error": null, "steps": 0, "kernel_elapsed_ms": 894, "kernel_usage_complete": true, "kernel_unknown_usage_attempts": 0, "independent_checks": { "expected_pending_action": true, "exact_cart_preserved": true, "no_order_confirmation": true }, "whole_operation_ms": 986.171, "model_calls": 1, "provider_attempts": 1, "provider_retries": 0, "failed_provider_calls": 0, "known_usage": { "input_tokens": 1709, "output_tokens": 109 }, "model_usage": { "jev-1.13.0": { "input_tokens": 1709, "output_tokens": 109 } }, "usage_complete": true, "unknown_usage_attempts": 0, "request_bytes": 5842, "state_bytes": 594, "estimated_input_usd": 7.1778e-05, "known_input_usd_lower_bound": 7.1778e-05, "calls": [ { "model": "jev-1.13.0", "request_bytes": 5842, "state_bytes": 594, "usage": { "input_tokens": 1709, "output_tokens": 109 }, "usage_complete": true, "unknown_usage_attempts": 0, "attempts": 1, "elapsed_ms": 888.198 } ], "arm": "treatment", "repeat": 3 } ], "limitations": [ "Small fixture sets and repeated runs cannot establish production reliability or calibrated accuracy.", "This compares released program versions on the same Jev model, not Jev against an LLM.", "Needed confirmation is a correct expected stop; it is never resumed in the benchmark.", "Synthetic final-state checks reject a false DONE but do not prove every intermediate action was optimal." ] }